diff --git a/.server-stderr.log b/.server-stderr.log index 18334a3..a1dfcb8 100644 --- a/.server-stderr.log +++ b/.server-stderr.log @@ -1,81 +1,17 @@ -INFO: Started server process [32744] +INFO: Started server process [12700] INFO: Waiting for application startup. INFO: Application startup complete. INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit) ERROR: Exception in ASGI application Traceback (most recent call last): - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\uvicorn\protocols\http\httptools_impl.py", line 421, in run_asgi - result = await app( # type: ignore[func-returns-value] - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\uvicorn\middleware\proxy_headers.py", line 56, in __call__ - return await self.app(scope, receive, send) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\applications.py", line 1159, in __call__ - await super().__call__(scope, receive, send) - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\applications.py", line 90, in __call__ - await self.middleware_stack(scope, receive, send) - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\errors.py", line 186, in __call__ - raise exc - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\errors.py", line 164, in __call__ - await self.app(scope, receive, _send) - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\exceptions.py", line 63, in __call__ - await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 53, in wrapped_app - raise exc - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 42, in wrapped_app - await app(scope, receive, sender) - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\middleware\asyncexitstack.py", line 18, in __call__ - await self.app(scope, receive, send) - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 660, in __call__ - await self.middleware_stack(scope, receive, send) - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 680, in app - await route.handle(scope, receive, send) - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 276, in handle - await self.app(scope, receive, send) - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 134, in app - await wrap_app_handling_exceptions(app, request)(scope, receive, send) - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 53, in wrapped_app - raise exc - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 42, in wrapped_app - await app(scope, receive, sender) - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 120, in app - response = await f(request) - ^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 674, in app - raw_response = await run_endpoint_function( - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 330, in run_endpoint_function - return await run_in_threadpool(dependant.call, **values) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\concurrency.py", line 32, in run_in_threadpool - return await anyio.to_thread.run_sync(func) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\to_thread.py", line 63, in run_sync - return await get_async_backend().run_sync_in_worker_thread( - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\_backends\_asyncio.py", line 2518, in run_sync_in_worker_thread - return await future - ^^^^^^^^^^^^ - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\_backends\_asyncio.py", line 1002, in run - result = context.run(func, *args) - ^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\MyProject\AI\crawler_platform\app\api\routes.py", line 291, in reset_project - config = load_project_config(request.config_path) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\MyProject\AI\crawler_platform\app\config\loader.py", line 42, in load_project_config - data = _load_mapping(config_path) - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\MyProject\AI\crawler_platform\app\config\loader.py", line 57, in _load_mapping - text = path.read_text(encoding="utf-8") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\pathlib.py", line 1027, in read_text - with self.open(mode='r', encoding=encoding, errors=errors) as f: - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\pathlib.py", line 1013, in open - return io.open(self, mode, buffering, encoding, errors, newline) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -FileNotFoundError: [Errno 2] No such file or directory: 'configs\\_subscription.yaml' -ERROR: Exception in ASGI application + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1936, in _exec_single_context + self.dialect.do_executemany( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\default.py", line 949, in do_executemany + cursor.executemany(statement, parameters) +sqlite3.OperationalError: database is locked + +The above exception was the direct cause of the following exception: + Traceback (most recent call last): File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\uvicorn\protocols\http\httptools_impl.py", line 421, in run_asgi result = await app( # type: ignore[func-returns-value] @@ -132,19 +68,648 @@ Traceback (most recent call last): File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\_backends\_asyncio.py", line 1002, in run result = context.run(func, *args) ^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\MyProject\AI\crawler_platform\app\api\routes.py", line 476, in crawl_site - config = load_project_config(request.config_path) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\MyProject\AI\crawler_platform\app\config\loader.py", line 42, in load_project_config - data = _load_mapping(config_path) + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\api\routes.py", line 432, in knowledge_gaps + with session_scope(database_url) as session: + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\contextlib.py", line 144, in __exit__ + next(self.gen) + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\database\session.py", line 46, in session_scope + session.commit() + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 2030, in commit + trans.commit(_to_root=True) + File "", line 2, in commit + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\state_changes.py", line 137, in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 1311, in commit + self._prepare_impl() + File "", line 2, in _prepare_impl + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\state_changes.py", line 137, in _go + ret_value = fn(self, *arg, **kw) + ^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 1286, in _prepare_impl + self.session.flush() + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 4331, in flush + self._flush(objects) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 4466, in _flush + with util.safe_reraise(): + ^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\util\langhelpers.py", line 121, in __exit__ + raise exc_value.with_traceback(exc_tb) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 4427, in _flush + flush_context.execute() + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\unitofwork.py", line 466, in execute + rec.execute(self) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\unitofwork.py", line 642, in execute + util.preloaded.orm_persistence.save_obj( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\persistence.py", line 85, in save_obj + _emit_update_statements( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\persistence.py", line 912, in _emit_update_statements + c = connection.execute( + ^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1419, in execute + return meth( + ^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\sql\elements.py", line 527, in _execute_on_connection + return connection._execute_clauseelement( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1641, in _execute_clauseelement + ret = self._execute_context( + ^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1846, in _execute_context + return self._exec_single_context( ^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\MyProject\AI\crawler_platform\app\config\loader.py", line 57, in _load_mapping - text = path.read_text(encoding="utf-8") - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\pathlib.py", line 1027, in read_text - with self.open(mode='r', encoding=encoding, errors=errors) as f: - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\pathlib.py", line 1013, in open - return io.open(self, mode, buffering, encoding, errors, newline) - ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -FileNotFoundError: [Errno 2] No such file or directory: 'configs\\_subscription.yaml' + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1986, in _exec_single_context + self._handle_dbapi_exception( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 2363, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1936, in _exec_single_context + self.dialect.do_executemany( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\default.py", line 949, in do_executemany + cursor.executemany(statement, parameters) +sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) database is locked +[SQL: UPDATE knowledge_gaps SET updated_at=? WHERE knowledge_gaps.id = ?] +[parameters: [('2026-05-13 03:58:35.233275', 1), ('2026-05-13 03:58:35.234725', 2), ('2026-05-13 03:58:35.238062', 3), ('2026-05-13 03:58:35.239065', 4), ('2026-05-13 03:58:35.240068', 5), ('2026-05-13 03:58:35.240068', 6), ('2026-05-13 03:58:35.241077', 7), ('2026-05-13 03:58:35.245134', 8) ... displaying 10 of 32 total bound parameter sets ... ('2026-05-13 03:58:35.287923', 31), ('2026-05-13 03:58:35.287923', 32)]] +(Background on this error at: https://sqlalche.me/e/20/e3q8) +ERROR: Exception in ASGI application +Traceback (most recent call last): + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1936, in _exec_single_context + self.dialect.do_executemany( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\default.py", line 949, in do_executemany + cursor.executemany(statement, parameters) +sqlite3.OperationalError: database is locked + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\uvicorn\protocols\http\httptools_impl.py", line 421, in run_asgi + result = await app( # type: ignore[func-returns-value] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\uvicorn\middleware\proxy_headers.py", line 56, in __call__ + return await self.app(scope, receive, send) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\applications.py", line 1159, in __call__ + await super().__call__(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\applications.py", line 90, in __call__ + await self.middleware_stack(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\errors.py", line 186, in __call__ + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\errors.py", line 164, in __call__ + await self.app(scope, receive, _send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\exceptions.py", line 63, in __call__ + await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 53, in wrapped_app + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 42, in wrapped_app + await app(scope, receive, sender) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\middleware\asyncexitstack.py", line 18, in __call__ + await self.app(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 660, in __call__ + await self.middleware_stack(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 680, in app + await route.handle(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 276, in handle + await self.app(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 134, in app + await wrap_app_handling_exceptions(app, request)(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 53, in wrapped_app + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 42, in wrapped_app + await app(scope, receive, sender) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 120, in app + response = await f(request) + ^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 674, in app + raw_response = await run_endpoint_function( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 330, in run_endpoint_function + return await run_in_threadpool(dependant.call, **values) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\concurrency.py", line 32, in run_in_threadpool + return await anyio.to_thread.run_sync(func) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\to_thread.py", line 63, in run_sync + return await get_async_backend().run_sync_in_worker_thread( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\_backends\_asyncio.py", line 2518, in run_sync_in_worker_thread + return await future + ^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\_backends\_asyncio.py", line 1002, in run + result = context.run(func, *args) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\api\routes.py", line 434, in knowledge_gaps + return KnowledgeGapDetector(session).list_open(project.id, max(min(limit, 300), 1)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\ontology\gap_detector.py", line 27, in list_open + self.detect(project_id, limit=limit) + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\ontology\gap_detector.py", line 22, in detect + gaps.extend(self._underconnected_entity_gaps(project_id)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\ontology\gap_detector.py", line 119, in _underconnected_entity_gaps + self._upsert_gap( + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\ontology\gap_detector.py", line 188, in _upsert_gap + self.session.flush() + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 4331, in flush + self._flush(objects) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 4466, in _flush + with util.safe_reraise(): + ^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\util\langhelpers.py", line 121, in __exit__ + raise exc_value.with_traceback(exc_tb) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 4427, in _flush + flush_context.execute() + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\unitofwork.py", line 466, in execute + rec.execute(self) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\unitofwork.py", line 642, in execute + util.preloaded.orm_persistence.save_obj( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\persistence.py", line 85, in save_obj + _emit_update_statements( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\persistence.py", line 912, in _emit_update_statements + c = connection.execute( + ^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1419, in execute + return meth( + ^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\sql\elements.py", line 527, in _execute_on_connection + return connection._execute_clauseelement( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1641, in _execute_clauseelement + ret = self._execute_context( + ^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1846, in _execute_context + return self._exec_single_context( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1986, in _exec_single_context + self._handle_dbapi_exception( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 2363, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1936, in _exec_single_context + self.dialect.do_executemany( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\default.py", line 949, in do_executemany + cursor.executemany(statement, parameters) +sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) database is locked +[SQL: UPDATE knowledge_gaps SET updated_at=? WHERE knowledge_gaps.id = ?] +[parameters: [('2026-05-13 03:59:40.992293', 1), ('2026-05-13 03:59:40.992293', 2), ('2026-05-13 03:59:40.992293', 4), ('2026-05-13 03:59:40.993551', 5), ('2026-05-13 03:59:40.994556', 6), ('2026-05-13 03:59:40.995558', 7), ('2026-05-13 03:59:40.995558', 8), ('2026-05-13 03:59:40.997570', 9) ... displaying 10 of 30 total bound parameter sets ... ('2026-05-13 03:59:41.015370', 31), ('2026-05-13 03:59:41.015370', 32)]] +(Background on this error at: https://sqlalche.me/e/20/e3q8) +C:\Users\lasta\MyProject\AI\crawler_platform\app\core\crawler\site_crawler.py:409: SAWarning: Identity map already had an identity for (, (1,), None), replacing it with newly flushed object. Are there load operations occurring inside of an event handler within the flush? + self.repository.session.flush() +ERROR: Exception in ASGI application +Traceback (most recent call last): + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1967, in _exec_single_context + self.dialect.do_execute( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) +sqlite3.OperationalError: database is locked + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\uvicorn\protocols\http\httptools_impl.py", line 421, in run_asgi + result = await app( # type: ignore[func-returns-value] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\uvicorn\middleware\proxy_headers.py", line 56, in __call__ + return await self.app(scope, receive, send) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\applications.py", line 1159, in __call__ + await super().__call__(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\applications.py", line 90, in __call__ + await self.middleware_stack(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\errors.py", line 186, in __call__ + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\errors.py", line 164, in __call__ + await self.app(scope, receive, _send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\exceptions.py", line 63, in __call__ + await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 53, in wrapped_app + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 42, in wrapped_app + await app(scope, receive, sender) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\middleware\asyncexitstack.py", line 18, in __call__ + await self.app(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 660, in __call__ + await self.middleware_stack(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 680, in app + await route.handle(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 276, in handle + await self.app(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 134, in app + await wrap_app_handling_exceptions(app, request)(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 53, in wrapped_app + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 42, in wrapped_app + await app(scope, receive, sender) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 120, in app + response = await f(request) + ^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 674, in app + raw_response = await run_endpoint_function( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 330, in run_endpoint_function + return await run_in_threadpool(dependant.call, **values) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\concurrency.py", line 32, in run_in_threadpool + return await anyio.to_thread.run_sync(func) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\to_thread.py", line 63, in run_sync + return await get_async_backend().run_sync_in_worker_thread( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\_backends\_asyncio.py", line 2518, in run_sync_in_worker_thread + return await future + ^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\_backends\_asyncio.py", line 1002, in run + result = context.run(func, *args) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\api\routes.py", line 327, in reset_project + deleted = repo.reset_project_runtime_data(project.id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\database\repository.py", line 113, in reset_project_runtime_data + .delete(synchronize_session=False) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\query.py", line 3217, in delete + self.session.execute( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 2351, in execute + return self._execute_internal( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 2249, in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\bulk_persistence.py", line 2033, in orm_execute_statement + return super().orm_execute_statement( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\context.py", line 306, in orm_execute_statement + result = conn.execute( + ^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1419, in execute + return meth( + ^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\sql\elements.py", line 527, in _execute_on_connection + return connection._execute_clauseelement( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1641, in _execute_clauseelement + ret = self._execute_context( + ^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1846, in _execute_context + return self._exec_single_context( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1986, in _exec_single_context + self._handle_dbapi_exception( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 2363, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1967, in _exec_single_context + self.dialect.do_execute( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) +sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) database is locked +[SQL: DELETE FROM feedback_logs WHERE feedback_logs.project_id = ?] +[parameters: (1,)] +(Background on this error at: https://sqlalche.me/e/20/e3q8) +ERROR: Exception in ASGI application +Traceback (most recent call last): + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1967, in _exec_single_context + self.dialect.do_execute( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) +sqlite3.OperationalError: database is locked + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\uvicorn\protocols\http\httptools_impl.py", line 421, in run_asgi + result = await app( # type: ignore[func-returns-value] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\uvicorn\middleware\proxy_headers.py", line 56, in __call__ + return await self.app(scope, receive, send) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\applications.py", line 1159, in __call__ + await super().__call__(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\applications.py", line 90, in __call__ + await self.middleware_stack(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\errors.py", line 186, in __call__ + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\errors.py", line 164, in __call__ + await self.app(scope, receive, _send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\exceptions.py", line 63, in __call__ + await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 53, in wrapped_app + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 42, in wrapped_app + await app(scope, receive, sender) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\middleware\asyncexitstack.py", line 18, in __call__ + await self.app(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 660, in __call__ + await self.middleware_stack(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 680, in app + await route.handle(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 276, in handle + await self.app(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 134, in app + await wrap_app_handling_exceptions(app, request)(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 53, in wrapped_app + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 42, in wrapped_app + await app(scope, receive, sender) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 120, in app + response = await f(request) + ^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 674, in app + raw_response = await run_endpoint_function( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 330, in run_endpoint_function + return await run_in_threadpool(dependant.call, **values) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\concurrency.py", line 32, in run_in_threadpool + return await anyio.to_thread.run_sync(func) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\to_thread.py", line 63, in run_sync + return await get_async_backend().run_sync_in_worker_thread( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\_backends\_asyncio.py", line 2518, in run_sync_in_worker_thread + return await future + ^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\_backends\_asyncio.py", line 1002, in run + result = context.run(func, *args) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\api\routes.py", line 327, in reset_project + deleted = repo.reset_project_runtime_data(project.id) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\database\repository.py", line 113, in reset_project_runtime_data + .delete(synchronize_session=False) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\query.py", line 3217, in delete + self.session.execute( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 2351, in execute + return self._execute_internal( + ^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 2249, in _execute_internal + result: Result[Any] = compile_state_cls.orm_execute_statement( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\bulk_persistence.py", line 2033, in orm_execute_statement + return super().orm_execute_statement( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\context.py", line 306, in orm_execute_statement + result = conn.execute( + ^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1419, in execute + return meth( + ^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\sql\elements.py", line 527, in _execute_on_connection + return connection._execute_clauseelement( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1641, in _execute_clauseelement + ret = self._execute_context( + ^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1846, in _execute_context + return self._exec_single_context( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1986, in _exec_single_context + self._handle_dbapi_exception( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 2363, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1967, in _exec_single_context + self.dialect.do_execute( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\default.py", line 952, in do_execute + cursor.execute(statement, parameters) +sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) database is locked +[SQL: DELETE FROM feedback_logs WHERE feedback_logs.project_id = ?] +[parameters: (1,)] +(Background on this error at: https://sqlalche.me/e/20/e3q8) +ERROR: Exception in ASGI application +Traceback (most recent call last): + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1936, in _exec_single_context + self.dialect.do_executemany( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\default.py", line 949, in do_executemany + cursor.executemany(statement, parameters) +sqlite3.OperationalError: database is locked + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\uvicorn\protocols\http\httptools_impl.py", line 421, in run_asgi + result = await app( # type: ignore[func-returns-value] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\uvicorn\middleware\proxy_headers.py", line 56, in __call__ + return await self.app(scope, receive, send) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\applications.py", line 1159, in __call__ + await super().__call__(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\applications.py", line 90, in __call__ + await self.middleware_stack(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\errors.py", line 186, in __call__ + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\errors.py", line 164, in __call__ + await self.app(scope, receive, _send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\exceptions.py", line 63, in __call__ + await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 53, in wrapped_app + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 42, in wrapped_app + await app(scope, receive, sender) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\middleware\asyncexitstack.py", line 18, in __call__ + await self.app(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 660, in __call__ + await self.middleware_stack(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 680, in app + await route.handle(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 276, in handle + await self.app(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 134, in app + await wrap_app_handling_exceptions(app, request)(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 53, in wrapped_app + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 42, in wrapped_app + await app(scope, receive, sender) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 120, in app + response = await f(request) + ^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 674, in app + raw_response = await run_endpoint_function( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 330, in run_endpoint_function + return await run_in_threadpool(dependant.call, **values) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\concurrency.py", line 32, in run_in_threadpool + return await anyio.to_thread.run_sync(func) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\to_thread.py", line 63, in run_sync + return await get_async_backend().run_sync_in_worker_thread( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\_backends\_asyncio.py", line 2518, in run_sync_in_worker_thread + return await future + ^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\_backends\_asyncio.py", line 1002, in run + result = context.run(func, *args) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\api\routes.py", line 434, in knowledge_gaps + return KnowledgeGapDetector(session).list_open(project.id, max(min(limit, 300), 1)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\ontology\gap_detector.py", line 27, in list_open + self.detect(project_id, limit=limit) + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\ontology\gap_detector.py", line 22, in detect + gaps.extend(self._underconnected_entity_gaps(project_id)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\ontology\gap_detector.py", line 119, in _underconnected_entity_gaps + self._upsert_gap( + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\ontology\gap_detector.py", line 188, in _upsert_gap + self.session.flush() + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 4331, in flush + self._flush(objects) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 4466, in _flush + with util.safe_reraise(): + ^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\util\langhelpers.py", line 121, in __exit__ + raise exc_value.with_traceback(exc_tb) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 4427, in _flush + flush_context.execute() + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\unitofwork.py", line 466, in execute + rec.execute(self) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\unitofwork.py", line 642, in execute + util.preloaded.orm_persistence.save_obj( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\persistence.py", line 85, in save_obj + _emit_update_statements( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\persistence.py", line 912, in _emit_update_statements + c = connection.execute( + ^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1419, in execute + return meth( + ^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\sql\elements.py", line 527, in _execute_on_connection + return connection._execute_clauseelement( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1641, in _execute_clauseelement + ret = self._execute_context( + ^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1846, in _execute_context + return self._exec_single_context( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1986, in _exec_single_context + self._handle_dbapi_exception( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 2363, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1936, in _exec_single_context + self.dialect.do_executemany( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\default.py", line 949, in do_executemany + cursor.executemany(statement, parameters) +sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) database is locked +[SQL: UPDATE knowledge_gaps SET updated_at=? WHERE knowledge_gaps.id = ?] +[parameters: [('2026-05-13 04:00:37.529636', 2), ('2026-05-13 04:00:37.530642', 4), ('2026-05-13 04:00:37.530642', 5), ('2026-05-13 04:00:37.530642', 6), ('2026-05-13 04:00:37.530642', 7), ('2026-05-13 04:00:37.530642', 8), ('2026-05-13 04:00:37.531648', 9), ('2026-05-13 04:00:37.531648', 10) ... displaying 10 of 26 total bound parameter sets ... ('2026-05-13 04:00:37.535699', 31), ('2026-05-13 04:00:37.535699', 32)]] +(Background on this error at: https://sqlalche.me/e/20/e3q8) +ERROR: Exception in ASGI application +Traceback (most recent call last): + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1936, in _exec_single_context + self.dialect.do_executemany( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\default.py", line 949, in do_executemany + cursor.executemany(statement, parameters) +sqlite3.OperationalError: database is locked + +The above exception was the direct cause of the following exception: + +Traceback (most recent call last): + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\uvicorn\protocols\http\httptools_impl.py", line 421, in run_asgi + result = await app( # type: ignore[func-returns-value] + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\uvicorn\middleware\proxy_headers.py", line 56, in __call__ + return await self.app(scope, receive, send) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\applications.py", line 1159, in __call__ + await super().__call__(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\applications.py", line 90, in __call__ + await self.middleware_stack(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\errors.py", line 186, in __call__ + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\errors.py", line 164, in __call__ + await self.app(scope, receive, _send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\middleware\exceptions.py", line 63, in __call__ + await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 53, in wrapped_app + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 42, in wrapped_app + await app(scope, receive, sender) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\middleware\asyncexitstack.py", line 18, in __call__ + await self.app(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 660, in __call__ + await self.middleware_stack(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 680, in app + await route.handle(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\routing.py", line 276, in handle + await self.app(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 134, in app + await wrap_app_handling_exceptions(app, request)(scope, receive, send) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 53, in wrapped_app + raise exc + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\_exception_handler.py", line 42, in wrapped_app + await app(scope, receive, sender) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 120, in app + response = await f(request) + ^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 674, in app + raw_response = await run_endpoint_function( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\fastapi\routing.py", line 330, in run_endpoint_function + return await run_in_threadpool(dependant.call, **values) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\starlette\concurrency.py", line 32, in run_in_threadpool + return await anyio.to_thread.run_sync(func) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\to_thread.py", line 63, in run_sync + return await get_async_backend().run_sync_in_worker_thread( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\_backends\_asyncio.py", line 2518, in run_sync_in_worker_thread + return await future + ^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\anyio\_backends\_asyncio.py", line 1002, in run + result = context.run(func, *args) + ^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\api\routes.py", line 434, in knowledge_gaps + return KnowledgeGapDetector(session).list_open(project.id, max(min(limit, 300), 1)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\ontology\gap_detector.py", line 27, in list_open + self.detect(project_id, limit=limit) + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\ontology\gap_detector.py", line 22, in detect + gaps.extend(self._underconnected_entity_gaps(project_id)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\ontology\gap_detector.py", line 119, in _underconnected_entity_gaps + self._upsert_gap( + File "C:\Users\lasta\MyProject\AI\crawler_platform\app\core\ontology\gap_detector.py", line 188, in _upsert_gap + self.session.flush() + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 4331, in flush + self._flush(objects) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 4466, in _flush + with util.safe_reraise(): + ^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\util\langhelpers.py", line 121, in __exit__ + raise exc_value.with_traceback(exc_tb) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\session.py", line 4427, in _flush + flush_context.execute() + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\unitofwork.py", line 466, in execute + rec.execute(self) + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\unitofwork.py", line 642, in execute + util.preloaded.orm_persistence.save_obj( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\persistence.py", line 85, in save_obj + _emit_update_statements( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\orm\persistence.py", line 912, in _emit_update_statements + c = connection.execute( + ^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1419, in execute + return meth( + ^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\sql\elements.py", line 527, in _execute_on_connection + return connection._execute_clauseelement( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1641, in _execute_clauseelement + ret = self._execute_context( + ^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1846, in _execute_context + return self._exec_single_context( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1986, in _exec_single_context + self._handle_dbapi_exception( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 2363, in _handle_dbapi_exception + raise sqlalchemy_exception.with_traceback(exc_info[2]) from e + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\base.py", line 1936, in _exec_single_context + self.dialect.do_executemany( + File "C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\Lib\site-packages\sqlalchemy\engine\default.py", line 949, in do_executemany + cursor.executemany(statement, parameters) +sqlalchemy.exc.OperationalError: (sqlite3.OperationalError) database is locked +[SQL: UPDATE knowledge_gaps SET updated_at=? WHERE knowledge_gaps.id = ?] +[parameters: [('2026-05-13 04:00:45.068406', 2), ('2026-05-13 04:00:45.069451', 4), ('2026-05-13 04:00:45.069451', 5), ('2026-05-13 04:00:45.069451', 6), ('2026-05-13 04:00:45.069451', 7), ('2026-05-13 04:00:45.069451', 8), ('2026-05-13 04:00:45.070461', 9), ('2026-05-13 04:00:45.070461', 10) ... displaying 10 of 26 total bound parameter sets ... ('2026-05-13 04:00:45.074511', 31), ('2026-05-13 04:00:45.074511', 32)]] +(Background on this error at: https://sqlalche.me/e/20/e3q8) diff --git a/.server-stdout.log b/.server-stdout.log index b00e50c..94a08cc 100644 --- a/.server-stdout.log +++ b/.server-stdout.log @@ -1,365 +1,227 @@ -INFO: 127.0.0.1:61567 - "GET / HTTP/1.1" 200 OK -INFO: 127.0.0.1:61567 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:61567 - "GET /projects/perfume_subscription/search?q=per HTTP/1.1" 200 OK -INFO: 127.0.0.1:65199 - "GET / HTTP/1.1" 200 OK -INFO: 127.0.0.1:65199 - "GET /static/assets/index-fQcYiMpd.js HTTP/1.1" 200 OK -INFO: 127.0.0.1:49213 - "GET /static/assets/index-BkBg_FAU.css HTTP/1.1" 200 OK -INFO: 127.0.0.1:54302 - "GET /favicon.ico HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:49213 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:49213 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:49213 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:62067 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:51685 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:49213 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:62067 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:51685 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:54302 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:62067 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:49213 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:54302 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:51685 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:49213 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:62067 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:58722 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:65199 - "POST /extractors/models HTTP/1.1" 200 OK -INFO: 127.0.0.1:62372 - "POST /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:62372 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:62372 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:62372 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:51459 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:49291 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:61989 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:59819 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:62372 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:51459 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:49291 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:62372 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:61989 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:59819 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:49291 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:51459 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:58522 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:59695 - "POST /projects/reset HTTP/1.1" 500 Internal Server Error -INFO: 127.0.0.1:49890 - "POST /crawl-site HTTP/1.1" 500 Internal Server Error -INFO: 127.0.0.1:64060 - "GET / HTTP/1.1" 200 OK -INFO: 127.0.0.1:62711 - "GET /static/assets/index-BkBg_FAU.css HTTP/1.1" 200 OK -INFO: 127.0.0.1:64060 - "GET /static/assets/index-fQcYiMpd.js HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /favicon.ico HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:62711 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:62711 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:62711 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53699 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:62711 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:53699 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:62711 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:53699 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60854 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:60854 - "GET / HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /static/assets/index-BkBg_FAU.css HTTP/1.1" 200 OK -INFO: 127.0.0.1:60854 - "GET /static/assets/index-fQcYiMpd.js HTTP/1.1" 200 OK -INFO: 127.0.0.1:53699 - "GET /favicon.ico HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:54258 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54088 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54088 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:53699 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:53699 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:54088 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:62711 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:62711 - "GET / HTTP/1.1" 200 OK -INFO: 127.0.0.1:62711 - "GET /static/assets/index-fQcYiMpd.js HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /static/assets/index-BkBg_FAU.css HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /favicon.ico HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:54258 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:54088 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:64247 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54088 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:64247 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:54088 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:64247 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54088 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53699 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53699 - "GET / HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /static/assets/index-BkBg_FAU.css HTTP/1.1" 200 OK -INFO: 127.0.0.1:53699 - "GET /static/assets/index-fQcYiMpd.js HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /favicon.ico HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:55839 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:54088 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:54088 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:54088 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:64247 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54088 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:54088 - "GET / HTTP/1.1" 200 OK -INFO: 127.0.0.1:54088 - "GET /static/assets/index-fQcYiMpd.js HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /static/assets/index-BkBg_FAU.css HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /favicon.ico HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:64247 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:64247 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:64247 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:64247 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET / HTTP/1.1" 200 OK -INFO: 127.0.0.1:55839 - "GET /static/assets/index-fQcYiMpd.js HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /static/assets/index-BkBg_FAU.css HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /favicon.ico HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:63671 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:64247 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:64247 - "GET / HTTP/1.1" 200 OK -INFO: 127.0.0.1:64247 - "GET /static/assets/index-fQcYiMpd.js HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /static/assets/index-BkBg_FAU.css HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /favicon.ico HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:53854 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET / HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /static/assets/index-BkBg_FAU.css HTTP/1.1" 200 OK -INFO: 127.0.0.1:54258 - "GET /static/assets/index-fQcYiMpd.js HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /favicon.ico HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:53854 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET / HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /static/assets/index-BkBg_FAU.css HTTP/1.1" 200 OK -INFO: 127.0.0.1:57675 - "GET /static/assets/index-fQcYiMpd.js HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /favicon.ico HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:51099 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:56792 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:56792 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:56792 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET / HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /static/assets/index-BkBg_FAU.css HTTP/1.1" 200 OK -INFO: 127.0.0.1:63671 - "GET /static/assets/index-fQcYiMpd.js HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /favicon.ico HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:63184 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:53207 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:53207 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:53207 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53207 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:56792 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:56792 - "GET / HTTP/1.1" 200 OK -INFO: 127.0.0.1:56792 - "GET /static/assets/index-fQcYiMpd.js HTTP/1.1" 200 OK -INFO: 127.0.0.1:53207 - "GET /static/assets/index-BkBg_FAU.css HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /favicon.ico HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:53207 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:53207 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:53207 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:54482 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54482 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:53207 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:54482 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:54482 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:53207 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET / HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "GET /static/assets/index-fQcYiMpd.js HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /static/assets/index-BkBg_FAU.css HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /favicon.ico HTTP/1.1" 404 Not Found -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:53207 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:61762 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53207 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:61762 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:53207 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:61762 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:51099 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63184 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:53207 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54482 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:53854 - "POST /extractors/models HTTP/1.1" 200 OK -INFO: 127.0.0.1:63984 - "POST /projects/reset HTTP/1.1" 200 OK -INFO: 127.0.0.1:63984 - "GET /projects HTTP/1.1" 200 OK -INFO: 127.0.0.1:63984 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK -INFO: 127.0.0.1:63984 - "GET /ontology/perfume HTTP/1.1" 200 OK -INFO: 127.0.0.1:57194 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:49791 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:61562 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63984 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:57194 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK +INFO: 127.0.0.1:55654 - "GET /health HTTP/1.1" 200 OK +INFO: 127.0.0.1:55656 - "GET /health HTTP/1.1" 200 OK +INFO: 127.0.0.1:55658 - "POST /crawl HTTP/1.1" 200 OK +INFO: 127.0.0.1:60702 - "GET /health HTTP/1.1" 200 OK +INFO: 127.0.0.1:51909 - "GET / HTTP/1.1" 200 OK +INFO: 127.0.0.1:63667 - "GET /static/assets/index-BkBg_FAU.css HTTP/1.1" 200 OK +INFO: 127.0.0.1:51909 - "GET /static/assets/index-BsoXaTIL.js HTTP/1.1" 200 OK +INFO: 127.0.0.1:60212 - "GET /favicon.ico HTTP/1.1" 404 Not Found +INFO: 127.0.0.1:63667 - "GET /projects HTTP/1.1" 200 OK +INFO: 127.0.0.1:63667 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK +INFO: 127.0.0.1:63667 - "GET /ontology/perfume HTTP/1.1" 200 OK +INFO: 127.0.0.1:52091 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:63120 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:52091 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK +INFO: 127.0.0.1:63667 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK +INFO: 127.0.0.1:60212 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK +INFO: 127.0.0.1:52091 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK +INFO: 127.0.0.1:52091 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK +INFO: 127.0.0.1:63120 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK +INFO: 127.0.0.1:63667 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:52091 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK +INFO: 127.0.0.1:63120 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60212 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK +INFO: 127.0.0.1:58507 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:51909 - "POST /extractors/models HTTP/1.1" 200 OK +INFO: 127.0.0.1:51909 - "POST /projects/reset HTTP/1.1" 200 OK +INFO: 127.0.0.1:51909 - "GET /projects HTTP/1.1" 200 OK +INFO: 127.0.0.1:51909 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK +INFO: 127.0.0.1:51909 - "GET /ontology/perfume HTTP/1.1" 200 OK +INFO: 127.0.0.1:60212 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:52091 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:63667 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60212 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK +INFO: 127.0.0.1:51909 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK +INFO: 127.0.0.1:52091 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK +INFO: 127.0.0.1:63667 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60212 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK +INFO: 127.0.0.1:51909 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK +INFO: 127.0.0.1:52091 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK +INFO: 127.0.0.1:63667 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK +INFO: 127.0.0.1:63120 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:58507 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK +INFO: 127.0.0.1:59175 - "POST /crawl-site HTTP/1.1" 200 OK +INFO: 127.0.0.1:59175 - "GET /crawl-site/jobs/1 HTTP/1.1" 200 OK +INFO: 127.0.0.1:59175 - "GET /crawl-site/jobs/1 HTTP/1.1" 200 OK +INFO: 127.0.0.1:59175 - "GET /crawl-site/jobs/1 HTTP/1.1" 200 OK +INFO: 127.0.0.1:59175 - "GET /crawl-site/jobs/1 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60573 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60167 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:54345 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:59175 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK +INFO: 127.0.0.1:53330 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK +INFO: 127.0.0.1:54345 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60573 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK +INFO: 127.0.0.1:59175 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60167 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK +INFO: 127.0.0.1:53330 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK +INFO: 127.0.0.1:54345 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK +INFO: 127.0.0.1:60573 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK +INFO: 127.0.0.1:62512 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 500 Internal Server Error +INFO: 127.0.0.1:54311 - "GET /crawl-site/jobs/1 HTTP/1.1" 200 OK +INFO: 127.0.0.1:61649 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:65400 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:64587 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK +INFO: 127.0.0.1:61649 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK +INFO: 127.0.0.1:54311 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK +INFO: 127.0.0.1:58516 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK +INFO: 127.0.0.1:65400 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK +INFO: 127.0.0.1:54311 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK +INFO: 127.0.0.1:64587 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:58516 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK +INFO: 127.0.0.1:65400 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK +INFO: 127.0.0.1:61649 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK +INFO: 127.0.0.1:54492 - "GET /health HTTP/1.1" 200 OK +INFO: 127.0.0.1:54231 - "GET / HTTP/1.1" 200 OK +INFO: 127.0.0.1:51157 - "GET /projects HTTP/1.1" 200 OK +INFO: 127.0.0.1:51157 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK +INFO: 127.0.0.1:51157 - "GET /ontology/perfume HTTP/1.1" 200 OK +INFO: 127.0.0.1:54317 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:64933 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK INFO: 127.0.0.1:62007 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:49791 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:57194 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:63984 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:61562 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:62007 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:49791 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:61260 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:52382 - "POST /crawl-site HTTP/1.1" 200 OK -INFO: 127.0.0.1:52382 - "GET /crawl-site/jobs/1 HTTP/1.1" 200 OK -INFO: 127.0.0.1:52382 - "GET /crawl-site/jobs/1 HTTP/1.1" 200 OK -INFO: 127.0.0.1:62003 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:59199 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:52496 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:62003 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:52382 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:52496 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:59199 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:59933 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:62003 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:52496 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:59199 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:52382 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:63677 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:64337 - "POST /crawl-site HTTP/1.1" 200 OK -INFO: 127.0.0.1:64337 - "GET /crawl-site/jobs/3 HTTP/1.1" 200 OK -INFO: 127.0.0.1:64337 - "GET /crawl-site/jobs/3 HTTP/1.1" 200 OK -INFO: 127.0.0.1:64337 - "GET /crawl-site/jobs/3 HTTP/1.1" 200 OK -INFO: 127.0.0.1:64337 - "GET /crawl-site/jobs/3 HTTP/1.1" 200 OK -INFO: 127.0.0.1:64337 - "GET /crawl-site/jobs/3 HTTP/1.1" 200 OK -INFO: 127.0.0.1:64337 - "GET /crawl-site/jobs/3 HTTP/1.1" 200 OK -INFO: 127.0.0.1:57632 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:64609 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:65462 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:64337 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:57632 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:64609 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:65462 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:57632 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:64337 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:50388 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:64609 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:65462 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:57076 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:57076 - "GET /crawl-site/jobs/3 HTTP/1.1" 200 OK -INFO: 127.0.0.1:61272 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:52325 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63194 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:61272 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:52325 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:63194 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:57076 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:63194 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:52325 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:57076 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:61272 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:50100 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:63550 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63550 - "GET /crawl-site/jobs/3 HTTP/1.1" 200 OK -INFO: 127.0.0.1:52790 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54997 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:58392 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63550 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK -INFO: 127.0.0.1:52790 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK -INFO: 127.0.0.1:58392 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK -INFO: 127.0.0.1:54997 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK -INFO: 127.0.0.1:55216 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK -INFO: 127.0.0.1:52790 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK -INFO: 127.0.0.1:58392 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK -INFO: 127.0.0.1:54997 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK -INFO: 127.0.0.1:63550 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK -INFO: 127.0.0.1:64403 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:54317 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK +INFO: 127.0.0.1:51157 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK +INFO: 127.0.0.1:62007 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK +INFO: 127.0.0.1:64933 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK +INFO: 127.0.0.1:54317 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:62007 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK +INFO: 127.0.0.1:51157 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK +INFO: 127.0.0.1:54317 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK +INFO: 127.0.0.1:64933 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK +INFO: 127.0.0.1:54231 - "POST /extractors/models HTTP/1.1" 200 OK +INFO: 127.0.0.1:64937 - "GET /health HTTP/1.1" 200 OK +INFO: 127.0.0.1:51597 - "GET /projects HTTP/1.1" 200 OK +INFO: 127.0.0.1:51597 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK +INFO: 127.0.0.1:51597 - "GET /ontology/perfume HTTP/1.1" 200 OK +INFO: 127.0.0.1:55174 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:56473 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55174 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK +INFO: 127.0.0.1:64026 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK +INFO: 127.0.0.1:56473 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK +INFO: 127.0.0.1:51597 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK +INFO: 127.0.0.1:64026 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55174 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK +INFO: 127.0.0.1:51597 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK +INFO: 127.0.0.1:56473 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK +INFO: 127.0.0.1:64026 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK +INFO: 127.0.0.1:55174 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK +INFO: 127.0.0.1:57269 - "POST /extractors/models HTTP/1.1" 200 OK +INFO: 127.0.0.1:61245 - "GET / HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET /projects HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET /ontology/perfume HTTP/1.1" 200 OK +INFO: 127.0.0.1:51571 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK +INFO: 127.0.0.1:51571 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK +INFO: 127.0.0.1:51571 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:51571 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK +INFO: 127.0.0.1:51571 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK +INFO: 127.0.0.1:51571 - "GET / HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /ontology/perfume HTTP/1.1" 200 OK +INFO: 127.0.0.1:57549 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK +INFO: 127.0.0.1:57549 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK +INFO: 127.0.0.1:57549 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK +INFO: 127.0.0.1:57549 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK +INFO: 127.0.0.1:57549 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK +INFO: 127.0.0.1:57549 - "GET / HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /ontology/perfume HTTP/1.1" 200 OK +INFO: 127.0.0.1:54868 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK +INFO: 127.0.0.1:54868 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK +INFO: 127.0.0.1:54868 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK +INFO: 127.0.0.1:54868 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60418 - "GET / HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /ontology/perfume HTTP/1.1" 200 OK +INFO: 127.0.0.1:61169 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:54868 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK +INFO: 127.0.0.1:54868 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK +INFO: 127.0.0.1:61169 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK +INFO: 127.0.0.1:54868 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK +INFO: 127.0.0.1:61169 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK +INFO: 127.0.0.1:54868 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK +INFO: 127.0.0.1:61169 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK +INFO: 127.0.0.1:61169 - "GET / HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /ontology/perfume HTTP/1.1" 200 OK +INFO: 127.0.0.1:54868 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK +INFO: 127.0.0.1:55778 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK +INFO: 127.0.0.1:54868 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55778 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK +INFO: 127.0.0.1:54868 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK +INFO: 127.0.0.1:54868 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK +INFO: 127.0.0.1:55778 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55978 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK +INFO: 127.0.0.1:54868 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK +INFO: 127.0.0.1:61169 - "POST /extractors/models HTTP/1.1" 200 OK +INFO: 127.0.0.1:55246 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 500 Internal Server Error +INFO: 127.0.0.1:54063 - "GET /crawl-site/jobs/1 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60064 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:54063 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK +INFO: 127.0.0.1:53121 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK +INFO: 127.0.0.1:54063 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60064 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:53121 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK +INFO: 127.0.0.1:54063 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK +INFO: 127.0.0.1:60064 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:53121 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60064 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK +INFO: 127.0.0.1:53121 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK +INFO: 127.0.0.1:54063 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK +INFO: 127.0.0.1:61169 - "POST /projects/reset HTTP/1.1" 200 OK +INFO: 127.0.0.1:61169 - "GET /projects HTTP/1.1" 200 OK +INFO: 127.0.0.1:61169 - "GET /projects/perfume_subscription HTTP/1.1" 200 OK +INFO: 127.0.0.1:61169 - "GET /ontology/perfume HTTP/1.1" 200 OK +INFO: 127.0.0.1:60812 - "GET /projects/perfume_subscription/ontology/proposals?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:53523 - "GET /projects/perfume_subscription/ontology/triples?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55554 - "GET /projects/perfume_subscription/ontology/registry HTTP/1.1" 200 OK +INFO: 127.0.0.1:61169 - "GET /projects/perfume_subscription/pipeline HTTP/1.1" 200 OK +INFO: 127.0.0.1:53523 - "GET /projects/perfume_subscription/claims?limit=200&include_candidates=true HTTP/1.1" 200 OK +INFO: 127.0.0.1:60812 - "GET /projects/perfume_subscription/entities?limit=100 HTTP/1.1" 200 OK +INFO: 127.0.0.1:60812 - "GET /projects/perfume_subscription/graph/query?kind=trend_summary HTTP/1.1" 200 OK +INFO: 127.0.0.1:53523 - "GET /projects/perfume_subscription/research/sessions?limit=20 HTTP/1.1" 200 OK +INFO: 127.0.0.1:55554 - "GET /projects/perfume_subscription/claims?limit=200&status=rejected HTTP/1.1" 200 OK +INFO: 127.0.0.1:60812 - "GET /projects/perfume_subscription/recommendation-tags HTTP/1.1" 200 OK +INFO: 127.0.0.1:53523 - "GET /projects/perfume_subscription/graph/neighborhood?limit=300 HTTP/1.1" 200 OK +INFO: 127.0.0.1:61169 - "GET /projects/perfume_subscription/extraction-logs?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:54030 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 200 OK +INFO: 127.0.0.1:54030 - "GET /crawl-site/jobs/1 HTTP/1.1" 404 Not Found +INFO: 127.0.0.1:57352 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 500 Internal Server Error +INFO: 127.0.0.1:54030 - "GET /crawl-site/jobs/1 HTTP/1.1" 404 Not Found +INFO: 127.0.0.1:54030 - "GET /crawl-site/jobs/1 HTTP/1.1" 404 Not Found +INFO: 127.0.0.1:54868 - "GET /projects/perfume_subscription/knowledge-gaps?limit=50 HTTP/1.1" 500 Internal Server Error +INFO: 127.0.0.1:54030 - "GET /crawl-site/jobs/1 HTTP/1.1" 404 Not Found +INFO: 127.0.0.1:54030 - "GET /crawl-site/jobs/1 HTTP/1.1" 404 Not Found diff --git a/Instructor_분석_및_기능명세.md b/Instructor_분석_및_기능명세.md new file mode 100644 index 0000000..6ae8276 --- /dev/null +++ b/Instructor_분석_및_기능명세.md @@ -0,0 +1,586 @@ +# Instructor 분석 및 범용 온톨로지 구축 플랫폼 기능명세 - 2순위 검토 + +## 1. 분석 대상 + +- 원본 경로: `C:\Users\lasta\MyProject\AI\참고\instructor-main` +- 프로젝트명: `instructor` +- 확인 버전: `1.15.1` +- 라이선스: MIT +- 언어/런타임: Python `>=3.9,<4.0` +- 성격: LLM 응답을 Pydantic 모델로 강제 변환하고 검증하는 구조화 출력 라이브러리 +- 핵심 가치: 자연어/문서/이미지 입력에서 엔티티, 관계, 속성, 근거를 안정적인 JSON/Pydantic 객체로 추출 + +Instructor는 크롤러나 온톨로지 저장소가 아니라, LLM 기반 추출 단계의 신뢰성 레이어다. 범용 온톨로지 구축 플랫폼에서는 “웹/문서에서 수집한 비정형 텍스트를 명세된 스키마로 추출하고, 검증 실패 시 자동 재질문하며, 결과를 typed 객체로 돌려주는 모듈”로 거의 원형 그대로 사용할 수 있다. + +## 2. 프로젝트 구조 요약 + +```text +instructor-main/ + instructor/ + __init__.py # 공개 API export + auto_client.py # provider/model 문자열 기반 자동 클라이언트 생성 + mode.py # provider별 응답 처리 모드 enum + core/ + client.py # Instructor/AsyncInstructor 래퍼 API + patch.py # provider create() 함수 monkey patch + retry.py # tenacity 기반 재시도 및 reask + hooks.py # 이벤트 훅 + exceptions.py # 예외 모델 + processing/ + response.py # 중앙 dispatcher, mode별 request/response 처리 + function_calls.py # OpenAISchema, provider 응답 파싱 + schema.py # OpenAI/Anthropic/Gemini schema 생성 + multimodal.py # image/audio/pdf 메시지 변환 + dsl/ + partial.py # streaming partial object + iterable.py # streaming iterable extraction + maybe.py # 추출 실패 가능성을 모델화 + parallel.py # 병렬 tool call 모델 + citation.py # 원문 근거 quote 검증 mixin + simple_type.py # str/int 등 단순 타입 wrapper + providers/ + openai, anthropic, gemini, genai, bedrock, cohere, ... + batch/ + processor.py, request.py # 배치 요청 생성/처리 + cache/ + __init__.py # AutoCache, DiskCache, cache key + validation/ + llm_validators.py # LLM 기반 field validator + cli/ + batch/files/jobs/usage # CLI 유틸리티 + docs/ # 사용자 문서, 통합 가이드, 튜토리얼 + examples/ # 추출, 지식그래프, SQL, FastAPI, batch 예제 + tests/ # 단위/통합/LLM provider 테스트 +``` + +## 3. 핵심 실행 흐름 + +Instructor의 기본 동작은 다음 순서다. + +1. 사용자가 Pydantic `BaseModel`로 원하는 출력 스키마를 정의한다. +2. `instructor.from_provider("openai/gpt-4o")` 또는 `from_openai()`로 provider client를 래핑한다. +3. `client.chat.completions.create(response_model=MyModel, messages=[...])`를 호출한다. +4. `core.patch.patch()`가 provider의 `create()` 호출을 가로채 `response_model`, `max_retries`, `strict`, `context`, `hooks`, `cache`를 처리한다. +5. `processing.response.handle_response_model()`이 Pydantic 모델을 provider별 tool schema 또는 JSON schema 요청으로 변환한다. +6. `core.retry.retry_sync()` 또는 `retry_async()`가 provider 호출을 실행한다. +7. `processing.response.process_response()`가 raw LLM 응답을 `OpenAISchema.from_response()`로 넘겨 모드별 파서를 선택한다. +8. Pydantic 검증이 성공하면 typed model을 반환하고 `_raw_response`에 원본 provider 응답을 붙인다. +9. JSON 파싱 또는 Pydantic 검증 실패 시 `handle_reask_kwargs()`가 에러 내용을 다음 프롬프트에 반영해 재시도한다. +10. 모든 재시도 실패 시 `InstructorRetryException`에 실패 이력, 마지막 응답, 사용량, 재현 가능한 create kwargs를 담아 예외를 발생시킨다. + +이 흐름은 온톨로지 플랫폼에서 `문서 청크 -> 후보 엔티티/관계 추출 -> 스키마 검증 -> 실패 재시도 -> 근거 포함 결과 저장` 파이프라인으로 바로 매핑된다. + +## 4. 주요 공개 API + +### 4.1 클라이언트 생성 + +- `instructor.from_provider(model: str, async_client=False, cache=None, mode=None, **kwargs)` + - `"provider/model-name"` 형식으로 provider를 자동 선택한다. + - 지원 provider: `openai`, `azure_openai`, `anthropic`, `google`, `vertexai`, `mistral`, `cohere`, `perplexity`, `groq`, `writer`, `bedrock`, `cerebras`, `deepseek`, `fireworks`, `ollama`, `openrouter`, `xai`, `litellm`. + - 기본 모델명을 `Instructor.default_model`에 저장하고 호출 시 `model` 생략을 허용한다. + +- `instructor.from_openai(client, model=None, mode=Mode.TOOLS, **kwargs)` + - 기존 OpenAI 호환 client를 래핑한다. + - OpenAI-compatible endpoint, OpenRouter, Ollama, vLLM류 연동에 유리하다. + +- `instructor.patch(client=..., create=..., mode=...)` + - provider client 또는 독립 create 함수를 직접 patch한다. + - 기존 코드 변경을 최소화하면서 `response_model` 기능을 추가할 수 있다. + +### 4.2 구조화 호출 + +- `client.create(response_model, messages, max_retries=3, strict=True, context=None, hooks=None, **kwargs)` + - 가장 중요한 API다. + - 반환값은 raw JSON이 아니라 Pydantic model instance다. + - `response_model=None`이면 provider raw response를 그대로 반환한다. + +- `client.create_with_completion(...)` + - `(parsed_model, raw_completion)` tuple을 반환한다. + - 디버깅, 감사로그, 추출 근거 저장에 유용하다. + +- `client.create_iterable(response_model, messages, **kwargs)` + - streaming으로 여러 객체를 순차 반환한다. + - 긴 문서에서 엔티티/관계 후보를 점진적으로 받을 때 적합하다. + +- `client.create_partial(response_model, messages, **kwargs)` + - streaming 중 불완전한 partial model을 계속 반환한다. + - UI에서 추출 진행 상태를 보여주거나 긴 ontology 생성 작업을 관찰할 때 유용하다. + +### 4.3 DSL 타입 + +- `Partial[T]` + - 스트리밍 중 채워지는 부분 객체. + - 대형 ontology schema 추출의 진행률 표시와 중간 검증에 적합하다. + +- `IterableModel[T]` + - 하나의 LLM 응답에서 다수의 typed item을 순차 추출한다. + - `EntityCandidate`, `RelationCandidate` 목록 추출에 적합하다. + +- `Maybe(T)` + - `result`, `error`, `message`를 가진 wrapper 모델을 동적으로 만든다. + - “해당 청크에 관계가 없을 수도 있음” 같은 불확실성을 명시적으로 표현한다. + +- `CitationMixin` + - `substring_quotes` 필드를 통해 추출 결과의 원문 근거를 검증한다. + - `context={"context": 원문}`을 전달하면 quote가 실제 원문에 존재하는지 fuzzy matching으로 정리한다. + - 온톨로지 신뢰도, human review, provenance 저장에 중요하다. + +- `ModelAdapter`, simple type adapter + - `str`, `int`, `list[str]` 같은 단순 타입 응답을 Pydantic 검증 경로로 통합한다. + +### 4.4 검증/재시도 + +- `max_retries` + - int 또는 `tenacity.Retrying`/`AsyncRetrying` 객체를 받을 수 있다. + - 검증 실패, JSON 파싱 실패 시 자동 reask를 수행한다. + +- `strict` + - strict JSON/Pydantic validation 여부를 제어한다. + - ontology 저장 전 단계는 `strict=True`를 기본값으로 권장한다. + +- `context` + - Pydantic validator에 전달되는 runtime context다. + - 도메인 ontology, 허용 relation type, source document metadata, language 같은 동적 검증 조건을 전달할 수 있다. + +- `llm_validator(statement, client, allow_override=False, model=..., temperature=0)` + - 특정 필드를 LLM으로 한 번 더 검증한다. + - “관계명은 ontology relation vocabulary에 맞아야 한다” 같은 semantic validation에 사용할 수 있으나 비용과 지연이 있으므로 핵심 필드에 제한하는 것이 좋다. + +### 4.5 캐시 + +- `AutoCache(maxsize=128)` + - thread-safe in-memory LRU cache. + - schema, model, messages, mode를 기반으로 cache key를 만든다. + +- `DiskCache(directory=".instructor_cache")` + - optional `diskcache` 의존성 기반 persistent cache. + +- 캐시 key 구성 요소 + - provider/model + - messages 또는 contents/chat_history + - mode + - response_model JSON schema + +온톨로지 플랫폼에서는 동일 문서 청크와 동일 schema로 재처리할 때 비용 절감을 기대할 수 있다. 단, prompt에 시간/외부 상태가 들어가면 cache 오염을 막기 위해 cache scope를 작업 단위로 제한해야 한다. + +### 4.6 Hooks/Observability + +지원 이벤트: + +- `completion:kwargs`: provider 호출 직전 +- `completion:response`: provider 응답 직후 +- `parse:error`: JSON/Pydantic parsing 실패 +- `completion:last_attempt`: 마지막 시도 직전/시점 +- `completion:error`: provider/network 등 일반 오류 + +온톨로지 플랫폼에서는 이 훅을 사용해 추출 요청 로그, retry 사유, token usage, 실패 샘플, provider별 품질 통계를 저장할 수 있다. + +## 5. Provider/Mode 명세 + +`mode.py`는 provider별 요청 포맷과 응답 파싱 전략을 enum으로 정의한다. + +주요 모드: + +- OpenAI 계열: `TOOLS`, `TOOLS_STRICT`, `JSON`, `MD_JSON`, `JSON_SCHEMA`, `RESPONSES_TOOLS` +- Anthropic: `ANTHROPIC_TOOLS`, `ANTHROPIC_REASONING_TOOLS`, `ANTHROPIC_JSON`, `ANTHROPIC_PARALLEL_TOOLS` +- Google/Gemini: `GEMINI_JSON`, `GEMINI_TOOLS`, `GENAI_TOOLS`, `GENAI_STRUCTURED_OUTPUTS`, `VERTEXAI_TOOLS`, `VERTEXAI_JSON` +- Mistral/Cohere/Cerebras/Fireworks/Writer/Bedrock/XAI 등 provider 전용 모드 +- `PARALLEL_TOOLS`: OpenAI 병렬 tool call +- `OPENROUTER_STRUCTURED_OUTPUTS`: OpenRouter 구조화 출력 + +플랫폼 적용 권장: + +- 기본 OpenAI-compatible provider: `Mode.TOOLS` 또는 provider native structured output +- schema 엄격성이 중요한 ontology extraction: `TOOLS_STRICT` 또는 `JSON_SCHEMA` +- local/open-source 모델: `from_provider("ollama/model")`, OpenAI-compatible base_url 또는 LiteLLM 경유 +- 여러 추출 타입을 한 번에 받을 경우: `PARALLEL_TOOLS`는 유용하나 streaming 미지원이므로 대량 처리에는 분리 호출도 고려 + +## 6. 온톨로지 플랫폼에 필요한 기능 매핑 + +### 6.1 엔티티 추출 + +원본 Instructor 기능: + +- Pydantic `EntityCandidate` 모델 정의 +- `IterableModel[EntityCandidate]` 또는 `list[EntityCandidate]` 추출 +- field validator로 label normalization, type validation +- retry/reask로 누락/타입 오류 자동 수정 + +플랫폼 기능: + +- 문서 청크에서 개체명, 표준명, 별칭, 타입, 설명, 근거 quote, confidence 추출 +- 기존 ontology vocabulary와 비교해 허용 타입만 통과 +- 중복 후보 병합 전 structured candidate pool 생성 + +권장 모델 예시: + +```python +class EntityCandidate(CitationMixin): + name: str + canonical_name: str + entity_type: str + aliases: list[str] = [] + description: str | None = None + confidence: float +``` + +### 6.2 관계 추출 + +원본 Instructor 기능: + +- nested model, enum/literal validation +- `Maybe(RelationCandidate)`로 관계 부재 표현 +- `context` 기반 validator에서 허용 relation vocabulary 검사 + +플랫폼 기능: + +- source entity, target entity, predicate, direction, evidence, confidence 추출 +- entity 후보와 relation 후보를 분리 추출 후 graph builder에서 연결 +- 관계 근거가 없는 경우 저장하지 않고 review queue로 이동 + +권장 모델 예시: + +```python +class RelationCandidate(CitationMixin): + source_name: str + target_name: str + relation_type: str + relation_label: str + confidence: float +``` + +### 6.3 속성/스키마 추출 + +원본 Instructor 기능: + +- nested Pydantic models +- JSON schema 기반 출력 강제 +- `strict=True` validation + +플랫폼 기능: + +- 엔티티별 속성명, 값, 단위, 데이터 타입, source span 추출 +- domain schema 후보 생성 +- ontology class/property 자동 제안 + +### 6.4 근거와 provenance + +원본 Instructor 기능: + +- `CitationMixin` +- `_raw_response` 보존 +- `create_with_completion()` + +플랫폼 기능: + +- 각 triple 또는 property assertion에 source document id, chunk id, quote, model, prompt hash, raw completion id 저장 +- 신뢰도 낮은 결과를 human review로 라우팅 + +### 6.5 대량 처리 + +원본 Instructor 기능: + +- `batch/` 모듈 +- provider별 batch request 생성 +- `create_iterable()` streaming +- cache + +플랫폼 기능: + +- 크롤링된 문서 청크를 batch job으로 변환 +- provider batch API 또는 내부 queue worker에서 처리 +- 실패한 청크만 재시도 +- 같은 schema/prompt 조합 재처리 시 cache 사용 + +### 6.6 멀티모달 추출 + +원본 Instructor 기능: + +- `processing.multimodal.Image`, `Audio` +- provider별 message conversion +- PDF/image/audio 예제 포함 + +플랫폼 기능: + +- 문서 이미지, 표, 영수증, PDF에서 구조화 정보 추출 +- 온톨로지 구축 대상이 제품/인물/기관/문헌 등일 때 이미지 기반 보조 evidence 확보 + +## 7. 상세 기능명세 + +### F-INST-001 Provider Client Wrapping + +- 목적: 다양한 LLM provider를 동일한 structured output API로 호출한다. +- 입력: provider/model 문자열, API key, base_url, async 여부, mode +- 출력: `Instructor` 또는 `AsyncInstructor` +- 성공 조건: `client.chat.completions.create(response_model=...)` 호출 가능 +- 적용 우선도: 필수 +- 원본 사용 가능성: 거의 변형 없이 사용 + +### F-INST-002 Pydantic Response Model Extraction + +- 목적: 비정형 LLM 응답을 Pydantic 모델로 검증된 객체로 반환한다. +- 입력: `response_model`, `messages`, provider kwargs +- 출력: Pydantic model instance +- 오류: `ValidationError`, `JSONDecodeError`, `InstructorRetryException` +- 적용 우선도: 필수 +- 원본 사용 가능성: 그대로 사용 + +### F-INST-003 Automatic Reask/Retry + +- 목적: 스키마 검증 실패 시 오류 내용을 LLM에 전달해 자동 수정한다. +- 입력: 실패 응답, exception, failed_attempts, mode +- 출력: 수정된 kwargs/messages로 재시도 +- 설정: `max_retries`, `timeout`, tenacity policy +- 적용 우선도: 필수 +- 원본 사용 가능성: 그대로 사용하되 retry 횟수/timeout 정책은 플랫폼 설정화 필요 + +### F-INST-004 Strict Schema Validation + +- 목적: ontology 저장소에 잘못된 shape의 데이터를 넣지 않는다. +- 입력: Pydantic schema, JSON response, strict flag +- 출력: valid model 또는 validation error +- 적용 우선도: 필수 +- 원본 사용 가능성: 그대로 사용 + +### F-INST-005 Citation/Evidence Validation + +- 목적: 추출 결과가 원문에 기반하는지 확인한다. +- 입력: `CitationMixin` 모델, `context={"context": source_text}` +- 출력: 원문에 존재하는 quote로 정리된 `substring_quotes` +- 적용 우선도: 필수 +- 원본 사용 가능성: 대부분 사용 가능. 한국어/긴 문서 fuzzy match 성능은 추가 검증 필요 + +### F-INST-006 Maybe Wrapper + +- 목적: 추출 대상이 없을 수 있는 상황을 예외가 아니라 정상 결과로 표현한다. +- 입력: `Maybe(EntityCandidate)` 또는 `Maybe(RelationCandidate)` +- 출력: `{result, error, message}` +- 적용 우선도: 높음 +- 원본 사용 가능성: 그대로 사용 + +### F-INST-007 Iterable Extraction + +- 목적: 긴 응답에서 여러 후보 객체를 안정적으로 추출한다. +- 입력: item model, stream response +- 출력: item generator 또는 `ListResponse` +- 적용 우선도: 높음 +- 원본 사용 가능성: 그대로 사용 + +### F-INST-008 Partial Streaming + +- 목적: 추출 중간 결과를 UI/로그/작업 상태에 반영한다. +- 입력: `Partial[Model]`, `stream=True` +- 출력: partial model stream +- 적용 우선도: 중간 +- 원본 사용 가능성: 그대로 사용 + +### F-INST-009 Parallel Tool Extraction + +- 목적: 한 요청에서 여러 구조화 모델을 동시에 추출한다. +- 입력: model list 또는 parallel wrapper, `Mode.PARALLEL_TOOLS` +- 출력: 여러 typed model +- 제약: streaming 미지원 +- 적용 우선도: 중간 +- 원본 사용 가능성: 그대로 사용하되 대량 처리에서는 비용/재시도 단위를 고려 + +### F-INST-010 Cache + +- 목적: 동일 청크/동일 schema 추출 요청의 비용을 줄인다. +- 입력: cache backend, messages, model, mode, response_model schema +- 출력: cached model 또는 miss 후 저장 +- 적용 우선도: 높음 +- 원본 사용 가능성: `AutoCache`는 개발/단일 프로세스용으로 그대로 사용, 운영은 Redis/DB backend 구현 권장 + +### F-INST-011 Hooks and Audit Logging + +- 목적: 추출 호출, 응답, 실패, 재시도, 마지막 실패를 관측한다. +- 입력: hook handler +- 출력: 내부 이벤트 +- 적용 우선도: 필수 +- 원본 사용 가능성: 그대로 사용하되 플랫폼 audit logger와 연결 필요 + +### F-INST-012 LLM Semantic Validator + +- 목적: Pydantic으로 표현하기 어려운 의미 검증을 LLM에 위임한다. +- 입력: validation statement, value, validator model +- 출력: valid/fixed value 또는 validation error +- 적용 우선도: 선택 +- 원본 사용 가능성: 제한적으로 사용. 비용/재현성/지연시간 때문에 핵심 relation 검증에만 권장 + +### F-INST-013 Batch Processing + +- 목적: 많은 문서 청크를 provider batch API 또는 내부 batch 구조로 처리한다. +- 입력: batch requests, provider config +- 출력: batch job, results +- 적용 우선도: 높음 +- 원본 사용 가능성: OpenAI/Anthropic 중심으로 재사용 가능. 플랫폼 job queue와 통합 필요 + +### F-INST-014 Multimodal Input Conversion + +- 목적: 이미지, 오디오, PDF 등 비텍스트 입력을 provider 메시지 형식으로 변환한다. +- 입력: path/url/base64/data URI +- 출력: provider-ready content block +- 적용 우선도: 중간 +- 원본 사용 가능성: 그대로 사용 가능하나 provider별 비용/지원 범위 검증 필요 + +### F-INST-015 Raw Response Preservation + +- 목적: 추출 결과의 감사 가능성과 재현성을 확보한다. +- 입력: provider raw response +- 출력: parsed model의 `_raw_response` +- 적용 우선도: 필수 +- 원본 사용 가능성: 그대로 사용. 저장소에는 필요한 metadata만 선별 저장 권장 + +## 8. 플랫폼 아키텍처 적용안 + +권장 구성: + +```text +Crawler / Document Loader + -> Chunker + -> InstructorExtractionService + - provider client registry + - response model registry + - prompt template registry + - retry/cache/hooks policy + -> Candidate Normalizer + -> Entity Resolution + -> Ontology Graph Builder + -> Human Review Queue + -> Graph DB / Relational Store +``` + +`InstructorExtractionService`는 Instructor를 직접 노출하지 말고 플랫폼 내부 adapter로 감싼다. + +필수 adapter 책임: + +- provider/model 설정 로딩 +- domain별 response_model 선택 +- prompt/context 생성 +- `context`에 ontology vocabulary와 source metadata 주입 +- hooks로 audit log 저장 +- cache scope 결정 +- `InstructorRetryException`을 플랫폼 표준 에러로 변환 +- raw response/token usage/provenance 저장 + +## 9. 기본 소스로 가져올 때의 권장 범위 + +거의 변형 없이 사용: + +- `instructor.core.patch` +- `instructor.core.client` +- `instructor.core.retry` +- `instructor.processing.response` +- `instructor.processing.function_calls` +- `instructor.processing.schema` +- `instructor.mode` +- `instructor.dsl.*` +- `instructor.cache` +- `instructor.validation.llm_validators` + +플랫폼에 맞게 감쌀 부분: + +- `auto_client.from_provider`: 플랫폼 provider registry와 secret manager에 맞게 thin wrapper 작성 +- `hooks`: audit/event bus에 연결 +- `cache`: 운영용 Redis/DB cache backend 추가 +- `batch`: 플랫폼 job queue, chunk id, dataset id와 매핑 +- `CitationMixin`: 한국어/긴 문서/정규화 quote에 대한 보강 validator 추가 가능 + +굳이 가져오지 않아도 되는 부분: + +- `docs/`, `examples/`, `scripts/` 전체 +- `cli/`는 운영 필요성이 생기기 전에는 제외 가능 +- provider 중 사용하지 않는 optional provider dependency + +## 10. 온톨로지 추출용 최소 구현 예시 + +```python +from pydantic import BaseModel, Field, field_validator +from instructor import CitationMixin, Maybe + + +class EntityCandidate(CitationMixin): + name: str = Field(description="Surface form found in the source text") + canonical_name: str = Field(description="Normalized canonical entity name") + entity_type: str = Field(description="Ontology class/type") + aliases: list[str] = Field(default_factory=list) + confidence: float = Field(ge=0, le=1) + + +class RelationCandidate(CitationMixin): + source_name: str + target_name: str + relation_type: str + confidence: float = Field(ge=0, le=1) + + @field_validator("relation_type") + @classmethod + def relation_must_be_allowed(cls, value: str, info): + allowed = (info.context or {}).get("allowed_relations", set()) + if allowed and value not in allowed: + raise ValueError(f"relation_type must be one of {sorted(allowed)}") + return value + + +MaybeRelation = Maybe(RelationCandidate) +``` + +호출 패턴: + +```python +client = instructor.from_provider("openai/gpt-4o-mini") + +result = client.chat.completions.create( + response_model=MaybeRelation, + messages=[ + {"role": "system", "content": "Extract ontology relation candidates only from the provided source."}, + {"role": "user", "content": source_chunk}, + ], + context={ + "context": source_chunk, + "allowed_relations": {"is_a", "part_of", "used_for", "located_in"}, + }, + max_retries=3, + strict=True, +) +``` + +## 11. 리스크와 보완 필요점 + +- Instructor는 ontology reasoner가 아니다. OWL/RDF reasoning, graph merge, entity resolution은 별도 모듈이 필요하다. +- Pydantic schema가 너무 크면 LLM 출력 품질이 떨어진다. 추출 단계를 엔티티, 관계, 속성, 검증으로 분리하는 것이 좋다. +- `CitationMixin`은 quote 존재성 검증에 가깝고 “의미적으로 올바른 근거”를 보장하지 않는다. confidence와 human review가 필요하다. +- LLM validator는 강력하지만 비용이 크다. 전체 필드가 아니라 고위험 필드에만 적용해야 한다. +- provider별 mode 동작 차이가 있다. 운영 전 provider별 golden test set이 필요하다. +- local model/Ollama/OpenRouter 사용 시 tool calling 품질이 모델마다 크게 다르다. JSON mode fallback을 준비해야 한다. +- cache는 prompt/schema/source version을 엄격히 key에 포함해야 한다. 원본 cache key는 schema와 messages를 포함하므로 안전한 편이지만, 플랫폼 metadata까지 포함하려면 wrapper 수준에서 messages/context에 명확히 반영해야 한다. + +## 12. 테스트 전략 + +원본 테스트에서 참고할 영역: + +- `tests/test_patch.py`: patch 동작 +- `tests/test_retry_json_mode.py`: JSON mode retry +- `tests/test_json_extraction.py`: JSON 추출 +- `tests/test_process_response.py`: 응답 dispatcher +- `tests/test_schema.py`, `tests/test_schema_utils.py`: schema 생성 +- `tests/dsl/test_partial.py`: partial streaming +- `tests/test_list_response.py`: list response +- `tests/test_cache_integration.py`: cache +- `tests/llm/test_core_providers/*`: provider capability 공통 테스트 + +플랫폼 추가 테스트: + +- ontology entity schema validation unit test +- relation vocabulary validator test +- source quote/provenance validation test +- retry 후 수정 성공 golden test +- invalid extraction이 graph DB에 저장되지 않는 integration test +- provider별 동일 chunk extraction 품질 비교 test +- cache hit/miss와 schema 변경 cache busting test + +## 13. 결론 + +Instructor는 범용 온톨로지 구축 플랫폼의 “LLM 구조화 추출 엔진”으로 매우 적합하다. 특히 Pydantic 중심 스키마, provider 추상화, 자동 재시도, streaming DSL, 근거 quote mixin, hooks, cache가 플랫폼 핵심 요구와 잘 맞는다. + +기본 소스로 사용할 때는 Instructor 자체를 크게 변형하기보다, 플랫폼 내부에 `InstructorExtractionService` adapter를 두고 provider 설정, prompt, ontology vocabulary, audit log, cache, job queue를 연결하는 방식이 가장 안전하다. 이렇게 하면 원본 업데이트를 따라가기 쉽고, 온톨로지 플랫폼 고유 로직은 adapter와 domain schema 레이어에 깔끔하게 남길 수 있다. diff --git a/ONTOLOGY_PLATFORM_MASTER_DESIGN.md b/ONTOLOGY_PLATFORM_MASTER_DESIGN.md new file mode 100644 index 0000000..80a6022 --- /dev/null +++ b/ONTOLOGY_PLATFORM_MASTER_DESIGN.md @@ -0,0 +1,649 @@ +# 범용 온톨로지 구축 플랫폼 통합 설계안 + +작성일: 2026-05-13 +대상 자료: `오픈소스분석자료` 폴더의 8개 분석 문서 + +## 0. 최종 결론 + +시작 프로젝트는 현재 저장소의 `crawler_platform`을 유지한다. 이미 FastAPI, SQLAlchemy, 프로젝트/소스/Page/Entity/Claim/Evidence 모델, 크롤링 파이프라인, 웹 UI, 연구 루프 일부가 존재하므로 이것을 버리고 외부 프로젝트 하나로 갈아타는 것은 손실이 크다. + +다만 8개 오픈소스 중 “기본 엔진” 역할은 `OntoCast`가 가장 적합하다. OntoCast는 문서 입력에서 RDF 온톨로지와 Facts를 만들고, GraphUpdate/SPARQL 증분 갱신, renderer/critic retry loop, entity aggregation, triple store abstraction을 갖고 있어 온톨로지 구축 코어에 가장 직접적이다. + +권장 구조는 다음과 같다. + +```text +crawler_platform # 제품/플랫폼 껍데기. 계속 유지 + Platform API / UI / DB / Jobs + Source & Dataset Management + Review / Approval / Versioning + Adapters + Trafilatura # 웹 본문/메타/구조 추출 + Crawl4AI # JS/동적/딥 크롤링, 필요 시 추가 + OntoCast Core # RDF ontology/facts 생성 엔진 + Guardrails # LLM 구조화 출력 검증 게이트 + Neo4j GraphRAG # KG projection, GraphRAG, Text2Cypher + Optional / reference only + Firecrawl # API/옵션 설계 참고, 초기 직접 통합 제외 + Knowledge Agent # gap/audit workflow 패턴 참고 + OpenDeepResearcher # 외부 검색 루프 패턴 참고 +``` + +초기 실제 통합 수는 최대한 줄인다. + +1. 1차 실제 통합: `Trafilatura`, `OntoCast`, `Guardrails` +2. 2차 실제 통합: `Neo4j GraphRAG` +3. 3차 실제 통합: `Crawl4AI` +4. 코드 통합 보류: `Firecrawl` +5. 패턴/프롬프트만 차용: `Knowledge Agent`, `OpenDeepResearcher` + +이렇게 하면 핵심 기능은 상용제품 수준으로 설계하면서도, 한 번에 여러 거대 프로젝트를 섞어서 생기는 버그를 피할 수 있다. + +## 1. 8개 오픈소스별 채택 판단 + +| 오픈소스 | 가장 큰 장점 | 채택 방식 | 초기 통합 여부 | +|---|---|---|---| +| OntoCast | 문서 기반 RDF ontology/facts 생성, GraphUpdate/SPARQL 증분 갱신, renderer/critic retry loop, entity aggregation | 코어 엔진으로 거의 원형 유지. API/UI는 현재 플랫폼에서 새로 감싼다 | 필수 P0 | +| Trafilatura | HTML 본문/메타데이터/링크/표/중복 fingerprint 추출이 안정적이고 Apache-2.0 | Python API를 adapter로 사용. `bare_extraction(output_format="python")` 중심 | 필수 P0 | +| Guardrails | Pydantic/JSON Schema 기반 LLM 출력 검증, validator, reask/fix/filter 정책 | `OntologyGuard` facade로 감싸고 온톨로지 전용 validator 추가 | 필수 P0 | +| Neo4j GraphRAG | GraphSchema, SimpleKGPipeline, Neo4jWriter, Vector/Hybrid/Text2Cypher Retriever | 고정 버전 dependency + wrapper. Neo4j는 canonical store가 아니라 projection/search 계층 | 필수 P1 | +| Crawl4AI | Python 기반 비동기 크롤링, Playwright, Markdown, deep crawl, dispatcher/cache | JS-heavy/dynamic source 전용 adapter. Trafilatura 실패 시 fallback | 필수 P2 | +| Firecrawl | 상용급 scrape/map/crawl/batch/search API 표면과 job 운영 모델 | API/옵션/상태 모델 참고. AGPL/TypeScript/운영 복잡도 때문에 초기 직접 병합 제외 | 보류 | +| Knowledge Agent | 지식 공백 탐지, 연구-큐레이션-감사-수정-개선 루프 | LangGraph workflow와 LightRAG 프롬프트 패턴만 차용. 코드 안정화 후 일부 도입 | 패턴 P2 | +| OpenDeepResearcher | 검색어 생성, 검색, 페이지 유용성 평가, 추가 검색 판단 반복 루프 | 작은 모듈로 재작성. 원본 notebook 코드는 그대로 제품 코드에 넣지 않음 | 패턴 P2 | + +## 2. 중복 기능 제거 원칙 + +중복되는 프로젝트를 동시에 같은 책임으로 쓰지 않는다. + +| 책임 | 최종 선택 | 제외/보류 | +|---|---|---| +| 정적 웹 본문 추출 | Trafilatura | Firecrawl scrape를 기본으로 쓰지 않음 | +| 동적 페이지/딥 크롤 | Crawl4AI | Firecrawl과 Crawl4AI 동시 기본 사용 금지 | +| 문서 기반 RDF 온톨로지 생성 | OntoCast | Neo4j GraphRAG의 자유 KG 추출을 canonical ontology로 직접 확정하지 않음 | +| LLM 출력 검증 | Guardrails | 자체 ad-hoc JSON validation만으로 끝내지 않음 | +| GraphRAG/질의응답 | Neo4j GraphRAG | OntoCast triple store에 질의응답 기능을 억지로 모두 구현하지 않음 | +| 외부 검색 연구 | Platform ResearchLoop | Knowledge Agent와 OpenDeepResearcher를 각각 독립 실행하지 않음 | +| Canonical 저장소 | RDF/Fuseki + relational metadata | Neo4j를 원본 truth store로 삼지 않음 | + +핵심 규칙: + +1. 모든 수집 결과는 먼저 `Document`와 `ContentUnit`으로 정규화한다. +2. 모든 LLM 산출물은 `Candidate` 상태로 저장하고 바로 published graph에 넣지 않는다. +3. 모든 엔티티/관계/트리플은 evidence와 provenance 없이는 승인할 수 없다. +4. RDF/Fuseki를 canonical semantic store로 둔다. +5. Neo4j는 projection, graph search, GraphRAG, Text2Cypher용으로 둔다. +6. Firecrawl은 초기에는 직접 통합하지 않고, API 설계와 운영 상태 모델만 참고한다. + +## 3. 상용제품 기준 전체 기능 설계 + +### 3.1 제품 모듈 + +```text +Ontology Studio Platform + Project & Tenant + - 프로젝트 생성/설정/권한 + - 도메인 정책, 언어 정책, LLM/embedding profile + - source trust policy, robots/license/privacy policy + + Dataset & Source + - 파일 업로드, URL seed, sitemap/feed discovery + - source catalog, update schedule + - source reliability score, blocklist, allowlist + + Ingestion + - Trafilatura static extraction + - Crawl4AI dynamic/deep crawling + - document parse: PDF/DOCX/HTML/JSON/CSV + - dedup, content hash, fingerprint + - provenance, raw artifact storage + + Ontology Build Engine + - ContentUnit chunking + - ontology selection or fresh ontology creation + - GraphUpdate/SPARQL delta generation + - facts extraction + - renderer/critic/retry loop + - entity aggregation and URI normalization + + Validation Gate + - Pydantic schema validation + - JSON repair/type normalization + - ontology-specific validator + - SHACL/OWL validation + - evidence alignment validation + - reask/fix/filter/refrain policy + + Review & Governance + - candidate entity/relation/triple review + - source evidence highlight + - GraphUpdate diff viewer + - approve/reject/merge/split/edit + - reviewer audit log + - schema draft -> published workflow + - rollback/release/version tagging + + Storage + - relational DB: project/job/source/page/review/audit metadata + - artifact store: raw HTML, markdown, extracted JSON, TTL, screenshots + - Fuseki/RDF store: canonical ontology/facts + - Neo4j: graph projection, vector/fulltext index, GraphRAG + + Search & Use + - SPARQL query + - graph neighborhood search + - vector/hybrid search + - GraphRAG answer with provenance + - read-only Text2Cypher + - export: TTL, RDF/XML, JSON-LD, CSV, Parquet + + Research & Improvement + - knowledge gap detection + - external search planning + - usefulness scoring + - context/evidence extraction + - repeated failure analysis + - schema/prompt/source policy improvement suggestions + + Operations + - async job queue + - progress/cancel/retry + - cost/budget tracking + - LLM cache + - metrics/logs/traces + - backup/restore + - admin safety controls +``` + +### 3.2 기준 데이터 모델 + +현재 `crawler_platform` 모델을 확장한다. + +필수 추가/정리 모델: + +| 모델 | 목적 | +|---|---| +| `Dataset` | 프로젝트 내 문서 묶음, import batch 단위 | +| `Document` | URL/파일/API 응답의 정규화 원문 | +| `ContentUnit` | chunk, source offsets, section/table/list 정보 | +| `Artifact` | raw html, markdown, body xml, TTL, JSON, screenshot 저장 위치 | +| `OntologySchemaVersion` | draft/published/archived schema, version, hash | +| `GraphDelta` | OntoCast GraphUpdate/SPARQL delta와 적용 상태 | +| `CandidateEntity` | 검수 전 엔티티 후보 | +| `CandidateRelation` | 검수 전 관계 후보 | +| `CandidateTriple` | 검수 전 RDF/property graph 후보 | +| `ValidationRun` | Guardrails/SHACL/OWL 검증 결과 | +| `ReviewDecision` | 승인/반려/수정/병합 이력 | +| `EntityMergeCandidate` | exact/fuzzy/embedding merge 후보 | +| `ResearchSession` | 외부 검색/공백 보완 세션 | +| `JobRun` | 수집/추출/검증/저장 작업 상태 | + +기존 `Page`, `Entity`, `Claim`, `Evidence`, `OntologyTriple`, `KnowledgeGap`은 유지하되 아래 필드를 보강한다. + +- `Page`: `markdown`, `body_xml_ref`, `fingerprint`, `language`, `change_status`, `last_checked_at` +- `Claim`: `candidate_status`, `validation_status`, `review_status`, `ontology_version` +- `Evidence`: `content_unit_id`, `char_start`, `char_end`, `selector`, `quote_hash` +- `OntologyTriple`: `graph_uri`, `ontology_version_id`, `rdf_subject`, `rdf_predicate`, `rdf_object`, `provenance_graph_uri` + +## 4. 최종 아키텍처 + +```mermaid +flowchart TB + UI["Ontology Studio UI"] --> API["FastAPI Platform API"] + API --> JOB["Job Queue / Worker"] + API --> DB["Relational Metadata DB"] + + JOB --> ING["Ingestion Pipeline"] + ING --> TRA["Trafilatura Adapter"] + ING --> C4A["Crawl4AI Adapter"] + ING --> DOC["Document / ContentUnit Store"] + + DOC --> ONTO["OntoCast Core Engine"] + ONTO --> GUARD["Guardrails Validation Gate"] + GUARD --> REVIEW["Human Review Queue"] + REVIEW --> RDF["Canonical RDF Store / Fuseki"] + + RDF --> NEO["Neo4j Projection"] + NEO --> RAG["GraphRAG / Text2Cypher / Hybrid Search"] + RDF --> EXPORT["TTL / JSON-LD / RDF Export"] + + DB --> OBS["Audit / Metrics / Cost Dashboard"] + JOB --> OBS + + RESEARCH["Research Loop"] --> ING + RESEARCH --> DOC + RESEARCH --> REVIEW +``` + +저장소 원칙: + +1. `Relational DB`: 제품 상태, 작업 상태, 검수/승인/감사 이력. +2. `Artifact Store`: 원문과 중간 산출물. +3. `RDF Store`: 승인된 canonical ontology/facts. +4. `Neo4j`: 검색/탐색/GraphRAG projection. + +## 5. 기능별 상세 설계 + +### 5.1 Project & Tenant + +상용제품 수준 필수 기능: + +- 프로젝트 생성/복제/보관 +- 프로젝트별 namespace/base IRI +- 프로젝트별 언어, ontology naming policy +- LLM profile, embedding profile +- source trust policy +- 승인 정책: 자동 승인 금지, 저위험 자동 승인, 고위험 수동 승인 +- 사용자/역할: admin, ontologist, reviewer, operator, viewer + +### 5.2 Source & Dataset + +기능: + +- URL seed 등록 +- sitemap/feed discovery +- 파일 업로드 +- API/DB source 등록 +- source trust score +- robots/license/privacy policy +- update schedule +- change detection +- 실패 URL과 denial reason 저장 + +채택 소스: + +- Trafilatura: feed/sitemap discovery, metadata extraction +- Crawl4AI: JS-heavy/dynamic page, deep crawl +- Firecrawl: map/crawl/search 옵션 설계 참고 + +### 5.3 Ingestion + +표준 파이프라인: + +```text +Source + -> URL/File discovery + -> fetch/render + -> raw artifact save + -> Trafilatura bare_extraction + -> metadata normalize + -> content hash/fingerprint + -> ContentUnit chunking + -> quality score + -> Document ready +``` + +수용 기준: + +- HTML 없이 텍스트만 있는 문서도 처리 +- JS 렌더링 필요 시 Crawl4AI fallback +- 동일 URL/동일 본문/near duplicate 구분 +- 제목/날짜/저자/canonical URL/source URL 보존 +- table/list/heading 구조를 잃지 않음 +- evidence offset 또는 selector를 가능한 한 보존 + +### 5.4 Ontology Build + +OntoCast를 중심에 둔다. + +기능: + +- ontology 선택 또는 신규 생성 +- RDFGraph/ Ontology/ContentUnit 모델 사용 +- GraphUpdate 기반 증분 갱신 +- facts renderer/critic loop +- ontology renderer/critic loop +- unit별 병렬 처리 +- entity aggregation +- URI 정규화 +- owl:sameAs 보존 +- budget/caching + +플랫폼에서 추가할 기능: + +- project/dataset/job 식별자 +- 다중 문서 corpus 처리 +- 비동기 job progress +- output artifact 저장 +- Korean/domain prompt profile +- versioning/diff/rollback +- human review 연결 + +### 5.5 Validation Gate + +Guardrails를 `OntologyGuard`로 감싼다. + +초기 필수 validator: + +| Validator | 기능 | 실패 정책 | +|---|---|---| +| `EntityIdFormatValidator` | ID/URI 형식 검증 | fix/reask | +| `UniqueEntityValidator` | 중복 엔티티 후보 검증 | reask/filter | +| `RelationEndpointExistsValidator` | 관계 양끝 엔티티 존재 확인 | reask | +| `PredicateVocabularyValidator` | 허용 predicate/ontology schema 매핑 | custom/reask | +| `EvidenceExistsValidator` | evidence가 원문 ContentUnit에 존재하는지 확인 | filter/reask | +| `NoHallucinatedClassValidator` | 근거 없는 class/property 생성 차단 | reask | +| `ConfidenceRangeValidator` | 0~1 confidence 보정 | fix | +| `SHACLShapeValidator` | SHACL/OWL 제약 검증 | exception/reask | +| `NoUnsafeCypherValidator` | Text2Cypher write/delete 차단 | exception | + +정책: + +- parsing/schema 오류는 reask 1회 +- 의미가 바뀔 수 있는 자동 fix는 금지 +- evidence 없는 triple은 저장 금지 +- 최종 실패는 review queue로 이동 + +### 5.6 Review & Versioning + +상용제품 차별화의 핵심이다. + +필수 화면/API: + +- candidate entity/relation/triple 목록 +- 원문 evidence highlight +- GraphUpdate diff +- accepted/rejected/edited 상태 +- schema draft/published 전환 +- version diff +- rollback +- merge/split editor +- reviewer comment +- audit log + +승인 상태: + +```text +generated + -> validated + -> pending_review + -> approved + -> published + -> superseded / rejected / archived +``` + +### 5.7 Storage & Projection + +Canonical: + +- Fuseki/RDF store에 승인된 ontology/facts 저장 +- provenance는 named graph 또는 side graph로 분리 + +Projection: + +- Neo4j에 RDF/property graph projection +- Document/Chunk/Entity/Relation/Evidence 연결 유지 +- vector/fulltext index 생성 +- GraphRAG와 Text2Cypher는 read-only API로 제공 + +보안: + +- Text2Cypher는 read-only 검사 +- result limit, timeout, 금지 키워드, 권한 필터 +- schema allowlist 적용 + +### 5.8 Research Loop + +Knowledge Agent와 OpenDeepResearcher는 독립 프로젝트로 붙이지 않는다. 현재 `crawler_platform.app.core.research` 아래에 하나의 `ResearchLoop`로 재구성한다. + +기능: + +- knowledge gap detection +- 검색어 생성 +- 검색 provider 인터페이스 +- URL 중복 제거 +- page usefulness scoring +- context/evidence extraction +- gap coverage 계산 +- 추가 검색 판단 +- source curation +- review queue로 후보 전달 + +차용: + +- Knowledge Agent: Analyst/Researcher/Curator/Auditor/Fixer/Advisor 역할 분리, LightRAG JSON 추출 프롬프트 +- OpenDeepResearcher: 반복형 검색 루프, page usefulness, 추가 검색 판단, 비동기 병렬 처리 + +필수 수정: + +- notebook/CLI/input 구조 제거 +- `eval` 금지 +- JSON schema + Guardrails 검증 +- API key/config 하드코딩 제거 +- session history DB 저장 +- cancellation/retry/progress 추가 + +## 6. 단계별 통합 계획 + +각 단계는 독립적으로 완료/검증한 뒤 다음 단계로 간다. 한 단계에서 실패하면 다음 오픈소스를 붙이지 않는다. + +### Phase 0. 기준선 고정 + +목표: + +- 현재 `crawler_platform` 기능과 테스트 기준선을 고정한다. + +작업: + +1. 현재 테스트 전체 실행 +2. DB schema snapshot 작성 +3. 주요 API smoke test 작성 +4. sample HTML 수집 -> entity/claim 저장 흐름 고정 + +완료 조건: + +- 기존 기능 regression 없음 +- 현재 DB 모델과 API 목록 문서화 + +### Phase 1. 공통 데이터 계약 먼저 구축 + +목표: + +- 외부 소스 통합 전 내부 표준 모델을 확정한다. + +작업: + +1. `Document`, `ContentUnit`, `Artifact`, `CandidateTriple`, `ValidationRun`, `ReviewDecision`, `GraphDelta` 모델 추가 +2. 기존 `Page/Claim/Evidence/OntologyTriple`과 호환 mapping 작성 +3. 모든 ingestion 결과가 `Document -> ContentUnit`으로 들어오게 한다. + +완료 조건: + +- 기존 crawl 결과가 새 Document/ContentUnit으로 저장됨 +- 기존 Claim/Evidence 저장이 깨지지 않음 + +### Phase 2. Trafilatura 통합 + +목표: + +- 정적 웹 문서를 ontology-ready document로 안정 정제한다. + +작업: + +1. `TrafilaturaAdapter` 추가 +2. `bare_extraction(output_format="python", with_metadata=True)` 사용 +3. metadata, body XML, fingerprint, links/tables를 Artifact/ContentUnit에 저장 +4. 기존 parser/fetcher와 교체하지 않고 source profile로 선택 가능하게 한다. + +완료 조건: + +- 한국어/영어 sample HTML 10개 이상에서 본문/메타/링크 추출 +- table/list/heading chunk 보존 +- 동일 본문 fingerprint 중복 감지 + +### Phase 3. OntoCast Core 통합 + +목표: + +- 문서에서 ontology/facts RDF 후보를 생성한다. + +작업: + +1. OntoCast의 `onto/`, `agent/`, `stategraph/`, `tool/` 핵심만 별도 package로 가져온다. +2. Robyn API/CLI는 가져오지 않는다. +3. `OntologyBuildService.process_document()` facade 작성 +4. ContentUnit -> OntoCast ContentUnit adapter 작성 +5. output TTL/GraphUpdate/Facts를 Artifact와 GraphDelta로 저장 + +완료 조건: + +- 단일 문서로 ontology TTL과 facts TTL 생성 +- GraphUpdate delta 저장 +- critic retry 결과와 budget 기록 +- 아직 published store에는 자동 반영하지 않음 + +### Phase 4. Guardrails 검증 게이트 통합 + +목표: + +- LLM 산출물의 schema 오류와 근거 없는 triple을 저장 전에 차단한다. + +작업: + +1. `OntologyGuard` facade 작성 +2. Pydantic output schema 정의 +3. 최소 validator 5개 구현 +4. OntoCast output과 기존 extractor output을 같은 검증 게이트에 통과 +5. ValidationRun 저장 + +완료 조건: + +- 잘못된 JSON/schema는 저장 차단 +- evidence 없는 triple은 후보에서 제거 또는 reask +- validation log가 UI/API에서 조회 가능 + +### Phase 5. Review/Versioning 최소 구현 + +목표: + +- 검증된 후보를 사람이 승인해야 canonical graph에 반영한다. + +작업: + +1. candidate list API +2. evidence 조회 API +3. approve/reject/edit API +4. GraphDelta publish API +5. ontology schema draft/published version 모델 + +완료 조건: + +- candidate -> approved -> published 상태 전이 +- rollback 가능한 version 기록 +- 누가 무엇을 승인했는지 audit log 저장 + +### Phase 6. Neo4j GraphRAG 통합 + +목표: + +- 승인된 RDF/claim을 Neo4j projection으로 만들고 검색/RAG를 제공한다. + +작업: + +1. `neo4j-graphrag` 고정 버전 의존성 추가 +2. RDF/Claim/ContentUnit -> Neo4j graph projection 작성 +3. Vector/Hybrid/VectorCypher retriever wrapper 작성 +4. GraphRAG API 작성 +5. Text2Cypher read-only sandbox 작성 + +완료 조건: + +- approved graph가 Neo4j에 projection됨 +- chunk -> entity -> evidence 검색 가능 +- Text2Cypher가 쓰기/삭제 명령을 차단 + +### Phase 7. Crawl4AI 통합 + +목표: + +- JS-heavy/dynamic site와 deep crawl을 안정 처리한다. + +작업: + +1. `Crawl4AIAdapter` 추가 +2. source profile: `static`, `dynamic_page`, `deep_discovery`, `structured_extract` +3. rendered HTML/markdown/screenshot 선택 저장 +4. rendered HTML을 Trafilatura에 다시 넣는 hybrid path 구성 + +완료 조건: + +- JS page에서 rendered HTML 추출 +- cache/session/dispatcher 설정 가능 +- deep crawl은 domain/path/rate limit 정책을 지킴 + +### Phase 8. Research Loop 통합 + +목표: + +- 사용자가 seed URL을 몰라도 지식 공백을 기반으로 외부 자료를 찾는다. + +작업: + +1. `ExternalResearchLoop` 추가 +2. search provider interface 작성 +3. query/usefulness/context/next-decision JSON schema 작성 +4. KnowledgeGap과 ResearchSession 연결 +5. 유용 context를 Document/ContentUnit/Candidate로 저장 + +완료 조건: + +- gap -> search query -> useful page -> evidence context -> candidate 저장 +- session history/cost/progress/cancel 지원 + +### Phase 9. Firecrawl optional adapter 검토 + +목표: + +- 필요할 때만 Firecrawl을 외부 수집 서비스로 붙인다. + +조건: + +- Crawl4AI 운영이 불안정하거나, Firecrawl의 self-host scrape/map/crawl API가 운영상 더 낫다고 판단될 때만 진행 +- AGPL/라이선스 검토 완료 +- TypeScript service를 Python 코드베이스에 직접 병합하지 않음 + +완료 조건: + +- `FirecrawlAdapter`가 외부 HTTP API만 호출 +- core ontology pipeline은 Firecrawl에 의존하지 않음 + +## 7. 개발 지시 원칙 + +다른 AI 에이전트에게 작업시킬 때 반드시 지킬 원칙: + +1. 한 번에 하나의 오픈소스만 붙인다. +2. 원본 코드를 직접 대량 수정하지 말고 adapter/facade를 만든다. +3. 외부 프로젝트 API가 흔들리면 wrapper만 고친다. +4. canonical data contract는 `Document`, `ContentUnit`, `Candidate`, `Evidence`, `GraphDelta`, `ValidationRun`이다. +5. published graph는 review/approval 없이 변경하지 않는다. +6. Firecrawl과 Crawl4AI를 같은 기본 crawler로 동시에 쓰지 않는다. +7. OntoCast와 Neo4j GraphRAG의 역할을 섞지 않는다. OntoCast는 RDF 생성/갱신, Neo4j는 projection/search/RAG다. +8. Guardrails를 통과하지 않은 LLM output은 DB에 확정 저장하지 않는다. +9. evidence/provenance 없는 entity/relation/triple은 폐기하거나 review queue로 보낸다. +10. 각 단계마다 unit test, integration test, DB migration test, sample data smoke test를 통과한 뒤 다음 단계로 간다. + +## 8. 최종 권장 구현 순서 요약 + +가장 안전한 순서: + +1. 현재 `crawler_platform` 기준선 테스트 고정 +2. 공통 데이터 계약 확장 +3. Trafilatura adapter +4. OntoCast core facade +5. Guardrails validation gate +6. review/versioning workflow +7. Neo4j GraphRAG projection/search +8. Crawl4AI dynamic crawler +9. Knowledge Agent/OpenDeepResearcher 기반 ResearchLoop +10. Firecrawl optional adapter + +가장 중요한 결정: + +- 기본 프로젝트: 현재 `crawler_platform` +- 기본 온톨로지 엔진: `OntoCast` +- 기본 웹 정제 엔진: `Trafilatura` +- 기본 검증 엔진: `Guardrails` +- 기본 검색/RAG 엔진: `Neo4j GraphRAG` +- 동적 크롤링 엔진: `Crawl4AI` +- Firecrawl: 초기 제외, 설계 참고 또는 선택 외부 서비스 + +이 설계는 “있는 소스를 최대한 그대로 쓰되, 제품으로 필요한 연결/검수/버전/운영 계층만 우리가 만든다”는 방향이다. 실제 통합 코드는 얇은 adapter와 안정된 내부 데이터 계약 위에 쌓아야 한다. diff --git a/Playwright_분석_및_기능명세.md b/Playwright_분석_및_기능명세.md new file mode 100644 index 0000000..c46e1e8 --- /dev/null +++ b/Playwright_분석_및_기능명세.md @@ -0,0 +1,882 @@ +# Playwright 분석 및 기능명세 + +작성일: 2026-05-13 +분석 대상: `C:\Users\lasta\MyProject\AI\참고\playwright-main` +프로젝트 성격: Microsoft Playwright 원본 모노레포 계열, Apache-2.0 라이선스 + +## 1. 요약 + +Playwright는 Chromium, Firefox, WebKit을 단일 API로 제어하는 브라우저 자동화 프레임워크다. 이 저장소는 단순 웹 크롤러가 아니라 다음 요소를 모두 포함한 대형 플랫폼이다. + +- 브라우저 실행 및 원격 제어 런타임 +- 브라우저 컨텍스트, 페이지, 프레임, 네트워크, 입력, 다운로드, 쿠키, 저장소 API +- E2E 테스트 러너와 fixture, worker, reporter, assertion 체계 +- 브라우저 세션 추적, 스크린샷, 비디오, HAR, trace viewer +- 코드 생성기, recorder, inspector, HTML reporter +- MCP/CLI 기반 AI agent용 브라우저 조작 도구 +- 브라우저별 패치와 배포 패키지 구성 + +범용 온톨로지 구축 플랫폼 관점에서 가장 가치 있는 부분은 테스트 러너 자체보다 `playwright-core`의 브라우저 자동화 계층, 네트워크/DOM/접근성 스냅샷 수집 계층, trace/HAR 증거화 계층, MCP/CLI 도구 계층이다. 사이트 탐색, 구조화 정보 추출, 출처 증거 보존, 동적 웹 페이지 처리, 로그인 세션 재사용, 수집 품질 검증에 거의 그대로 사용할 수 있다. + +## 2. 저장소 구조 + +주요 루트 디렉터리: + +- `packages`: 실제 제품 패키지와 런타임 소스 +- `packages/playwright-core`: 브라우저 자동화 핵심 +- `packages/playwright`: 테스트 러너, CLI, reporter, worker, fixture +- `packages/trace-viewer`: trace zip 시각화 UI +- `packages/html-reporter`: 테스트 결과/실행 결과 HTML 리포트 UI +- `packages/recorder`: 코드 생성/recording 관련 UI 및 로직 +- `packages/dashboard`: Playwright CLI 세션 모니터링 대시보드 +- `docs`: 공식 문서 원본 +- `tests`: 브라우저, 테스트 러너, MCP, 컴포넌트 테스트 등 검증 코드 +- `utils`: 빌드, 타입 생성, 문서 lint, 브라우저 롤링, 패키징 도구 +- `browser_patches`: 브라우저별 패치 관리 + +패키지 목록 중 중요 항목: + +- `playwright-core`: 브라우저 제어 엔진. 플랫폼에서 직접 재사용할 최우선 후보. +- `playwright`: test runner와 reporter. 플랫폼 내부 검증 자동화나 수집 시나리오 검증에 선택적으로 사용. +- `playwright-client`: 클라이언트 번들. +- `protocol`: client-server channel protocol 정의. +- `injected`: 브라우저 페이지 안에 주입되는 selector, locator, utility 스크립트. +- `trace`, `trace-viewer`: 실행 증거, DOM snapshot, network, console, screenshot을 재생/분석. +- `html-reporter`: 실행 결과 UI. +- `recorder`: 사용자 행동을 자동화 코드로 변환. +- `playwright-ct-*`: React/Vue 컴포넌트 테스트. 온톨로지 플랫폼에는 직접 우선순위 낮음. +- `playwright-browser-*`, `playwright-chromium/firefox/webkit`: 브라우저별 배포 패키지. + +## 3. 기술 스택 + +- 언어: TypeScript, JavaScript +- 런타임: Node.js `>=18` +- 패키지 구조: npm workspaces +- 빌드: 자체 `utils/build/build.js`, esbuild, TypeScript +- UI: React 기반 trace viewer, html reporter, dashboard +- 통신: WebSocket, pipe transport, CDP, WebDriver BiDi, 자체 channel protocol +- schema/agent tool: `zod`, `@modelcontextprotocol/sdk` +- 라이선스: Apache-2.0 + +## 4. 핵심 아키텍처 + +Playwright의 핵심 구조는 server/client/protocol로 나뉜다. + +### 4.1 Server 계층 + +위치: `packages/playwright-core/src/server` + +Server 계층은 실제 브라우저 프로세스를 띄우고, 브라우저별 프로토콜을 다루며, 페이지/프레임/네트워크/입력/저장소 같은 고수준 객체를 제공한다. + +핵심 파일: + +- `playwright.ts`: Chromium, Firefox, WebKit, Electron, Android 객체를 생성하는 루트 객체. +- `browserType.ts`: 브라우저 launch/connect/persistent context 진입점. +- `browser.ts`: 브라우저 연결과 context 생명주기. +- `browserContext.ts`: 독립 세션, 쿠키, 권한, 라우팅, tracing, storage state. +- `page.ts`: 페이지 단위 조작과 이벤트. +- `frames.ts`: frame navigation, lifecycle, DOM interaction. +- `network.ts`: request/response, routing, headers, timing. +- `fetch.ts`: API request context. +- `selectors.ts`, `frameSelectors.ts`, `dom.ts`: locator/selector 기반 DOM 조작. +- `screenshotter.ts`, `videoRecorder.ts`, `trace`: 증거 수집. +- `chromium`, `firefox`, `webkit`, `bidi`: 브라우저별 protocol adapter. +- `dispatchers`: server 객체를 channel protocol로 노출. + +온톨로지 플랫폼 재사용 포인트: + +- 동적 페이지 렌더링 후 본문/링크/메타데이터 추출 +- SPA, 무한 스크롤, 로그인 뒤 페이지 수집 +- request/response 기반 원천 URL, MIME, redirect, status 기록 +- DOM snapshot, screenshot, video, trace를 출처 증거로 보존 +- 브라우저 context 단위 격리로 사이트별 정책/쿠키/세션 분리 + +### 4.2 Client 계층 + +위치: `packages/playwright-core/src/client` + +Client 계층은 사용자가 보는 API 객체다. `Playwright`, `BrowserType`, `Browser`, `BrowserContext`, `Page`, `Locator`, `Frame`, `Request`, `Response` 등이 channel protocol 위에서 동작한다. + +핵심 파일: + +- `playwright.ts`: client-side 루트 객체와 브라우저 타입 접근. +- `browserType.ts`: `launch`, `connect`, `launchPersistentContext`. +- `browser.ts`: browser/context 관리. +- `browserContext.ts`: 쿠키, route, tracing, storage state. +- `page.ts`: navigation, screenshot, PDF, event, locator. +- `locator.ts`: 안정적인 DOM 대상 지정. +- `network.ts`: request/response 모델. +- `tracing.ts`: trace start/stop. +- `fetch.ts`: APIRequestContext. + +온톨로지 플랫폼에서는 이 client API를 감싸는 `BrowserAcquisitionService` 또는 `WebEvidenceCollector`를 두는 것이 적합하다. 원본 API를 변경하지 않고 domain workflow만 추가하면 유지보수가 쉽다. + +### 4.3 Protocol/Dispatcher 계층 + +위치: + +- `packages/playwright-core/src/protocol` +- `packages/playwright-core/src/server/dispatchers` +- `packages/protocol` + +Server 객체와 client 객체 사이의 메시지 계약이다. 브라우저 객체를 직접 넘기지 않고 channel owner/dispatcher로 추상화한다. + +재사용 의미: + +- 장기적으로 Python/FastAPI 백엔드와 Node Playwright worker를 분리할 때 이 구조를 참고할 수 있다. +- 온톨로지 플랫폼의 “수집 작업 서버”도 command/event protocol로 설계하면 browser worker를 독립 프로세스로 운용하기 쉽다. + +### 4.4 Injected 계층 + +위치: `packages/injected` + +브라우저 페이지 내부에 주입되어 selector, locator, accessibility 기반 검색, DOM 조작 보조를 수행한다. Playwright의 강점인 auto-wait와 locator 안정성은 이 계층과 server/client 조합에서 나온다. + +온톨로지 플랫폼 재사용 포인트: + +- 단순 CSS selector보다 안정적인 요소 선택 +- accessible role/name 기반 탐색 +- 클릭/입력 전 요소 actionability 확인 +- 의미 있는 DOM 후보 추출의 기반 + +### 4.5 Tools/MCP/CLI 계층 + +위치: `packages/playwright-core/src/tools` + +AI agent와 CLI가 브라우저를 조작할 수 있게 도구 단위로 기능을 쪼갠 계층이다. + +주요 하위 디렉터리: + +- `backend`: 실제 브라우저 조작 tool 구현 +- `mcp`: Model Context Protocol 서버 및 브라우저 모델 +- `cli-client`: command-line agent client +- `cli-daemon`: browser session daemon +- `dashboard`: 실행 중인 browser session 시각화 +- `trace`: trace 분석용 CLI + +`backend/tools.ts`는 다음 도구 묶음을 등록한다. + +- navigation +- screenshot +- snapshot +- form +- keyboard/mouse +- network/route +- cookies/storage/webstorage +- evaluate/runCode +- pdf/video/tracing +- tabs/dialogs/files/devtools +- verify/wait/console + +온톨로지 플랫폼에서 매우 중요하다. “LLM이 브라우저를 조작해 정보원을 탐색하고, 구조화 데이터를 추출하고, 증거를 남기는” 기능을 만들 때 이 도구 계층을 거의 그대로 감싸서 사용할 수 있다. + +## 5. 주요 기능 분석 + +### 5.1 브라우저 자동화 + +기능: + +- Chromium, Firefox, WebKit 실행 +- headless/headed 모드 +- browser context 격리 +- persistent context 지원 +- proxy, geolocation, timezone, locale, permissions, viewport, user agent 설정 +- page/frame navigation +- click, fill, type, press, hover, drag 등 입력 자동화 +- dialog, download, file chooser 처리 + +온톨로지 플랫폼 적용: + +- 동적 문서 페이지 렌더링 +- 검색 엔진/사이트 내 검색 자동화 +- 페이지 내 탭, 필터, 페이지네이션 탐색 +- 로그인 필요 지식베이스 접근 +- 사이트별 수집 프로파일 구성 + +### 5.2 Locator와 auto-wait + +기능: + +- `getByRole`, `getByText`, `getByLabel`, `getByPlaceholder`, `getByTestId` +- CSS/XPath selector +- element actionability 자동 대기 +- assertion retry +- strict locator 정책 + +적용: + +- 사이트 UI가 느리게 로딩되어도 수집 안정성 확보 +- 관리자 콘솔/문서 포털/검색 UI 자동화 +- DOM 변화가 잦은 사이트에서 selector 취약성 감소 + +### 5.3 네트워크 관찰 및 제어 + +기능: + +- request/response 이벤트 +- route interception +- HAR recording/replay +- header/cookie/postData/status/timing 접근 +- APIRequestContext +- WebSocket route 일부 지원 + +적용: + +- 수집 문서의 원천 URL, redirect chain, status code, content-type 저장 +- JSON API가 노출되는 사이트에서 DOM 대신 API 응답 직접 추출 +- 크롤링 금지/인증 오류/레이트리밋 감지 +- 동일 URL 재수집 시 변경 여부 판단 + +### 5.4 스냅샷, 스크린샷, 비디오, trace + +기능: + +- screenshot +- video recording +- trace start/stop +- DOM snapshot +- console/network/action timeline 기록 +- trace viewer UI + +적용: + +- 온톨로지 엔티티/관계 추출의 근거 보존 +- LLM 추출 결과 검수 화면 제공 +- “왜 이 관계가 생성되었는가”를 클릭 가능한 증거로 제시 +- 수집 실패 재현 및 디버깅 + +### 5.5 PDF와 문서화 + +기능: + +- Chromium 기반 `page.pdf` +- screenshot 기반 시각 증거 + +적용: + +- 웹 문서를 PDF evidence artifact로 저장 +- 온톨로지 버전별 출처 snapshot 생성 + +### 5.6 테스트 러너 + +위치: `packages/playwright/src` + +기능: + +- test/expect API +- fixture +- parallel worker +- retry, timeout, shard +- project matrix +- reporter: list, line, dot, json, junit, html, blob, github +- webServer plugin +- watch/UI mode + +온톨로지 플랫폼에서는 제품 테스트뿐 아니라 “수집 recipe 검증”에 사용할 수 있다. 예를 들어 특정 사이트 수집 recipe가 정상적으로 title/body/date/source evidence를 얻는지 Playwright Test로 검증할 수 있다. + +### 5.7 Recorder와 codegen + +기능: + +- 브라우저 조작을 코드로 생성 +- selector 후보 생성 +- 사용자의 실제 클릭/입력을 시나리오로 변환 + +적용: + +- 비개발자가 사이트 수집 절차를 녹화해 recipe 초안 생성 +- 수집 자동화 script를 빠르게 제작 +- 로그인, 검색, 필터, 다운로드 흐름을 저장 + +### 5.8 MCP와 AI Agent 브라우저 조작 + +기능: + +- MCP server 제공 +- 접근성 tree/snapshot 기반 agent interaction +- navigation, click, type, screenshot, network, storage 등 tool schema +- extension/CDP relay 구조 일부 포함 + +적용: + +- LLM 기반 웹 탐색 agent +- 온톨로지 후보 개념/관계 발견을 위한 반자동 탐색 +- 사람이 지시한 목표를 브라우저 조작 task로 변환 +- “페이지에서 제품군/속성/관계 후보를 찾아라” 같은 agent workflow + +### 5.9 Reporter와 Viewer + +기능: + +- HTML reporter +- trace viewer +- timeline, action list, network tab, console tab, snapshot tab +- test result drill-down + +적용: + +- 수집 작업 리포트 UI의 기본 소스로 사용 가능 +- 추출 품질, 실패 URL, 에러, 네트워크 로그, 스크린샷을 한 화면에서 검토 +- provenance/evidence viewer 구현 참고 + +## 6. 범용 온톨로지 구축 플랫폼에 필요한 기능명세 + +아래 명세는 Playwright 원본을 가능한 한 변형 없이 사용하고, 우리 플랫폼 계층에서 orchestration과 domain logic을 얹는 방향이다. + +### 6.1 브라우저 수집 엔진 + +목적: 동적 웹 페이지를 안정적으로 열고, DOM/텍스트/네트워크/시각 증거를 수집한다. + +기능 요구사항: + +- URL 단위 수집 작업 생성 +- browser type 선택: chromium 기본, 필요 시 firefox/webkit +- headless/headed 선택 +- context 설정: viewport, locale, timezone, userAgent, proxy, geolocation, permissions +- navigation timeout, action timeout 설정 +- 페이지 load strategy 설정: `load`, `domcontentloaded`, `networkidle` +- redirect chain 기록 +- final URL 기록 +- HTTP status, response headers, content-type 기록 +- DOM HTML 저장 +- innerText/textContent 저장 +- screenshot 저장 +- 선택적 PDF 저장 +- 선택적 trace 저장 +- console error/warning 기록 +- request failure 기록 +- cookie/storage state 저장 및 재사용 + +권장 원본 사용: + +- `playwright-core/src/client/page.ts` +- `browserContext.ts` +- `network.ts` +- `tracing.ts` +- `screenshotter.ts` +- `fetch.ts` + +플랫폼 래퍼 예시: + +- `BrowserAcquisitionService.collect(url, profile)` +- `EvidenceBundle` +- `BrowserSessionProfile` +- `NetworkEvidence` +- `DomEvidence` + +### 6.2 사이트 탐색 및 링크 발견 + +목적: 온톨로지 구축에 필요한 문서/목록/상세 페이지 후보를 발견한다. + +기능 요구사항: + +- 시작 URL seed 등록 +- 동일 도메인/허용 도메인 링크 추출 +- link text, href, role, bounding box, surrounding text 기록 +- canonical URL 정규화 +- 중복 URL 제거 +- robots/policy는 플랫폼 정책 계층에서 처리 +- 페이지네이션 버튼 탐색 +- 검색어 기반 사이트 내부 검색 수행 +- 무한 스크롤 페이지 처리 +- sitemap/API endpoint 발견은 별도 모듈과 결합 + +권장 원본 사용: + +- locator +- frame/page evaluate +- network request observation +- tools backend `navigate`, `snapshot`, `mouse`, `keyboard`, `wait` + +### 6.3 구조화 추출 준비 + +목적: LLM/규칙 기반 추출기가 쓰기 좋은 입력을 만든다. + +기능 요구사항: + +- 본문 후보 영역 탐지 +- 제목, heading hierarchy 추출 +- table/list/card 구조 추출 +- form/search/filter UI 추출 +- image alt/caption/source 추출 +- metadata: title, description, og tags, schema.org JSON-LD 추출 +- accessibility snapshot 저장 +- network JSON 응답 후보 저장 +- DOM path와 locator candidate 저장 + +권장 원본 사용: + +- injected selector/locator 구조 +- page accessibility snapshot 계열 도구 +- `snapshot` backend tool +- evaluate/runCode tool + +주의: + +- Playwright는 온톨로지 추출기가 아니다. 엔티티/관계/속성 스키마 추출은 플랫폼의 별도 AI extraction layer가 담당해야 한다. +- Playwright는 “신뢰도 높은 웹 상태와 증거를 제공하는 하부 엔진”으로 두는 것이 맞다. + +### 6.4 Agent 기반 웹 조사 + +목적: LLM이 브라우저를 조작하며 지식 후보를 찾고 검증하게 한다. + +기능 요구사항: + +- MCP tool 목록을 플랫폼 agent에게 제공 +- agent별 browser context 격리 +- 세션별 action log 저장 +- agent action마다 screenshot/snapshot 선택 저장 +- 허용 도메인, 다운로드, 파일 업로드, 외부 이동 제한 +- 사람이 중간에 개입 가능한 headed/session dashboard 제공 +- agent task 결과를 evidence bundle과 연결 + +권장 원본 사용: + +- `packages/playwright-core/src/tools/backend` +- `packages/playwright-core/src/tools/mcp` +- `packages/playwright-core/src/tools/cli-daemon` +- `packages/playwright-core/src/tools/dashboard` + +플랫폼 기능명: + +- `AgentBrowserSession` +- `BrowserToolGateway` +- `AgentEvidenceRecorder` +- `HumanReviewDashboard` + +### 6.5 수집 Recipe 녹화 및 재생 + +목적: 사용자가 사이트별 수집 절차를 만들고 반복 실행한다. + +기능 요구사항: + +- 브라우저 조작 녹화 +- 생성된 locator/code 확인 +- recipe step 편집 +- 변수화: 검색어, 카테고리, 기간, 페이지 수 +- replay 실행 +- 실패 step에서 screenshot/trace 제공 +- recipe version 관리 + +권장 원본 사용: + +- `packages/recorder` +- `packages/playwright-core/src/server/recorder` +- `packages/playwright-core/src/server/codegen` + +플랫폼 Recipe 모델: + +- `open(url)` +- `click(locator)` +- `fill(locator, value)` +- `press(key)` +- `waitFor(condition)` +- `extract(targetSpec)` +- `paginate(strategy)` +- `saveEvidence(policy)` + +### 6.6 증거/출처 관리 + +목적: 온톨로지 결과의 출처와 재현성을 보장한다. + +기능 요구사항: + +- 모든 추출 결과는 source URL과 evidence id를 가진다. +- evidence bundle에는 HTML, text, screenshot, network summary, trace path를 포함한다. +- relationship triple마다 근거 DOM locator 또는 text span을 연결한다. +- trace viewer 또는 유사 UI에서 action/network/snapshot을 열람한다. +- 재수집 시 이전 evidence와 diff한다. + +권장 원본 사용: + +- tracing +- HAR +- trace viewer +- html reporter UI 구조 +- network events + +플랫폼 데이터 모델: + +- `EvidenceBundle(id, url, capturedAt, browserProfile, artifacts)` +- `Artifact(type, path, mime, hash)` +- `ExtractionClaim(entityId, predicate, object, evidenceRefs, confidence)` +- `SourceSpan(evidenceId, selector, textStart, textEnd, quote)` + +### 6.7 수집 품질 검증 + +목적: 수집 및 추출 pipeline의 신뢰성을 자동 검증한다. + +기능 요구사항: + +- URL 접근 성공률 +- 본문 길이 최소 기준 +- title/heading 존재 여부 +- HTTP status allowlist +- screenshot blank 여부 +- 주요 selector 존재 여부 +- JSON-LD/schema.org 존재 여부 +- extraction output schema validation +- 실패 시 retry, fallback browser, fallback wait strategy + +권장 원본 사용: + +- Playwright Test runner +- expect matcher +- html reporter +- trace on retry + +플랫폼 기능명: + +- `CrawlerRecipeTest` +- `EvidenceQualityGate` +- `ExtractionRegressionSuite` + +## 7. 재사용 우선순위 + +### 1순위: 거의 그대로 사용 + +- npm 패키지 `playwright` 또는 `playwright-core` +- browser/page/context/network/locator/tracing API +- screenshot/PDF/HAR/trace 기능 +- storage state 재사용 +- MCP/CLI backend tool 개념 + +이 영역은 원본 수정 없이 wrapper를 작성하는 방식이 적합하다. + +### 2순위: 일부 UI/구조 차용 + +- trace viewer +- html reporter +- dashboard +- recorder/codegen + +이 영역은 UI와 데이터 모델이 Playwright 테스트 중심이라 그대로 붙이기보다 “evidence viewer”, “collection report”, “recipe recorder”로 재명명하고 데이터 adapter를 두는 것이 좋다. + +### 3순위: 참고만 권장 + +- browser patches +- component test packages +- browser package publishing logic +- Playwright 자체 protocol generator/build system + +온톨로지 플랫폼에는 과하고 유지보수 비용이 높다. + +## 8. 통합 설계안 + +권장 구조: + +```text +Ontology Platform + API / Job Orchestrator + CollectionJob + ExtractionJob + ValidationJob + + Browser Automation Layer + Playwright wrapper + Browser session pool + Site profile manager + Agent tool gateway + + Evidence Layer + HTML/Text/Screenshot/PDF/Trace/HAR store + Evidence metadata DB + Hash/version manager + + Extraction Layer + DOM cleaner + JSON-LD parser + Table/list extractor + LLM extractor + Ontology mapper + + Ontology Layer + Entity model + Relation model + Schema/versioning + Graph DB adapter + + Review UI + Evidence viewer + Trace viewer adapter + Extraction diff + Human validation workflow +``` + +Playwright는 `Browser Automation Layer`와 `Evidence Layer`의 핵심 엔진으로 둔다. 온톨로지 의미 추론, 스키마 정렬, 엔티티 병합, 그래프 저장은 별도 계층으로 분리한다. + +## 9. 구체 API 명세 초안 + +### 9.1 CollectionProfile + +```ts +type CollectionProfile = { + id: string; + browser: 'chromium' | 'firefox' | 'webkit'; + headless: boolean; + viewport?: { width: number; height: number }; + locale?: string; + timezoneId?: string; + userAgent?: string; + proxy?: { + server: string; + username?: string; + password?: string; + }; + permissions?: string[]; + storageStatePath?: string; + navigationTimeoutMs: number; + actionTimeoutMs: number; + trace: 'off' | 'on' | 'retain-on-failure'; + screenshot: 'off' | 'page' | 'full-page'; + pdf: boolean; + har: boolean; +}; +``` + +### 9.2 CollectionJob + +```ts +type CollectionJob = { + id: string; + seeds: string[]; + profileId: string; + allowedDomains: string[]; + maxDepth: number; + maxPages: number; + crawlMode: 'single-page' | 'same-domain' | 'recipe' | 'agent'; + recipeId?: string; + agentGoal?: string; + evidencePolicy: EvidencePolicy; +}; +``` + +### 9.3 EvidencePolicy + +```ts +type EvidencePolicy = { + keepHtml: boolean; + keepText: boolean; + keepScreenshot: boolean; + keepPdf: boolean; + keepTrace: boolean; + keepHar: boolean; + keepNetworkSummary: boolean; + hashArtifacts: boolean; +}; +``` + +### 9.4 EvidenceBundle + +```ts +type EvidenceBundle = { + id: string; + jobId: string; + url: string; + finalUrl: string; + capturedAt: string; + status?: number; + contentType?: string; + title?: string; + artifacts: EvidenceArtifact[]; + network: NetworkEvidence[]; + console: ConsoleEvidence[]; + extractionInput: ExtractionInput; +}; +``` + +### 9.5 ExtractionInput + +```ts +type ExtractionInput = { + title?: string; + headings: Array<{ level: number; text: string; selector?: string }>; + mainText: string; + links: Array<{ text: string; href: string; selector?: string }>; + tables: Array<{ selector?: string; rows: string[][] }>; + jsonLd: unknown[]; + metadata: Record; + accessibilitySnapshot?: unknown; +}; +``` + +### 9.6 OntologyExtractionResult + +```ts +type OntologyExtractionResult = { + evidenceBundleId: string; + entities: ExtractedEntity[]; + relations: ExtractedRelation[]; + attributes: ExtractedAttribute[]; + warnings: string[]; +}; +``` + +### 9.7 ExtractedEntity + +```ts +type ExtractedEntity = { + id: string; + label: string; + typeCandidates: string[]; + aliases: string[]; + sourceRefs: SourceRef[]; + confidence: number; +}; +``` + +### 9.8 ExtractedRelation + +```ts +type ExtractedRelation = { + subjectId: string; + predicate: string; + objectIdOrValue: string; + relationType: 'entity-entity' | 'entity-value'; + sourceRefs: SourceRef[]; + confidence: number; +}; +``` + +### 9.9 SourceRef + +```ts +type SourceRef = { + evidenceBundleId: string; + artifactType: 'html' | 'text' | 'screenshot' | 'pdf' | 'trace' | 'network'; + selector?: string; + textQuote?: string; + startOffset?: number; + endOffset?: number; + screenshotRegion?: { x: number; y: number; width: number; height: number }; +}; +``` + +## 10. 주요 워크플로우 명세 + +### 10.1 단일 URL 수집 + +1. CollectionJob 생성 +2. CollectionProfile 로드 +3. Playwright browser/context/page 생성 +4. URL 이동 +5. response/status/final URL 기록 +6. DOM, text, metadata, link, JSON-LD 추출 +7. screenshot/PDF/trace/HAR 저장 +8. EvidenceBundle 생성 +9. ExtractionInput 생성 +10. Ontology extraction queue로 전달 + +### 10.2 사이트 탐색 수집 + +1. seed URL 수집 +2. 링크 후보 추출 +3. URL canonicalization 및 domain filter +4. 우선순위 큐에 추가 +5. maxDepth/maxPages까지 반복 +6. 각 페이지 EvidenceBundle 저장 +7. 중복 본문/중복 URL 제거 +8. extraction batch 생성 + +### 10.3 Agent 조사 + +1. 사용자가 조사 목표 입력 +2. AgentBrowserSession 생성 +3. MCP/backend tools 제공 +4. agent가 navigate/search/click/snapshot 반복 +5. 중요 페이지에서 evidence 저장 +6. agent가 후보 entity/relation/sourceRefs 제안 +7. 사람이 evidence viewer에서 검수 +8. 승인된 claim만 ontology graph에 반영 + +### 10.4 Recipe 생성 + +1. 사용자가 headed browser로 사이트 접속 +2. recorder가 행동 기록 +3. codegen/locator 후보 생성 +4. 플랫폼 Recipe DSL로 변환 +5. 변수와 반복/페이지네이션 설정 +6. 샘플 실행으로 검증 +7. recipe version 저장 + +## 11. 변경 없이 사용 가능한 코드/개념 + +- Playwright npm public API +- browser context isolation +- locator and auto-wait +- tracing API +- storage state +- route/network observation +- screenshot/PDF +- API request context +- HTML reporter/trace viewer의 UI 패턴 +- MCP backend tool 분해 방식 + +## 12. 수정 또는 Adapter가 필요한 영역 + +- Playwright test result 중심 데이터 모델을 ontology evidence 중심 모델로 변환 +- trace viewer를 EvidenceBundle과 연결하는 adapter +- recorder output을 플랫폼 Recipe DSL로 변환 +- MCP tool 권한/보안 정책 +- 수집 대상 도메인 제한 +- 다운로드/파일 시스템 접근 제한 +- 대량 크롤링 스케줄링, 큐, retry, backpressure +- robots/약관/레이트리밋 정책 + +## 13. 위험요소와 주의사항 + +- Playwright 원본 전체를 fork해서 수정하면 유지보수 비용이 매우 커진다. +- 브라우저 바이너리와 patch 관리까지 직접 들고 가는 것은 권장하지 않는다. +- 크롤러 규모가 커지면 browser context/page pool 관리가 필요하다. +- trace/video/screenshot은 저장소 비용이 크므로 evidence policy가 필요하다. +- 로그인 세션 저장은 보안 민감 정보이므로 암호화와 접근 제어가 필요하다. +- LLM agent에게 unrestricted browser tool을 주면 외부 이동/다운로드/입력 위험이 있다. +- Playwright는 추출 의미론을 보장하지 않는다. 온톨로지 품질은 extraction/validation layer에서 관리해야 한다. + +## 14. 구현 로드맵 + +### Phase 1. Playwright 수집 래퍼 + +- `BrowserAcquisitionService` 작성 +- 단일 URL HTML/text/screenshot/network summary 수집 +- storage state 지원 +- EvidenceBundle 저장 + +### Phase 2. 구조화 입력 생성 + +- heading/link/table/jsonLd/metadata 추출 +- 본문 후보 추출 +- extraction input schema 고정 +- 품질 gate 추가 + +### Phase 3. Recipe 기반 수집 + +- Playwright action step DSL 정의 +- recorder/codegen 연계 검토 +- replay 및 실패 trace 저장 + +### Phase 4. Agent 브라우저 + +- MCP/backend tools adapter +- agent session isolation +- action log/evidence 자동 연결 +- 도메인/권한 policy 적용 + +### Phase 5. Evidence Viewer + +- trace viewer 또는 유사 UI 통합 +- screenshot/DOM/text/source span 연결 +- relation claim 검수 화면 + +## 15. 결론 + +이 프로젝트는 범용 온톨로지 구축 플랫폼의 “웹 기반 지식 수집 엔진”으로 매우 적합하다. 다만 원본을 플랫폼 내부로 깊게 fork하기보다, Playwright는 가능한 한 공식 API와 tool 계층을 그대로 사용하고, 우리 쪽에서 다음 계층을 추가하는 방식이 좋다. + +- 수집 orchestration +- evidence data model +- ontology extraction input normalization +- LLM/규칙 기반 entity/relation extraction +- graph persistence +- human review workflow + +즉, Playwright는 “브라우저로 세상을 안정적으로 관찰하고 증거를 남기는 엔진”으로 쓰고, 범용 온톨로지 플랫폼은 그 위에서 “관찰을 지식 그래프로 바꾸는 시스템”으로 설계하는 것이 가장 현실적이다. diff --git a/configs/perfume_subscription.yaml b/configs/perfume_subscription.yaml index bdd0524..a0dbe5c 100644 --- a/configs/perfume_subscription.yaml +++ b/configs/perfume_subscription.yaml @@ -34,21 +34,21 @@ sources: parser: generic fetcher: playwright rate_limit_per_minute: 20 - respect_robots_txt: true + respect_robots_txt: false - name: marketplace type: marketplace trust_level: 0.8 parser: generic fetcher: requests rate_limit_per_minute: 15 - respect_robots_txt: true + respect_robots_txt: false - name: review_site type: review trust_level: 0.7 parser: generic fetcher: requests rate_limit_per_minute: 10 - respect_robots_txt: true + respect_robots_txt: false ontology: entity_types: - Product diff --git a/crawler_platform.db b/crawler_platform.db index 17d148b..4fd6c81 100644 Binary files a/crawler_platform.db and b/crawler_platform.db differ diff --git a/crawler_platform/app/api/routes.py b/crawler_platform/app/api/routes.py index fc44ff4..5a8fbb3 100644 --- a/crawler_platform/app/api/routes.py +++ b/crawler_platform/app/api/routes.py @@ -34,6 +34,8 @@ class CrawlRequest(BaseModel): extractor_provider: str = "lm_studio" extractor_model: str | None = None extractor_base_url: str | None = "http://localhost:1234/v1" + check_robots_txt: bool = False + respect_robots_txt: bool | None = None class SiteCrawlRequest(CrawlRequest): @@ -48,6 +50,8 @@ class DiscoverRequest(BaseModel): source_name: str url: str limit: int = 30 + check_robots_txt: bool = False + respect_robots_txt: bool | None = None class RecommendRequest(BaseModel): @@ -145,6 +149,13 @@ def _apply_claim_review(claim: models.Claim, status: str, reason: str | None) -> claim.last_seen_at = models.utcnow() +def apply_crawl_request_overrides(config, request: CrawlRequest | DiscoverRequest) -> None: + check_robots_txt = request.respect_robots_txt + if check_robots_txt is None: + check_robots_txt = request.check_robots_txt + config.source_by_name(request.source_name).respect_robots_txt = check_robots_txt + + def crawl_job_response(job: models.CrawlJob) -> dict[str, Any]: metadata = job.metadata_json or {} return { @@ -176,6 +187,7 @@ def run_site_crawl_job(database_url: str, job_id: int, request_data: dict[str, A request = SiteCrawlRequest(**request_data) try: config = load_project_config(request.config_path) + apply_crawl_request_overrides(config, request) with session_scope(database_url) as session: job = session.get(models.CrawlJob, job_id) if job is None: @@ -444,6 +456,7 @@ def register_routes(app, database_url: str) -> None: @app.post("/crawl") def crawl(request: CrawlRequest): config = load_project_config(request.config_path) + apply_crawl_request_overrides(config, request) with session_scope(database_url) as session: repo = KnowledgeRepository(session) pipeline = CrawlPipeline( @@ -474,6 +487,7 @@ def register_routes(app, database_url: str) -> None: @app.post("/crawl-site") def crawl_site(request: SiteCrawlRequest, background_tasks: BackgroundTasks): config = load_project_config(request.config_path) + apply_crawl_request_overrides(config, request) with session_scope(database_url) as session: repo = KnowledgeRepository(session) project = repo.upsert_project(config) @@ -526,10 +540,18 @@ def register_routes(app, database_url: str) -> None: @app.post("/discover") def discover(request: DiscoverRequest): config = load_project_config(request.config_path) + apply_crawl_request_overrides(config, request) source_config = config.source_by_name(request.source_name) robots = RobotsPolicy() - if not robots.allowed(request.url, source_config.respect_robots_txt): - return {"ok": False, "error": "robots.txt does not allow discovery for this URL", "links": []} + robots_decision = robots.check(request.url, source_config.respect_robots_txt) + if not robots_decision.allowed: + return { + "ok": False, + "error": f"{robots_decision.reason}: {request.url}", + "robots_status": robots_decision.status, + "robots_reason": robots_decision.reason, + "links": [], + } fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute) result = fetcher.fetch(request.url) links = discover_links(result.analysis_html, result.final_url or request.url, request.limit) @@ -538,6 +560,8 @@ def register_routes(app, database_url: str) -> None: "status_code": result.status_code, "final_url": result.final_url, "crawl_status": result.crawl_status, + "robots_status": robots_decision.status, + "robots_reason": robots_decision.reason, "warnings": result.warnings, "links": [asdict(link) for link in links], } @@ -545,6 +569,7 @@ def register_routes(app, database_url: str) -> None: @app.post("/research/run") def run_research(request: ResearchRunRequest): config = load_project_config(request.config_path) + apply_crawl_request_overrides(config, request) if request.project_name and request.project_name != config.project_name: raise HTTPException( status_code=400, diff --git a/crawler_platform/app/config/loader.py b/crawler_platform/app/config/loader.py index 958d3f4..332d117 100644 --- a/crawler_platform/app/config/loader.py +++ b/crawler_platform/app/config/loader.py @@ -16,7 +16,7 @@ class SourceConfig: parser: str = "generic" fetcher: str = "requests" rate_limit_per_minute: int = 30 - respect_robots_txt: bool = True + respect_robots_txt: bool = False @dataclass(slots=True) diff --git a/crawler_platform/app/core/crawler/discovery.py b/crawler_platform/app/core/crawler/discovery.py index db10985..d3b9967 100644 --- a/crawler_platform/app/core/crawler/discovery.py +++ b/crawler_platform/app/core/crawler/discovery.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from urllib.parse import parse_qs, unquote, urljoin, urlparse +from urllib.parse import parse_qs, urlencode, unquote, urljoin, urlparse, urlunparse from bs4 import BeautifulSoup @@ -21,7 +21,7 @@ def discover_links(html: str, base_url: str, limit: int = 30) -> list[Discovered raw_href = anchor.get("href", "") if should_skip_raw_href(raw_href): continue - url = normalize_search_redirect(urljoin(base_url, raw_href)) + url = normalize_cafe24_product_url(normalize_search_redirect(urljoin(base_url, raw_href))) if not url or url in seen or not url.startswith(("http://", "https://")) or should_skip_url(url): continue seen.add(url) @@ -58,12 +58,18 @@ def should_skip_url(url: str) -> bool: skip_path_tokens = [ "/member/", "/order/", + "/myshop/", + "/event/list", "/exec/front/newcoupon/", + "/board/free/list", + "/board/faq/list", "/board/free/modify", "/board/free/reply", ] if any(token in path for token in skip_path_tokens): return True + if path.endswith("/product/search.html"): + return True if "facebook.com/" in path or "instagram.com/" in path: return True if "coupon_no=" in query: @@ -82,6 +88,33 @@ def normalize_search_redirect(url: str) -> str: return url +def normalize_cafe24_product_url(url: str) -> str: + parsed = urlparse(url) + path = unquote(parsed.path) + parts = [part for part in path.strip("/").split("/") if part] + query = parse_qs(parsed.query) + + if path.endswith("/product/detail.html") and query.get("product_no"): + return urlunparse( + parsed._replace( + query=urlencode({"product_no": query["product_no"][0]}), + fragment="", + ) + ) + + if parts and parts[0] == "product": + product_no = next((part for part in parts[1:] if part.isdigit()), None) + if product_no: + return urlunparse( + parsed._replace( + path="/product/detail.html", + query=urlencode({"product_no": product_no}), + fragment="", + ) + ) + return url + + def classify_url(url: str) -> str: host = urlparse(url).netloc.lower() if "smartstore.naver.com" in host or "brand.naver.com" in host: diff --git a/crawler_platform/app/core/crawler/fetchers.py b/crawler_platform/app/core/crawler/fetchers.py index 680d0af..31ffc54 100644 --- a/crawler_platform/app/core/crawler/fetchers.py +++ b/crawler_platform/app/core/crawler/fetchers.py @@ -42,6 +42,16 @@ class FetchResult: return self.rendered_html or self.raw_html or self.html +@dataclass(slots=True) +class RobotsDecision: + allowed: bool + checked: bool + status: str + reason: str + robots_url: str | None = None + user_agent: str = DEFAULT_USER_AGENT + + class RateLimiter: def __init__(self, per_minute: int = 30): self.delay = 60 / max(per_minute, 1) @@ -60,11 +70,26 @@ class RobotsPolicy: self._cache: dict[str, RobotFileParser] = {} def allowed(self, url: str, respect_robots_txt: bool = True) -> bool: + return self.check(url, respect_robots_txt).allowed + + def check(self, url: str, respect_robots_txt: bool = True) -> RobotsDecision: if not respect_robots_txt: - return True + return RobotsDecision( + allowed=True, + checked=False, + status="disabled", + reason="robots.txt check disabled by source/request config", + user_agent=self.user_agent, + ) parsed = urlparse(url) if parsed.scheme in {"", "file"}: - return True + return RobotsDecision( + allowed=True, + checked=False, + status="local", + reason="robots.txt is not applicable to local or scheme-less URLs", + user_agent=self.user_agent, + ) robots_url = f"{parsed.scheme}://{parsed.netloc}/robots.txt" parser = self._cache.get(robots_url) if parser is None: @@ -72,10 +97,29 @@ class RobotsPolicy: parser.set_url(robots_url) try: parser.read() - except Exception: - return False + except Exception as exc: + return RobotsDecision( + allowed=True, + checked=True, + status="unavailable", + reason=f"robots.txt unavailable ({exc.__class__.__name__}); allowing crawl", + robots_url=robots_url, + user_agent=self.user_agent, + ) self._cache[robots_url] = parser - return parser.can_fetch(self.user_agent, url) + allowed = parser.can_fetch(self.user_agent, url) + return RobotsDecision( + allowed=allowed, + checked=True, + status="allowed" if allowed else "blocked", + reason=( + "robots.txt allows crawling" + if allowed + else f"robots.txt blocks crawling for user-agent {self.user_agent}" + ), + robots_url=robots_url, + user_agent=self.user_agent, + ) class BaseFetcher: @@ -83,6 +127,23 @@ class BaseFetcher: raise NotImplementedError +class FallbackFetcher(BaseFetcher): + def __init__(self, primary: BaseFetcher, fallback: BaseFetcher, fallback_label: str = "fallback"): + self.primary = primary + self.fallback = fallback + self.fallback_label = fallback_label + + def fetch(self, url: str) -> FetchResult: + try: + return self.primary.fetch(url) + except Exception as exc: + result = self.fallback.fetch(url) + result.warnings.append( + f"primary fetcher failed ({exc.__class__.__name__}: {exc}); used {self.fallback_label}" + ) + return result + + class RequestsFetcher(BaseFetcher): def __init__( self, @@ -170,7 +231,11 @@ class PlaywrightFetcher(BaseFetcher): def make_fetcher(kind: str, rate_limit_per_minute: int = 30) -> BaseFetcher: if kind in {"playwright", "browser"}: - return PlaywrightFetcher() + return FallbackFetcher( + PlaywrightFetcher(), + RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute), + fallback_label="requests", + ) return RequestsFetcher(rate_limit_per_minute=rate_limit_per_minute) diff --git a/crawler_platform/app/core/crawler/page_classifier.py b/crawler_platform/app/core/crawler/page_classifier.py index 771857b..024cb7b 100644 --- a/crawler_platform/app/core/crawler/page_classifier.py +++ b/crawler_platform/app/core/crawler/page_classifier.py @@ -49,7 +49,7 @@ def classify_page( if any(token in path for token in ["shopinfo", "company", "about", "brand-story", "brand_story"]): return "BrandStoryPage" if any(token in path for token in ["product/detail", "/product/", "/products/", "/goods/", "/item/"]): - if _looks_like_category_path(path, combined): + if _looks_like_category_path(path, combined) and not _looks_like_product_detail_path(path): return "CategoryPage" return "ProductPage" if any(token in path for token in ["category", "/collections", "/collection", "/shop/", "/list"]): @@ -142,6 +142,14 @@ def _has_product_detail_signal(text: str) -> bool: return note_count >= 1 or commerce_count >= 2 +def _looks_like_product_detail_path(path: str) -> bool: + parts = [part for part in path.strip("/").split("/") if part] + if "product" not in parts: + return False + product_index = parts.index("product") + return any(part.isdigit() for part in parts[product_index + 1 :]) + + def _looks_like_category_path(path: str, text: str) -> bool: category_tokens = ["category", "cate_no", "display_group", "sort_method"] if any(token in path or token in text for token in category_tokens): diff --git a/crawler_platform/app/core/crawler/pipeline.py b/crawler_platform/app/core/crawler/pipeline.py index 63a5ecd..0c98373 100644 --- a/crawler_platform/app/core/crawler/pipeline.py +++ b/crawler_platform/app/core/crawler/pipeline.py @@ -22,6 +22,8 @@ class CrawlResult: clean_text_length: int = 0 raw_text_length: int = 0 warnings: list[str] | None = None + robots_status: str = "unchecked" + robots_reason: str | None = None class CrawlPipeline: @@ -39,8 +41,9 @@ class CrawlPipeline: def crawl_url(self, project_config: ProjectConfig, source_name: str, url: str) -> CrawlResult: source_config = project_config.source_by_name(source_name) - if not self.robots_policy.allowed(url, source_config.respect_robots_txt): - raise PermissionError(f"robots.txt does not allow crawling: {url}") + robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt) + if not robots_decision.allowed: + raise PermissionError(f"{robots_decision.reason}: {url}") fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute) fetch_result = fetcher.fetch(url) @@ -87,6 +90,8 @@ class CrawlPipeline: clean_text_length=len(parsed.text or ""), raw_text_length=len(parsed.raw_text or ""), warnings=warnings, + robots_status=robots_decision.status, + robots_reason=robots_decision.reason, ) context = ExtractionPageContext( @@ -117,4 +122,6 @@ class CrawlPipeline: clean_text_length=len(parsed.text or ""), raw_text_length=len(parsed.raw_text or ""), warnings=warnings, + robots_status=robots_decision.status, + robots_reason=robots_decision.reason, ) diff --git a/crawler_platform/app/core/crawler/site_crawler.py b/crawler_platform/app/core/crawler/site_crawler.py index fed11c3..f6016a9 100644 --- a/crawler_platform/app/core/crawler/site_crawler.py +++ b/crawler_platform/app/core/crawler/site_crawler.py @@ -7,7 +7,7 @@ from urllib.parse import urldefrag, urlparse from crawler_platform.app.config.loader import ProjectConfig from crawler_platform.app.core.crawler.discovery import discover_links -from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher +from crawler_platform.app.core.crawler.fetchers import RobotsDecision, RobotsPolicy, make_fetcher from crawler_platform.app.core.crawler.page_classifier import ( classify_page as classify_page_type, should_analyze_page as should_analyze_page_type, @@ -35,6 +35,8 @@ class SiteCrawlPageResult: clean_text_length: int = 0 removed_noise_zones_count: int = 0 warnings: list[str] = field(default_factory=list) + robots_status: str = "unchecked" + robots_reason: str | None = None error: str | None = None @@ -120,14 +122,23 @@ class SiteCrawler: progress_callback, ) continue - if not self.robots_policy.allowed(url, source_config.respect_robots_txt): - error = f"robots.txt does not allow crawling: {url}" + robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt) + if not robots_decision.allowed: + error = f"{robots_decision.reason}: {url}" self._finish_job(job, "blocked", error) result.skipped_count += 1 result.errors.append(error) self._record_page_result( result, - SiteCrawlPageResult(url=url, depth=depth, status="blocked", page_type="unknown", error=error), + SiteCrawlPageResult( + url=url, + depth=depth, + status="blocked", + page_type="unknown", + robots_status=robots_decision.status, + robots_reason=robots_decision.reason, + error=error, + ), len(queue), progress_callback, ) @@ -150,6 +161,7 @@ class SiteCrawler: analyze_page_types=analyze_page_types, result=result, job=job, + robots_decision=robots_decision, progress_callback=progress_callback, ) except Exception as exc: @@ -184,6 +196,7 @@ class SiteCrawler: analyze_page_types: set[str], result: SiteCrawlResult, job: models.CrawlJob, + robots_decision: RobotsDecision, progress_callback: Callable[[SiteCrawlResult, SiteCrawlPageResult], None] | None, ) -> None: fetch_result = fetcher.fetch(url) @@ -215,6 +228,8 @@ class SiteCrawler: page_type="unknown", page_id=page.id, crawl_status=fetch_result.crawl_status, + robots_status=robots_decision.status, + robots_reason=robots_decision.reason, warnings=fetch_result.warnings, error=error, ), @@ -277,6 +292,8 @@ class SiteCrawler: "clean_text_length": len(parsed.text or ""), "removed_noise_zones_count": int(parsed.metadata.get("removed_noise_zones_count") or 0), "warnings": warnings, + "robots_status": robots_decision.status, + "robots_reason": robots_decision.reason, } if parsed.extraction_status == "failed": diff --git a/crawler_platform/app/core/database/repository.py b/crawler_platform/app/core/database/repository.py index 6e3fa8c..00906b2 100644 --- a/crawler_platform/app/core/database/repository.py +++ b/crawler_platform/app/core/database/repository.py @@ -185,7 +185,7 @@ class KnowledgeRepository: else: claim_status = "active" - if claim_status not in {"active", "validated_claim"}: + if claim_status not in {"active", "validated_claim", "candidate_claim", "rule_candidate"}: self._log_extraction(project_id, page, bundle) return [] diff --git a/crawler_platform/app/core/extractor/ai_provider.py b/crawler_platform/app/core/extractor/ai_provider.py index d812397..ce38d2a 100644 --- a/crawler_platform/app/core/extractor/ai_provider.py +++ b/crawler_platform/app/core/extractor/ai_provider.py @@ -64,6 +64,9 @@ class LLMJsonExtractor(AIExtractor): try: raw = self.complete_json(page_text, project_config, compact=compact_mode, context=context) bundle = self._bundle_from_raw(raw, mode_name) + enriched = self._merge_rule_fallback_claims(bundle, page_text, project_config, mode_name) + if enriched.entities and enriched.claims: + return self.normalize_to_ontology(enriched, project_config.ontology) if bundle.entities and bundle.claims: return self.normalize_to_ontology(bundle, project_config.ontology) errors.append(f"{mode_name}: AI returned no usable entities or claims") @@ -103,12 +106,18 @@ class LLMJsonExtractor(AIExtractor): context: ExtractionPageContext | None = None, ) -> dict[str, Any]: if self.provider == "lm_studio": - char_limit = 1200 if compact else 2200 - max_tokens = 220 if compact else 420 + char_limit = 700 if compact else 1100 + max_tokens = 260 if compact else 360 else: char_limit = 2200 if compact else 4000 max_tokens = 400 if compact else 800 - prompt = build_extraction_prompt(page_text, project_config, char_limit=char_limit, context=context) + prompt = build_extraction_prompt( + page_text, + project_config, + char_limit=char_limit, + context=context, + compact=self.provider == "lm_studio", + ) if self.provider == "openai": return self._complete_openai_compatible( prompt, @@ -154,6 +163,45 @@ class LLMJsonExtractor(AIExtractor): claim.confidence_reason = f"{claim.confidence_reason}; AI fallback: {error}" if claim.confidence_reason else error return bundle + def _merge_rule_fallback_claims( + self, + bundle: ExtractionBundle, + page_text: str, + project_config: ProjectConfig, + mode: str, + ) -> ExtractionBundle: + rule_bundle = self._rule_bundle(page_text, project_config) + if not rule_bundle.claims: + return bundle + merged = ExtractionBundle( + entities=dedupe_entities([*bundle.entities, *rule_bundle.entities]), + claims=dedupe_claims([*bundle.claims, *rule_bundle.claims]), + extractor_name=f"{self.name}_with_rule_claims", + provider=self.provider, + raw_output={ + **bundle.raw_output, + "extraction_mode": mode, + "rule_claim_merge": True, + "rule_entity_count": len(rule_bundle.entities), + "rule_claim_count": len(rule_bundle.claims), + }, + ) + for claim in merged.claims: + claim.metadata.setdefault("rule_claim_merge", True) + claim.confidence_reason = ( + f"{claim.confidence_reason}; AI entity output enriched with rule claims" + if claim.confidence_reason + else "AI entity output enriched with rule claims" + ) + return merged + + def _rule_bundle(self, page_text: str, project_config: ProjectConfig) -> ExtractionBundle: + if project_config.domain == "perfume": + return PerfumeRuleBasedExtractor().extract(page_text, project_config) + from crawler_platform.app.core.extractor.rule_based import GenericRuleBasedExtractor + + return GenericRuleBasedExtractor().extract(page_text, project_config) + def _bundle_from_raw(self, raw: dict[str, Any], mode: str) -> ExtractionBundle: return ExtractionBundle( entities=parse_entities(raw.get("entities", [])), @@ -275,7 +323,10 @@ def build_extraction_prompt( project_config: ProjectConfig, char_limit: int = 4000, context: ExtractionPageContext | None = None, + compact: bool = False, ) -> str: + if compact: + return build_compact_extraction_prompt(page_text, project_config, char_limit=char_limit, context=context) if context is not None: prompt_input = json.dumps(context.to_payload(text_limit=char_limit), ensure_ascii=False, indent=2) else: @@ -338,6 +389,35 @@ Input payload: """.strip() +def build_compact_extraction_prompt( + page_text: str, + project_config: ProjectConfig, + char_limit: int = 1000, + context: ExtractionPageContext | None = None, +) -> str: + if context is not None: + text = prepare_page_text_for_prompt(context.clean_text, char_limit) + title = context.title or "" + page_type = context.page_type + url = context.final_url or context.url + else: + text = prepare_page_text_for_prompt(page_text, char_limit) + title = "" + page_type = "UnknownPage" + url = "" + predicates = ", ".join((project_config.ontology or {}).get("predicates", [])[:12]) + return ( + "Return minified JSON only: {\"entities\":[],\"claims\":[]}.\n" + f"Domain perfume. PageType={page_type}. URL={url}. Title={title}\n" + "Entity types: Perfume, Brand, Note, Accord, Mood, Season, Occasion, Price.\n" + f"Predicates: {predicates}.\n" + "Extract only explicit product facts. Max 6 entities, 8 claims. " + "Every claim needs short evidence_text from text. " + "Use null for missing object_name/object_type/object_value.\n" + f"Text:\n{text}" + ) + + def prepare_page_text_for_prompt(page_text: str, char_limit: int) -> str: noisy_terms = { "first page", @@ -567,6 +647,36 @@ def parse_entities(items: list[dict[str, Any]]) -> list[ExtractedEntity]: return entities +def dedupe_entities(entities: list[ExtractedEntity]) -> list[ExtractedEntity]: + seen: set[tuple[str, str]] = set() + result: list[ExtractedEntity] = [] + for entity in entities: + key = (entity.entity_type.strip().lower(), entity.name.strip().lower()) + if key in seen: + continue + seen.add(key) + result.append(entity) + return result + + +def dedupe_claims(claims: list[ExtractedClaim]) -> list[ExtractedClaim]: + seen: set[tuple[str, str, str, str]] = set() + result: list[ExtractedClaim] = [] + for claim in claims: + object_key = claim.object_name or json.dumps(claim.object_value, ensure_ascii=False, sort_keys=True, default=str) + key = ( + claim.subject_name.strip().lower(), + claim.subject_type.strip().lower(), + claim.predicate.strip(), + str(object_key).strip().lower(), + ) + if key in seen: + continue + seen.add(key) + result.append(claim) + return result + + def parse_claims(items: list[dict[str, Any]]) -> list[ExtractedClaim]: claims: list[ExtractedClaim] = [] for item in items: diff --git a/crawler_platform/app/core/research/graph_research_loop.py b/crawler_platform/app/core/research/graph_research_loop.py index 2792172..1abf197 100644 --- a/crawler_platform/app/core/research/graph_research_loop.py +++ b/crawler_platform/app/core/research/graph_research_loop.py @@ -200,8 +200,9 @@ class GraphResearchLoop: url = item.target if same_domain_only and seed_host and normalized_host(url) != seed_host: return {"status": "skipped", "reason": "outside same-domain research boundary"} - if not self.robots_policy.allowed(url, source_config.respect_robots_txt): - return {"status": "skipped", "reason": f"robots.txt blocked {url}"} + robots_decision = self.robots_policy.check(url, source_config.respect_robots_txt) + if not robots_decision.allowed: + return {"status": "skipped", "reason": f"{robots_decision.reason}: {url}"} fetch_result = fetcher.fetch(url) parser_result = parser.parse(fetch_result.analysis_html, fetch_result.final_url or url) @@ -229,6 +230,8 @@ class GraphResearchLoop: "final_url": fetch_result.final_url, "crawl_status": fetch_result.crawl_status, "page_type": page_type, + "robots_status": robots_decision.status, + "robots_reason": robots_decision.reason, "raw_text_length": len(parser_result.raw_text or ""), "clean_text_length": len(parser_result.text or ""), "main_content_preview": (parser_result.main_content or parser_result.text)[:800], diff --git a/crawler_platform/app/domains/perfume/extractor.py b/crawler_platform/app/domains/perfume/extractor.py index 2ddd472..6eba95b 100644 --- a/crawler_platform/app/domains/perfume/extractor.py +++ b/crawler_platform/app/domains/perfume/extractor.py @@ -118,7 +118,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): price = find_price(page_text) if price: attrs["price"] = {k: v for k, v in price.items() if k != "evidence"} - entities = [ExtractedEntity("Perfume", product_name, attrs, confidence=0.68)] + entities = [ExtractedEntity("Perfume", product_name, attrs, confidence=0.74)] if brand: entities.append(ExtractedEntity("Brand", brand, confidence=0.62)) for field, entity_type in [ @@ -187,7 +187,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): "hasPrice", object_value=card["price"], evidence_text=str(card.get("evidence") or card["name"]), - confidence=0.76, + confidence=0.84, confidence_reason="Korean product listing price pattern matched", ) ) @@ -236,7 +236,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): "hasPrice", object_value={k: v for k, v in price.items() if k != "evidence"}, evidence_text=price["evidence"], - confidence=0.7, + confidence=0.86, confidence_reason="price pattern matched", ) ) @@ -249,10 +249,19 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): def extract_product_name(page_text: str) -> str: - for line in page_text.splitlines()[:8]: + candidates: list[tuple[int, str]] = [] + for line in page_text.splitlines()[:12]: clean = line.strip() - if clean and not looks_like_navigation(clean) and not is_template_placeholder(clean): - return clean[:240] + if not clean or looks_like_navigation(clean) or is_template_placeholder(clean): + continue + if looks_like_metric_or_price(clean): + continue + candidates.append((product_line_score(clean), clean[:240])) + strong = [candidate for candidate in candidates if candidate[0] > 0] + if strong: + return max(strong, key=lambda item: item[0])[1] + if candidates: + return candidates[0][1] return first_non_empty_line(page_text) or "Unknown Perfume" @@ -265,14 +274,40 @@ def extract_brand(page_text: str, product_name: str) -> str | None: match = re.search(pattern, page_text, flags=re.IGNORECASE) if match: return cleanup_value(match.group("brand")) + inferred = infer_site_brand(page_text) + if inferred: + return inferred lines = [line.strip() for line in page_text.splitlines() if line.strip()] if len(lines) >= 2 and lines[1].lower() not in product_name.lower(): candidate = cleanup_value(lines[1]) - if len(candidate) <= 80 and not looks_like_navigation(candidate): + if len(candidate) <= 80 and not looks_like_navigation(candidate) and product_line_score(candidate) <= 0: return candidate return None +def product_line_score(value: str) -> int: + lower = value.lower() + score = 0 + if re.search(r"\d+\s*(?:ml|g|개입)", lower): + score += 4 + if any(keyword in value for keyword in ["향수", "디퓨저", "스프레이", "핸드크림", "미스트", "샤쉐", "퍼퓸"]): + score += 3 + if value.startswith("[") or any(keyword in value for keyword in ["기획", "추가할인", "모음"]): + score += 2 + if "912" in value: + score += 2 + if "시작" in value and not re.search(r"\d+\s*(?:ml|g|개입)", lower): + score -= 4 + return score + + +def looks_like_metric_or_price(value: str) -> bool: + clean = value.replace(",", "").strip() + if re.fullmatch(r"\d+(?:\.\d+)?", clean): + return True + return bool(re.fullmatch(r"\d+(?:\.\d+)?\s*(?:원|krw|usd)?", clean, flags=re.IGNORECASE)) + + def extract_product_cards(page_text: str) -> list[dict[str, object]]: lines = [line.strip() for line in page_text.splitlines() if line.strip()] cards: list[dict[str, object]] = [] diff --git a/crawler_platform/app/web/frontend/src/i18n.js b/crawler_platform/app/web/frontend/src/i18n.js index b351770..49094ba 100644 --- a/crawler_platform/app/web/frontend/src/i18n.js +++ b/crawler_platform/app/web/frontend/src/i18n.js @@ -26,6 +26,7 @@ const dict = { "sidebar.max_depth": "최대 깊이", "sidebar.max_pages": "최대 페이지", "sidebar.same_domain": "같은 도메인만", + "sidebar.respect_robots": "robots.txt 준수", "sidebar.crawl_site": "시드부터 사이트 크롤", "sidebar.stop_crawl": "현재 크롤 중지", @@ -262,6 +263,7 @@ const dict = { "sidebar.max_depth": "Max depth", "sidebar.max_pages": "Max pages", "sidebar.same_domain": "Same domain", + "sidebar.respect_robots": "Respect robots.txt", "sidebar.crawl_site": "Crawl site from seed", "sidebar.stop_crawl": "Stop current crawl", diff --git a/crawler_platform/app/web/frontend/src/sidebar.js b/crawler_platform/app/web/frontend/src/sidebar.js index 7736769..552c9dd 100644 --- a/crawler_platform/app/web/frontend/src/sidebar.js +++ b/crawler_platform/app/web/frontend/src/sidebar.js @@ -85,6 +85,10 @@ function sidebarHtml() { + @@ -102,6 +106,7 @@ export function requestBase() { extractor_provider: $("extractorProvider").value, extractor_model: $("extractorModel").value.trim() || null, extractor_base_url: $("extractorBaseUrl").value.trim() || null, + check_robots_txt: $("respectRobotsTxt")?.checked ?? false, }; } @@ -331,6 +336,7 @@ async function discover() { source_name: $("sourceSelect").value, url: $("crawlUrl").value.trim(), limit: 30, + check_robots_txt: $("respectRobotsTxt")?.checked ?? false, }), }); if (!result.ok) { diff --git a/crawler_platform/app/web/static/assets/index-BsoXaTIL.js b/crawler_platform/app/web/static/assets/index-BsoXaTIL.js new file mode 100644 index 0000000..fbd7d3c --- /dev/null +++ b/crawler_platform/app/web/static/assets/index-BsoXaTIL.js @@ -0,0 +1,931 @@ +(function(){const e=document.createElement("link").relList;if(e&&e.supports&&e.supports("modulepreload"))return;for(const n of document.querySelectorAll('link[rel="modulepreload"]'))a(n);new MutationObserver(n=>{for(const i of n)if(i.type==="childList")for(const s of i.addedNodes)s.tagName==="LINK"&&s.rel==="modulepreload"&&a(s)}).observe(document,{childList:!0,subtree:!0});function r(n){const i={};return n.integrity&&(i.integrity=n.integrity),n.referrerPolicy&&(i.referrerPolicy=n.referrerPolicy),n.crossOrigin==="use-credentials"?i.credentials="include":n.crossOrigin==="anonymous"?i.credentials="omit":i.credentials="same-origin",i}function a(n){if(n.ep)return;n.ep=!0;const i=r(n);fetch(n.href,i)}})();const od="ocp.lang",ld=["ko","en"],ud="ko",Sl={ko:{"app.title":"RPG Onta","app.subtitle":"프로젝트 크롤, 온톨로지 매핑, 클레임 리뷰, 추천 태그","header.refresh":"새로고침","header.lang.ko":"한","header.lang.en":"EN","sidebar.projects":"프로젝트","sidebar.config_path":"설정 파일 경로","sidebar.create_project":"프로젝트 생성","sidebar.reset_project":"현재 프로젝트 데이터 리셋","sidebar.crawl":"크롤","sidebar.source":"소스","sidebar.url_or_seed":"URL / 시드 URL","sidebar.analyzer":"분석기","sidebar.model":"모델","sidebar.base_url":"Base URL","sidebar.test_analyzer":"분석기 테스트","sidebar.discover":"링크 탐색","sidebar.crawl_url":"URL 크롤","sidebar.max_depth":"최대 깊이","sidebar.max_pages":"최대 페이지","sidebar.same_domain":"같은 도메인만","sidebar.respect_robots":"robots.txt 준수","sidebar.crawl_site":"시드부터 사이트 크롤","sidebar.stop_crawl":"현재 크롤 중지","tabs.overview":"개요","tabs.graph":"그래프","tabs.ontology":"온톨로지","tabs.entities":"엔티티","tabs.claims":"클레임","tabs.debug":"디버그","tabs.research":"리서치","tabs.tags":"태그","tabs.recommend":"추천","graph.search_placeholder":"엔티티명 검색...","graph.entity_type_all":"전체 타입","graph.predicate_all":"전체 관계","graph.min_confidence":"최소 신뢰도","graph.layout":"레이아웃","graph.layout.cose":"Force","graph.layout.circle":"원형","graph.layout.concentric":"동심원","graph.layout.grid":"그리드","graph.layout.breadthfirst":"트리","graph.reload":"그래프 새로고침","graph.fit":"맞춤","graph.empty":"아직 데이터가 없습니다. 크롤이 클레임을 만들면 그래프에 노드가 나타납니다.","graph.stats.nodes":"노드","graph.stats.edges":"엣지","graph.stats.shown":"표시","overview.project":"프로젝트","overview.domain":"도메인","overview.sources":"소스","overview.entities":"엔티티","overview.sources_heading":"소스","pipeline.heading":"파이프라인","pipeline.refresh":"새로고침","pipeline.stage.crawled":"크롤됨","pipeline.stage.extracted":"추출됨","pipeline.stage.claims":"클레임","pipeline.stage.validated":"승인됨","pipeline.stage.graph":"그래프 트리플","pipeline.retained":"유지율","pipeline.entity_types_heading":"엔티티 타입 분포","pipeline.recent_pages":"최근 페이지","pipeline.recent_claims":"최근 클레임","pipeline.empty.pages":"아직 크롤된 페이지가 없습니다.","pipeline.empty.claims":"아직 생성된 클레임이 없습니다.","search.placeholder":"엔티티 · 클레임 · 페이지 검색...","search.aria_label":"전역 검색","search.group.entities":"엔티티","search.group.claims":"클레임","search.group.pages":"페이지","search.group.predicates":"술어","search.empty":"결과 없음","search.hint":"최소 1글자 이상 입력하세요","table.name":"이름","table.type":"타입","table.trust":"신뢰도","table.robots":"robots","table.rate":"속도제한","table.domain":"도메인","table.status":"상태","table.confidence":"신뢰도","table.relation":"관계","table.subject":"주체","table.object":"객체","table.support":"근거 수","table.predicate":"술어","table.tag":"태그","table.brand":"브랜드","table.product":"상품","table.count":"수","table.no_data":"데이터 없음.","table.id":"ID","table.metadata":"메타데이터","table.reason":"사유","table.gap":"갭","table.target":"대상","table.priority":"우선순위","table.description":"설명","table.score":"점수","table.reasons":"근거","ontology.refresh_registry":"레지스트리 새로고침","ontology.configured_entity_types":"설정된 엔티티 타입","ontology.configured_predicates":"설정된 술어","ontology.registry_entity_types":"레지스트리 엔티티 타입","ontology.registry_relation_types":"레지스트리 관계 타입","ontology.triples":"온톨로지 트리플","ontology.proposals":"스키마 제안","ontology.knowledge_gaps":"지식 갭","entities.all_types":"전체 타입","entities.load":"불러오기","entities.merge_source_placeholder":"병합할 엔티티 ID","entities.merge_target_placeholder":"유지할 엔티티 ID","entities.merge":"병합","claims.show_candidates":"후보 포함","claims.refresh":"클레임 새로고침","claims.save":"저장","claims.confidence_label":"신뢰도","claims.reason_label":"사유","workbench.status_filter":"상태","workbench.status.all":"전체","workbench.status.active":"활성","workbench.status.validated_claim":"승인됨","workbench.status.rejected":"거절됨","workbench.search_placeholder":"주체/술어/객체 검색...","workbench.predicate_filter":"관계","workbench.predicate_all":"전체 관계","workbench.empty":"표시할 클레임이 없습니다.","workbench.select_hint":"왼쪽 목록에서 클레임을 선택하세요.","workbench.bulk_count":"{count}개 선택됨","workbench.action.accept":"승인","workbench.action.reject":"거절","workbench.action.unreview":"리뷰 취소","workbench.action.save_confidence":"신뢰도 저장","workbench.action.accept_similar":"동일 술어+객체 일괄 승인","workbench.shortcuts":"단축키","workbench.shortcut.nav":"J/K · 이동","workbench.shortcut.accept":"A · 승인","workbench.shortcut.reject":"R · 거절","workbench.shortcut.bulk_accept":"Shift+A · 선택 일괄 승인","workbench.shortcut.bulk_reject":"Shift+R · 선택 일괄 거절","workbench.shortcut.toggle":"Space · 선택 토글","workbench.shortcut.search":"/ · 검색","workbench.shortcut.escape":"Esc · 선택 해제","workbench.detail.subject":"주체","workbench.detail.predicate":"술어","workbench.detail.object":"객체","workbench.detail.status":"상태","workbench.detail.confidence":"신뢰도","workbench.detail.evidence":"근거 텍스트","workbench.detail.source":"출처","workbench.detail.page":"페이지","workbench.detail.breakdown":"신뢰도 분해","workbench.detail.reason_placeholder":"사유 (선택)","workbench.toast.reviewed":"리뷰 적용됨","workbench.toast.bulk_done":"{count}개 적용됨","debug.refresh_logs":"추출 로그 새로고침","research.heading":"그래프 리서치","research.goal":"목표","research.seed_entity":"시드 엔티티 ID","research.max_steps":"최대 스텝","research.min_relevance":"최소 관련도","research.run":"그래프 리서치 실행","research.load_sessions":"세션 불러오기","research.graph_query":"그래프 쿼리","research.filter_placeholder":"브랜드 또는 태그 필터","research.run_query":"쿼리 실행","research.query.trend_summary":"트렌드 요약","research.query.brand_products":"브랜드 상품","research.query.products_by_tag":"태그별 상품","research.query.relation_summary":"관계 요약","research.query.entity_type_summary":"엔티티 타입 요약","tags.load":"태그 불러오기","recommend.preferred_notes":"선호 노트","recommend.avoided_notes":"기피 노트","recommend.preferred_moods":"선호 무드","recommend.season":"계절","recommend.occasion":"상황","recommend.test":"추천 테스트","inspector.title":"인스펙터","inspector.hint":"엔티티/클레임/페이지를 선택하면 상세가 표시됩니다.","inspector.project_info":"프로젝트 정보","inspector.no_selection":"선택 없음","inspector.latest_activity":"최근 활동","inspector.latest_activity.empty":"최근 크롤/추출 활동이 없습니다.","extractor.rule_based":"규칙 기반","extractor.openai":"OpenAI API","extractor.ollama":"Ollama","extractor.lm_studio":"LM Studio","toast.project_created":"프로젝트 생성됨","toast.reset_confirm":"현재 프로젝트의 크롤 데이터(페이지/엔티티/클레임)를 리셋하시겠습니까?","toast.reset_done":"리셋 완료","toast.url_copied":"URL이 입력란에 복사됨","toast.url_crawl_completed":"URL 크롤 완료","toast.crawl_failed":"크롤 실패","toast.site_crawl_started":"사이트 크롤 시작","toast.site_crawl_failed":"사이트 크롤 실패","toast.site_crawl_completed":"사이트 크롤 완료","toast.stop_requested":"중지 요청됨","toast.stop_failed":"중지 실패","toast.analyzer_connected":"분석기 연결됨","toast.analyzer_failed":"분석기 실패","toast.discovery_failed":"탐색 실패","toast.claim_updated":"클레임 업데이트됨","toast.entity_merged":"엔티티 병합됨","toast.merge_check":"엔티티 ID를 확인하세요","toast.research_completed":"리서치 완료","toast.research_failed":"리서치 실패","toast.config_path_required":"설정 파일 경로가 필요합니다","status.starting_site_crawl":"시드부터 사이트 크롤 시작 중...","status.discovering":"링크 탐색 중...","status.testing_analyzer":"분석기 테스트 중...","status.crawling_url":"URL 1건 크롤 중...","status.stopping_crawl":"사이트 크롤 중지 중","status.research_running":"그래프 리서치 실행 중...","status.no_crawl":"준비됨"},en:{"app.title":"Ontology Crawler","app.subtitle":"Project crawler, ontology mapping, claim review, recommendation tags","header.refresh":"Refresh","header.lang.ko":"한","header.lang.en":"EN","sidebar.projects":"Projects","sidebar.config_path":"Config path","sidebar.create_project":"Create project","sidebar.reset_project":"Reset current project data","sidebar.crawl":"Crawl","sidebar.source":"Source","sidebar.url_or_seed":"URL / Seed URL","sidebar.analyzer":"Analyzer","sidebar.model":"Model","sidebar.base_url":"Base URL","sidebar.test_analyzer":"Test analyzer","sidebar.discover":"Discover links","sidebar.crawl_url":"Crawl URL","sidebar.max_depth":"Max depth","sidebar.max_pages":"Max pages","sidebar.same_domain":"Same domain","sidebar.respect_robots":"Respect robots.txt","sidebar.crawl_site":"Crawl site from seed","sidebar.stop_crawl":"Stop current crawl","tabs.overview":"Overview","tabs.graph":"Graph","tabs.ontology":"Ontology","tabs.entities":"Entities","tabs.claims":"Claims","tabs.debug":"Debug","tabs.research":"Research","tabs.tags":"Tags","tabs.recommend":"Recommend","graph.search_placeholder":"Search entity name...","graph.entity_type_all":"All types","graph.predicate_all":"All relations","graph.min_confidence":"Min confidence","graph.layout":"Layout","graph.layout.cose":"Force","graph.layout.circle":"Circle","graph.layout.concentric":"Concentric","graph.layout.grid":"Grid","graph.layout.breadthfirst":"Tree","graph.reload":"Reload graph","graph.fit":"Fit","graph.empty":"No data yet. As crawling produces claims, nodes will appear on the graph.","graph.stats.nodes":"nodes","graph.stats.edges":"edges","graph.stats.shown":"shown","overview.project":"Project","overview.domain":"Domain","overview.sources":"Sources","overview.entities":"Entities","overview.sources_heading":"Sources","pipeline.heading":"Pipeline","pipeline.refresh":"Refresh","pipeline.stage.crawled":"Crawled","pipeline.stage.extracted":"Extracted","pipeline.stage.claims":"Claims","pipeline.stage.validated":"Validated","pipeline.stage.graph":"Graph triples","pipeline.retained":"retained","pipeline.entity_types_heading":"Entity type breakdown","pipeline.recent_pages":"Recent pages","pipeline.recent_claims":"Recent claims","pipeline.empty.pages":"No pages crawled yet.","pipeline.empty.claims":"No claims generated yet.","search.placeholder":"Search entities, claims, pages...","search.aria_label":"Global search","search.group.entities":"Entities","search.group.claims":"Claims","search.group.pages":"Pages","search.group.predicates":"Predicates","search.empty":"No results","search.hint":"Type at least 1 character","table.name":"Name","table.type":"Type","table.trust":"Trust","table.robots":"Robots","table.rate":"Rate","table.domain":"Domain","table.status":"Status","table.confidence":"Confidence","table.relation":"Relation","table.subject":"Subject","table.object":"Object","table.support":"Support","table.predicate":"Predicate","table.tag":"Tag","table.brand":"Brand","table.product":"Product","table.count":"Count","table.no_data":"No data.","table.id":"ID","table.metadata":"Metadata","table.reason":"Reason","table.gap":"Gap","table.target":"Target","table.priority":"Priority","table.description":"Description","table.score":"Score","table.reasons":"Reasons","ontology.refresh_registry":"Refresh registry","ontology.configured_entity_types":"Configured Entity Types","ontology.configured_predicates":"Configured Predicates","ontology.registry_entity_types":"Registry Entity Types","ontology.registry_relation_types":"Registry Relation Types","ontology.triples":"Ontology Triples","ontology.proposals":"Schema Proposals","ontology.knowledge_gaps":"Knowledge Gaps","entities.all_types":"All types","entities.load":"Load","entities.merge_source_placeholder":"Entity ID to merge","entities.merge_target_placeholder":"Entity ID to keep","entities.merge":"Merge","claims.show_candidates":"Show candidates","claims.refresh":"Refresh claims","claims.save":"Save","claims.confidence_label":"confidence","claims.reason_label":"reason","workbench.status_filter":"Status","workbench.status.all":"All","workbench.status.active":"Active","workbench.status.validated_claim":"Validated","workbench.status.rejected":"Rejected","workbench.search_placeholder":"Search subject/predicate/object...","workbench.predicate_filter":"Predicate","workbench.predicate_all":"All predicates","workbench.empty":"No claims to display.","workbench.select_hint":"Select a claim from the list on the left.","workbench.bulk_count":"{count} selected","workbench.action.accept":"Accept","workbench.action.reject":"Reject","workbench.action.unreview":"Unreview","workbench.action.save_confidence":"Save confidence","workbench.action.accept_similar":"Accept all with same predicate+object","workbench.shortcuts":"Shortcuts","workbench.shortcut.nav":"J/K · navigate","workbench.shortcut.accept":"A · accept","workbench.shortcut.reject":"R · reject","workbench.shortcut.bulk_accept":"Shift+A · bulk accept","workbench.shortcut.bulk_reject":"Shift+R · bulk reject","workbench.shortcut.toggle":"Space · toggle select","workbench.shortcut.search":"/ · search","workbench.shortcut.escape":"Esc · clear selection","workbench.detail.subject":"Subject","workbench.detail.predicate":"Predicate","workbench.detail.object":"Object","workbench.detail.status":"Status","workbench.detail.confidence":"Confidence","workbench.detail.evidence":"Evidence text","workbench.detail.source":"Source","workbench.detail.page":"Page","workbench.detail.breakdown":"Confidence breakdown","workbench.detail.reason_placeholder":"reason (optional)","workbench.toast.reviewed":"Review applied","workbench.toast.bulk_done":"{count} updated","debug.refresh_logs":"Refresh extraction logs","research.heading":"Semantic Exploration","research.goal":"Goal","research.seed_entity":"Seed entity ID","research.max_steps":"Max steps","research.min_relevance":"Min relevance","research.run":"Run graph research","research.load_sessions":"Load sessions","research.graph_query":"Graph Query","research.filter_placeholder":"brand or tag filter","research.run_query":"Run query","research.query.trend_summary":"Trend summary","research.query.brand_products":"Brand products","research.query.products_by_tag":"Products by tag","research.query.relation_summary":"Relation summary","research.query.entity_type_summary":"Entity type summary","tags.load":"Load tags","recommend.preferred_notes":"Preferred notes","recommend.avoided_notes":"Avoided notes","recommend.preferred_moods":"Preferred moods","recommend.season":"Season","recommend.occasion":"Occasion","recommend.test":"Test recommendation","inspector.title":"Inspector","inspector.hint":"Select an entity/claim/page to see details.","inspector.project_info":"Project info","inspector.no_selection":"Nothing selected","inspector.latest_activity":"Latest activity","inspector.latest_activity.empty":"No recent crawl or extraction activity.","extractor.rule_based":"Rule-based","extractor.openai":"OpenAI API","extractor.ollama":"Ollama","extractor.lm_studio":"LM Studio","toast.project_created":"Project created","toast.reset_confirm":"Reset current project crawl data (pages/entities/claims)?","toast.reset_done":"Reset completed","toast.url_copied":"URL copied to input","toast.url_crawl_completed":"URL crawl completed","toast.crawl_failed":"Crawl failed","toast.site_crawl_started":"Site crawl started","toast.site_crawl_failed":"Site crawl failed","toast.site_crawl_completed":"Site crawl completed","toast.stop_requested":"Stop requested","toast.stop_failed":"Stop failed","toast.analyzer_connected":"Analyzer connected","toast.analyzer_failed":"Analyzer failed","toast.discovery_failed":"Discovery failed","toast.claim_updated":"Claim updated","toast.entity_merged":"Entity merged","toast.merge_check":"Check entity IDs","toast.research_completed":"Research completed","toast.research_failed":"Research failed","toast.config_path_required":"Config path is required","status.starting_site_crawl":"Starting site crawl from seed...","status.discovering":"Discovering links...","status.testing_analyzer":"Testing analyzer...","status.crawling_url":"Crawling one URL...","status.stopping_crawl":"Stopping site crawl","status.research_running":"Running graph research...","status.no_crawl":"Ready"}};let Hn=Bv();const co=new Set;function Bv(){try{const e=localStorage.getItem(od);if(e&&ld.includes(e))return e}catch{}return(navigator.language||"").toLowerCase().startsWith("en")?"en":ud}function kl(){return Hn}function Dv(t){if(!(!ld.includes(t)||t===Hn)){Hn=t;try{localStorage.setItem(od,t)}catch{}document.documentElement.setAttribute("lang",t),xt(document),co.forEach(e=>{try{e(t)}catch(r){console.warn("i18n listener failed",r)}})}}function fr(t){return co.add(t),()=>co.delete(t)}function V(t,e){const r=Sl[Hn]||Sl[ud];return t in r?r[t]:e??t}function xt(t=document){t.querySelectorAll("[data-i18n]").forEach(e=>{const r=e.getAttribute("data-i18n");r&&(e.textContent=V(r))}),t.querySelectorAll("[data-i18n-placeholder]").forEach(e=>{const r=e.getAttribute("data-i18n-placeholder");r&&e.setAttribute("placeholder",V(r))}),t.querySelectorAll("[data-i18n-title]").forEach(e=>{const r=e.getAttribute("data-i18n-title");r&&e.setAttribute("title",V(r))}),t.querySelectorAll("[data-i18n-aria-label]").forEach(e=>{const r=e.getAttribute("data-i18n-aria-label");r&&e.setAttribute("aria-label",V(r))})}function Rv(t){t.innerHTML=` +
+
+

+

+
+
+
+ + + +
+
+ + +
+ +
+
+ +
+ +
+ +
+ +
+ `,document.documentElement.setAttribute("lang",kl()),xt(t);const e=t.querySelectorAll(".lang-btn"),r=()=>{const a=kl();e.forEach(n=>{n.classList.toggle("active",n.dataset.lang===a)})};return e.forEach(a=>{a.addEventListener("click",()=>Dv(a.dataset.lang))}),r(),fr(r),{sidebarHost:document.getElementById("sidebarHost"),workspaceHost:document.getElementById("workspaceHost"),inspectorHost:document.getElementById("inspectorHost")}}async function Fe(t,e={}){const r=await fetch(t,{headers:{"Content-Type":"application/json"},...e});if(!r.ok){let a=`${r.status} ${r.statusText}`;try{const n=await r.json();a=n.detail||n.error||a}catch{}throw new Error(a)}return r.json()}const ue={projects:[],selectedProject:null,projectDetail:null,ontology:null,siteCrawlPoll:null,siteCrawlPageCount:0,siteCrawlPollErrorCount:0,activeSiteCrawlJobId:null,latestActivity:null,selection:null},F=t=>document.getElementById(t);function Di(t){return String(t||"").split(",").map(e=>e.trim()).filter(Boolean)}function z(t){return String(t??"").replaceAll("&","&").replaceAll("<","<").replaceAll(">",">").replaceAll('"',""").replaceAll("'","'")}function Pl(t){return`${z(t)}`}function Mt(t,e){return e.length?` + + ${t.map(r=>``).join("")} + ${e.map(r=>`${r.map(a=>``).join("")}`).join("")} +
${z(r)}
${String(a??"")}
+ `:`
${z(V("table.no_data"))}
`}let Ri=null;function Ae(t){const e=document.getElementById("toast");e&&(e.textContent=t,e.classList.add("show"),Ri&&window.clearTimeout(Ri),Ri=window.setTimeout(()=>e.classList.remove("show"),2400))}function Av(t,{onProjectChanged:e,onCrawlUpdate:r,refreshAll:a}){t.innerHTML=Lv(),xt(t),F("createProjectBtn").addEventListener("click",()=>_v(e)),F("initProjectBtn").addEventListener("click",()=>Iv(e)),F("discoverBtn").addEventListener("click",Fv),F("crawlBtn").addEventListener("click",()=>Ov(a)),F("siteCrawlBtn").addEventListener("click",()=>Nv(r,a)),F("stopSiteCrawlBtn").addEventListener("click",zv),F("extractorProvider").addEventListener("change",Bl),F("testExtractorBtn").addEventListener("click",()=>dd({announce:!0})),Bl(),Vv(),fr(()=>{xt(t),Go(),si(cd)})}function Lv(){return` +
+

+
+ + +
+ +
+
+ +
+

+ + + +
+ + + +
+
+ + +
+
+ + + + +
+ + +
+ +
+ `}function Ho(){return{config_path:F("configPath").value.trim(),source_name:F("sourceSelect").value,url:F("crawlUrl").value.trim(),extractor_provider:F("extractorProvider").value,extractor_model:F("extractorModel").value.trim()||null,extractor_base_url:F("extractorBaseUrl").value.trim()||null,check_robots_txt:F("respectRobotsTxt")?.checked??!1}}function Go(){const t=F("projectList");t&&(t.innerHTML="",ue.projects.forEach(e=>{const r=document.createElement("button");r.className=`project-item ${ue.selectedProject===e.name?"active":""}`,r.innerHTML=`${z(e.name)}${z(e.domain)}`,r.addEventListener("click",()=>{const a=new CustomEvent("project:select",{detail:e.name});window.dispatchEvent(a)}),t.appendChild(r)}))}function Mv(t){const e=F("sourceSelect");e&&(e.innerHTML="",(t?.sources??[]).forEach(r=>{const a=document.createElement("option");a.value=r.name,a.textContent=`${r.name} (${r.type})`,e.appendChild(a)}))}async function _v(t){const e=F("configPath").value.trim();if(e)try{const r=await Fe("/projects",{method:"POST",body:JSON.stringify({config_path:e})});Ae(`${V("toast.project_created")}: ${r.name}`),ue.selectedProject=r.name,await t?.()}catch(r){Ae(r.message)}}async function Iv(t){const e=F("configPath").value.trim();if(!e){Ae(V("toast.config_path_required"));return}if(window.confirm(V("toast.reset_confirm")))try{const r=await Fe("/projects/reset",{method:"POST",body:JSON.stringify({config_path:e,project_name:ue.selectedProject})});if(ue.selectedProject=r.name,await t?.(),r.reset){const a=r.deleted?.claims??0,n=r.deleted?.entities??0,i=r.deleted?.pages??0;F("crawlResult").textContent=`${V("toast.reset_done")}: pages ${i}, entities ${n}, claims ${a}`,Ae(`${V("toast.reset_done")}: ${r.name}`)}else F("crawlResult").textContent=`${V("toast.project_created")}: ${r.name}`,Ae(`${V("toast.project_created")}: ${r.name}`)}catch(r){Ae(r.message)}}async function Ov(t){if(ue.selectedProject){F("crawlResult").textContent=V("status.crawling_url");try{const e=await Fe("/crawl",{method:"POST",body:JSON.stringify(Ho())});F("crawlResult").textContent=`URL done: page ${e.page_id}, ${e.crawl_status}/${e.extraction_status}, ${e.page_type}, raw ${e.raw_text_length}, clean ${e.clean_text_length}, claims ${e.claim_count}, entities ${e.entity_count}`,ue.latestActivity={kind:"url_crawl",result:e},Ae(V("toast.url_crawl_completed")),await t?.()}catch(e){F("crawlResult").textContent=`${V("toast.crawl_failed")}: ${e.message}`,Ae(V("toast.crawl_failed"))}}}let cd=null;async function Nv(t,e){if(ue.selectedProject){fo(),ue.siteCrawlPageCount=0,ue.siteCrawlPollErrorCount=0,F("crawlResult").textContent=V("status.starting_site_crawl"),F("discoveredLinks").innerHTML="";try{const r=await Fe("/crawl-site",{method:"POST",body:JSON.stringify({...Ho(),max_depth:Number(F("siteMaxDepth").value||0),max_pages:Number(F("siteMaxPages").value||1),same_domain_only:F("sameDomainOnly").checked,analyze_page_types:["ProductPage","BrandStoryPage","ReviewPage"]})});F("crawlResult").textContent=`Site crawl queued: job ${r.job_id}`,ue.activeSiteCrawlJobId=r.job_id,F("stopSiteCrawlBtn").disabled=!1,si(r),vo(r.job_id,t,e),Ae(V("toast.site_crawl_started"))}catch(r){F("crawlResult").textContent=`${V("toast.site_crawl_failed")}: ${r.message}`,Ae(V("toast.site_crawl_failed"))}}}async function zv(){const t=ue.activeSiteCrawlJobId;if(t){F("stopSiteCrawlBtn").disabled=!0,F("crawlResult").textContent=`${V("status.stopping_crawl")}: job ${t}`;try{const e=await Fe(`/crawl-site/jobs/${t}/cancel`,{method:"POST"});si(e),Ae(V("toast.stop_requested"))}catch(e){F("stopSiteCrawlBtn").disabled=!1,F("crawlResult").textContent=`${V("toast.stop_failed")}: ${e.message}`,Ae(V("toast.stop_failed"))}}}function fo(){ue.siteCrawlPoll&&(window.clearTimeout(ue.siteCrawlPoll),ue.siteCrawlPoll=null)}async function vo(t,e,r){let a;try{a=await Fe(`/crawl-site/jobs/${t}`)}catch(s){const o=ue.siteCrawlPollErrorCount=(ue.siteCrawlPollErrorCount??0)+1;if(F("crawlResult").textContent=`Site crawl status check failed (retry ${o}): ${s.message}`,o>=5){fo(),Ae(V("toast.site_crawl_failed"));return}ue.siteCrawlPoll=window.setTimeout(()=>vo(t,e,r),3e3);return}ue.siteCrawlPollErrorCount=0;const i=(a.progress??{}).pages?.length??0;if(si(a),e?.(a),i!==ue.siteCrawlPageCount){ue.siteCrawlPageCount=i;try{await r?.()}catch(s){console.warn("refreshAll during poll failed",s)}}if(["completed","failed","canceled"].includes(a.status)){fo(),ue.activeSiteCrawlJobId=null,F("stopSiteCrawlBtn").disabled=!0,Ae(a.status==="completed"?V("toast.site_crawl_completed"):`${V("toast.site_crawl_failed")}: ${a.status}`);return}ue.siteCrawlPoll=window.setTimeout(()=>vo(t,e,r),1500)}function si(t){if(cd=t,!t)return;const e=t.progress??{},r=e.pages??[];F("crawlResult").textContent=`Site ${t.status}: visited ${e.visited_count??0}, analyzed ${e.analyzed_count??0}, skipped ${e.skipped_count??0}, queued ${e.queued_count??0}`,F("discoveredLinks").innerHTML=r.map($v).join(""),document.querySelectorAll("[data-discovered-url]").forEach(a=>{a.addEventListener("click",()=>{F("crawlUrl").value=a.dataset.discoveredUrl,Ae(V("toast.url_copied"))})}),ue.latestActivity={kind:"site_crawl",job:t}}function $v(t){const e=`raw ${t.raw_text_length??0}, clean ${t.clean_text_length??0}, removed ${t.removed_noise_zones_count??0}`,r=(t.warnings??[]).length?`, warnings: ${(t.warnings??[]).join("; ")}`:"";return` + + `}async function Fv(){F("crawlResult").textContent=V("status.discovering"),F("discoveredLinks").innerHTML="";try{const t=await Fe("/discover",{method:"POST",body:JSON.stringify({config_path:F("configPath").value.trim(),source_name:F("sourceSelect").value,url:F("crawlUrl").value.trim(),limit:30,check_robots_txt:F("respectRobotsTxt")?.checked??!1})});if(!t.ok){F("crawlResult").textContent=t.error??V("toast.discovery_failed");return}F("crawlResult").textContent=`Discovered ${t.links.length} links`,F("discoveredLinks").innerHTML=t.links.map(qv).join(""),document.querySelectorAll("[data-discovered-url]").forEach(e=>{e.addEventListener("click",()=>{F("crawlUrl").value=e.dataset.discoveredUrl,Ae(V("toast.url_copied"))})})}catch(t){F("crawlResult").textContent=`${V("toast.discovery_failed")}: ${t.message}`,Ae(V("toast.discovery_failed"))}}function qv(t){return` + + `}function Bl(){const t=F("extractorProvider").value;F("extractorOptions").classList.toggle("active",t!=="rule_based"),t==="ollama"&&!F("extractorBaseUrl").value.trim()?F("extractorBaseUrl").placeholder="http://localhost:11434/api/chat":t==="lm_studio"&&!F("extractorBaseUrl").value.trim()?(F("extractorBaseUrl").value="http://localhost:1234/v1",F("extractorBaseUrl").placeholder="http://localhost:1234/v1"):F("extractorBaseUrl").placeholder="optional provider endpoint"}async function dd({announce:t=!0}={}){const e=F("extractorProvider").value,r=F("extractorBaseUrl").value.trim();t&&(F("crawlResult").textContent=V("status.testing_analyzer"));try{const a=await Fe("/extractors/models",{method:"POST",body:JSON.stringify({provider:e,base_url:r||null})});if(!a.ok){t&&(F("crawlResult").textContent=`${V("toast.analyzer_failed")}: ${a.error}`,Ae(V("toast.analyzer_failed")));return}const n=a.models??[];n.length&&!F("extractorModel").value.trim()&&(F("extractorModel").value=n[0].id),t&&(F("crawlResult").textContent=n.length?`${V("toast.analyzer_connected")}. Models: ${n.map(i=>i.id).join(", ")}`:`${V("toast.analyzer_connected")}. No models returned.`,Ae(V("toast.analyzer_connected")))}catch(a){t&&Ae(a.message)}}async function Vv(){if(F("extractorProvider").value==="lm_studio")try{await dd({announce:!1})}catch{}}let Rn=null;function Hv(t){Rn=t,An(),fr(An),window.addEventListener("inspector:update",An)}function ur(){An()}function An(){Rn&&(Rn.innerHTML=Gv(),xt(Rn))}function Gv(){const t=ue.selection;return` +
+

+

+
+ ${t?Wv(t):Uv()} + `}function Uv(){const t=ue.projectDetail,e=ue.latestActivity;return` +
+

+ ${t?` +
+
${z(V("overview.project"))}
${z(t.name)}
+
${z(V("overview.domain"))}
${z(t.domain)}
+
${z(V("overview.sources"))}
${(t.sources??[]).length}
+
+ `:'

'} +
+
+

+ ${e?Kv(e):'

'} +
+ `}function Kv(t){if(t.kind==="site_crawl"){const e=t.job??{},r=e.progress??{};return` +
+
job
${z(e.job_id??"-")}
+
status
${z(e.status??"-")}
+
visited
${r.visited_count??0}
+
analyzed
${r.analyzed_count??0}
+
skipped
${r.skipped_count??0}
+
queued
${r.queued_count??0}
+
+ `}if(t.kind==="url_crawl"){const e=t.result??{};return` +
+
page
${z(e.page_id??"-")}
+
page_type
${z(e.page_type??"-")}
+
crawl
${z(e.crawl_status??"-")}
+
extraction
${z(e.extraction_status??"-")}
+
claims
${e.claim_count??0}
+
entities
${e.entity_count??0}
+
+ `}return""}function Wv(t){if(t.kind==="entity"){const e=t.data;return` +
+

${z(e.name)}

+
+
${z(V("table.id"))}
${z(e.id)}
+
${z(V("table.type"))}
${z(e.type)}
+
+ 
${z(JSON.stringify(e.metadata??{},null,2))}
+
+ `}if(t.kind==="claim"){const e=t.data;return` +
+

${z(e.subject)} · ${z(e.predicate)}

+
+
${z(V("table.object"))}
${z(e.object??JSON.stringify(e.object_value??""))}
+
${z(V("table.status"))}
${z(e.status??"")}
+
${z(V("table.confidence"))}
${Math.round((e.confidence??0)*100)}%
+
+ ${e.evidence_text?`

${z(e.evidence_text)}

`:""} +
+ `}if(t.kind==="graph_node"){const e=t.data;return` +
+

${z(e.label)}

+
+
${z(V("table.id"))}
${z(e.entityId)}
+
${z(V("table.type"))}
${z(e.type??"")}
+ ${e.canonical?`
canonical
${z(e.canonical)}
`:""} +
+ ${e.metadata&&Object.keys(e.metadata).length?`
${z(JSON.stringify(e.metadata,null,2))}
`:""} +
+ `}if(t.kind==="graph_edge"){const e=t.data;return` +
+

${z(e.predicate)}

+
+
${z(V("table.confidence"))}
${Math.round((e.confidence??0)*100)}%
+ ${e.targetName?`
${z(V("table.object"))}
${z(e.targetName)}
`:""} +
claim
${z(e.claimId)}
+
+ ${e.metadata&&Object.keys(e.metadata).length?`
${z(JSON.stringify(e.metadata,null,2))}
`:""} +
+ `}return""}function ho(t,e){(e==null||e>t.length)&&(e=t.length);for(var r=0,a=Array(e);r=t.length?{done:!0}:{done:!1,value:t[a++]}},e:function(l){throw l},f:n}}throw new TypeError(`Invalid attempt to iterate non-iterable instance. +In order to be iterable, non-array objects must have a [Symbol.iterator]() method.`)}var i,s=!0,o=!1;return{s:function(){r=r.call(t)},n:function(){var l=r.next();return s=l.done,l},e:function(l){o=!0,i=l},f:function(){try{s||r.return==null||r.return()}finally{if(o)throw i}}}}function fd(t,e,r){return(e=vd(e))in t?Object.defineProperty(t,e,{value:r,enumerable:!0,configurable:!0,writable:!0}):t[e]=r,t}function Zv(t){if(typeof Symbol<"u"&&t[Symbol.iterator]!=null||t["@@iterator"]!=null)return Array.from(t)}function Qv(t,e){var r=t==null?null:typeof Symbol<"u"&&t[Symbol.iterator]||t["@@iterator"];if(r!=null){var a,n,i,s,o=[],l=!0,u=!1;try{if(i=(r=r.call(t)).next,e===0){if(Object(r)!==r)return;l=!1}else for(;!(l=(a=i.call(r)).done)&&(o.push(a.value),o.length!==e);l=!0);}catch(c){u=!0,n=c}finally{try{if(!l&&r.return!=null&&(s=r.return(),Object(s)!==s))return}finally{if(u)throw n}}return o}}function Jv(){throw new TypeError(`Invalid attempt to destructure non-iterable instance. +In order to be iterable, non-array objects must have a [Symbol.iterator]() method.`)}function eh(){throw new TypeError(`Invalid attempt to spread non-iterable instance. +In order to be iterable, non-array objects must have a [Symbol.iterator]() method.`)}function at(t,e){return jv(t)||Qv(t,e)||Uo(t,e)||Jv()}function Gn(t){return Yv(t)||Zv(t)||Uo(t)||eh()}function th(t,e){if(typeof t!="object"||!t)return t;var r=t[Symbol.toPrimitive];if(r!==void 0){var a=r.call(t,e);if(typeof a!="object")return a;throw new TypeError("@@toPrimitive must return a primitive value.")}return String(t)}function vd(t){var e=th(t,"string");return typeof e=="symbol"?e:e+""}function ot(t){"@babel/helpers - typeof";return ot=typeof Symbol=="function"&&typeof Symbol.iterator=="symbol"?function(e){return typeof e}:function(e){return e&&typeof Symbol=="function"&&e.constructor===Symbol&&e!==Symbol.prototype?"symbol":typeof e},ot(t)}function Uo(t,e){if(t){if(typeof t=="string")return ho(t,e);var r={}.toString.call(t).slice(8,-1);return r==="Object"&&t.constructor&&(r=t.constructor.name),r==="Map"||r==="Set"?Array.from(t):r==="Arguments"||/^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(r)?ho(t,e):void 0}}var it=typeof window>"u"?null:window,Dl=it?it.navigator:null;it&&it.document;var rh=ot(""),hd=ot({}),ah=ot(function(){}),nh=typeof HTMLElement>"u"?"undefined":ot(HTMLElement),Qa=function(e){return e&&e.instanceString&&Ze(e.instanceString)?e.instanceString():null},me=function(e){return e!=null&&ot(e)==rh},Ze=function(e){return e!=null&&ot(e)===ah},Ue=function(e){return!It(e)&&(Array.isArray?Array.isArray(e):e!=null&&e instanceof Array)},Ne=function(e){return e!=null&&ot(e)===hd&&!Ue(e)&&e.constructor===Object},ih=function(e){return e!=null&&ot(e)===hd},se=function(e){return e!=null&&ot(e)===ot(1)&&!isNaN(e)},sh=function(e){return se(e)&&Math.floor(e)===e},Un=function(e){if(nh!=="undefined")return e!=null&&e instanceof HTMLElement},It=function(e){return Ja(e)||pd(e)},Ja=function(e){return Qa(e)==="collection"&&e._private.single},pd=function(e){return Qa(e)==="collection"&&!e._private.single},Ko=function(e){return Qa(e)==="core"},gd=function(e){return Qa(e)==="stylesheet"},oh=function(e){return Qa(e)==="event"},Sr=function(e){return e==null?!0:!!(e===""||e.match(/^\s+$/))},lh=function(e){return typeof HTMLElement>"u"?!1:e instanceof HTMLElement},uh=function(e){return Ne(e)&&se(e.x1)&&se(e.x2)&&se(e.y1)&&se(e.y2)},ch=function(e){return ih(e)&&Ze(e.then)},dh=function(){return Dl&&Dl.userAgent.match(/msie|trident|edge/i)},pa=function(e,r){r||(r=function(){if(arguments.length===1)return arguments[0];if(arguments.length===0)return"undefined";for(var i=[],s=0;sr?1:0},mh=function(e,r){return-1*md(e,r)},xe=Object.assign!=null?Object.assign.bind(Object):function(t){for(var e=arguments,r=1;r1&&(p-=1),p<1/6?v+(y-v)*6*p:p<1/2?y:p<2/3?v+(y-v)*(2/3-p)*6:v}var d=new RegExp("^"+hh+"$").exec(e);if(d){if(a=parseInt(d[1]),a<0?a=(360- -1*a%360)%360:a>360&&(a=a%360),a/=360,n=parseFloat(d[2]),n<0||n>100||(n=n/100,i=parseFloat(d[3]),i<0||i>100)||(i=i/100,s=d[4],s!==void 0&&(s=parseFloat(s),s<0||s>1)))return;if(n===0)o=l=u=Math.round(i*255);else{var f=i<.5?i*(1+n):i+n-i*n,h=2*i-f;o=Math.round(255*c(h,f,a+1/3)),l=Math.round(255*c(h,f,a)),u=Math.round(255*c(h,f,a-1/3))}r=[o,l,u,s]}return r},xh=function(e){var r,a=new RegExp("^"+fh+"$").exec(e);if(a){r=[];for(var n=[],i=1;i<=3;i++){var s=a[i];if(s[s.length-1]==="%"&&(n[i]=!0),s=parseFloat(s),n[i]&&(s=s/100*255),s<0||s>255)return;r.push(Math.floor(s))}var o=n[1]||n[2]||n[3],l=n[1]&&n[2]&&n[3];if(o&&!l)return;var u=a[4];if(u!==void 0){if(u=parseFloat(u),u<0||u>1)return;r.push(u)}}return r},Eh=function(e){return Ch[e.toLowerCase()]},bd=function(e){return(Ue(e)?e:null)||Eh(e)||bh(e)||xh(e)||wh(e)},Ch={transparent:[0,0,0,0],aliceblue:[240,248,255],antiquewhite:[250,235,215],aqua:[0,255,255],aquamarine:[127,255,212],azure:[240,255,255],beige:[245,245,220],bisque:[255,228,196],black:[0,0,0],blanchedalmond:[255,235,205],blue:[0,0,255],blueviolet:[138,43,226],brown:[165,42,42],burlywood:[222,184,135],cadetblue:[95,158,160],chartreuse:[127,255,0],chocolate:[210,105,30],coral:[255,127,80],cornflowerblue:[100,149,237],cornsilk:[255,248,220],crimson:[220,20,60],cyan:[0,255,255],darkblue:[0,0,139],darkcyan:[0,139,139],darkgoldenrod:[184,134,11],darkgray:[169,169,169],darkgreen:[0,100,0],darkgrey:[169,169,169],darkkhaki:[189,183,107],darkmagenta:[139,0,139],darkolivegreen:[85,107,47],darkorange:[255,140,0],darkorchid:[153,50,204],darkred:[139,0,0],darksalmon:[233,150,122],darkseagreen:[143,188,143],darkslateblue:[72,61,139],darkslategray:[47,79,79],darkslategrey:[47,79,79],darkturquoise:[0,206,209],darkviolet:[148,0,211],deeppink:[255,20,147],deepskyblue:[0,191,255],dimgray:[105,105,105],dimgrey:[105,105,105],dodgerblue:[30,144,255],firebrick:[178,34,34],floralwhite:[255,250,240],forestgreen:[34,139,34],fuchsia:[255,0,255],gainsboro:[220,220,220],ghostwhite:[248,248,255],gold:[255,215,0],goldenrod:[218,165,32],gray:[128,128,128],grey:[128,128,128],green:[0,128,0],greenyellow:[173,255,47],honeydew:[240,255,240],hotpink:[255,105,180],indianred:[205,92,92],indigo:[75,0,130],ivory:[255,255,240],khaki:[240,230,140],lavender:[230,230,250],lavenderblush:[255,240,245],lawngreen:[124,252,0],lemonchiffon:[255,250,205],lightblue:[173,216,230],lightcoral:[240,128,128],lightcyan:[224,255,255],lightgoldenrodyellow:[250,250,210],lightgray:[211,211,211],lightgreen:[144,238,144],lightgrey:[211,211,211],lightpink:[255,182,193],lightsalmon:[255,160,122],lightseagreen:[32,178,170],lightskyblue:[135,206,250],lightslategray:[119,136,153],lightslategrey:[119,136,153],lightsteelblue:[176,196,222],lightyellow:[255,255,224],lime:[0,255,0],limegreen:[50,205,50],linen:[250,240,230],magenta:[255,0,255],maroon:[128,0,0],mediumaquamarine:[102,205,170],mediumblue:[0,0,205],mediumorchid:[186,85,211],mediumpurple:[147,112,219],mediumseagreen:[60,179,113],mediumslateblue:[123,104,238],mediumspringgreen:[0,250,154],mediumturquoise:[72,209,204],mediumvioletred:[199,21,133],midnightblue:[25,25,112],mintcream:[245,255,250],mistyrose:[255,228,225],moccasin:[255,228,181],navajowhite:[255,222,173],navy:[0,0,128],oldlace:[253,245,230],olive:[128,128,0],olivedrab:[107,142,35],orange:[255,165,0],orangered:[255,69,0],orchid:[218,112,214],palegoldenrod:[238,232,170],palegreen:[152,251,152],paleturquoise:[175,238,238],palevioletred:[219,112,147],papayawhip:[255,239,213],peachpuff:[255,218,185],peru:[205,133,63],pink:[255,192,203],plum:[221,160,221],powderblue:[176,224,230],purple:[128,0,128],red:[255,0,0],rosybrown:[188,143,143],royalblue:[65,105,225],saddlebrown:[139,69,19],salmon:[250,128,114],sandybrown:[244,164,96],seagreen:[46,139,87],seashell:[255,245,238],sienna:[160,82,45],silver:[192,192,192],skyblue:[135,206,235],slateblue:[106,90,205],slategray:[112,128,144],slategrey:[112,128,144],snow:[255,250,250],springgreen:[0,255,127],steelblue:[70,130,180],tan:[210,180,140],teal:[0,128,128],thistle:[216,191,216],tomato:[255,99,71],turquoise:[64,224,208],violet:[238,130,238],wheat:[245,222,179],white:[255,255,255],whitesmoke:[245,245,245],yellow:[255,255,0],yellowgreen:[154,205,50]},wd=function(e){for(var r=e.map,a=e.keys,n=a.length,i=0;i=l||A<0||m&&M>=f}function T(){var R=e();if(x(R))return k(R);v=setTimeout(T,C(R))}function k(R){return v=void 0,b&&c?w(R):(c=d=void 0,h)}function P(){v!==void 0&&clearTimeout(v),p=0,c=y=d=v=void 0}function B(){return v===void 0?h:k(e())}function D(){var R=e(),A=x(R);if(c=arguments,d=this,y=R,A){if(v===void 0)return E(y);if(m)return clearTimeout(v),v=setTimeout(T,l),w(y)}return v===void 0&&(v=setTimeout(T,l)),h}return D.cancel=P,D.flush=B,D}return Gi=s,Gi}var Mh=Lh(),an=en(Mh),Ui=it?it.performance:null,Cd=Ui&&Ui.now?function(){return Ui.now()}:function(){return Date.now()},_h=(function(){if(it){if(it.requestAnimationFrame)return function(t){it.requestAnimationFrame(t)};if(it.mozRequestAnimationFrame)return function(t){it.mozRequestAnimationFrame(t)};if(it.webkitRequestAnimationFrame)return function(t){it.webkitRequestAnimationFrame(t)};if(it.msRequestAnimationFrame)return function(t){it.msRequestAnimationFrame(t)}}return function(t){t&&setTimeout(function(){t(Cd())},1e3/60)}})(),Kn=function(e){return _h(e)},cr=Cd,Vr=9261,Td=65599,oa=5381,Sd=function(e){for(var r=arguments.length>1&&arguments[1]!==void 0?arguments[1]:Vr,a=r,n;n=e.next(),!n.done;)a=a*Td+n.value|0;return a},Va=function(e){var r=arguments.length>1&&arguments[1]!==void 0?arguments[1]:Vr;return r*Td+e|0},Ha=function(e){var r=arguments.length>1&&arguments[1]!==void 0?arguments[1]:oa;return(r<<5)+r+e|0},Ih=function(e,r){return e*2097152+r},gr=function(e){return e[0]*2097152+e[1]},hn=function(e,r){return[Va(e[0],r[0]),Ha(e[1],r[1])]},Ul=function(e,r){var a={value:0,done:!1},n=0,i=e.length,s={next:function(){return n=0;n--)e[n]===r&&e.splice(n,1)},Zo=function(e){e.splice(0,e.length)},Uh=function(e,r){for(var a=0;a"u"?"undefined":ot(Set))!==Wh?Set:jh,ui=function(e,r){var a=arguments.length>2&&arguments[2]!==void 0?arguments[2]:!0;if(e===void 0||r===void 0||!Ko(e)){Ye("An element must have a core reference and parameters set");return}var n=r.group;if(n==null&&(r.data&&r.data.source!=null&&r.data.target!=null?n="edges":n="nodes"),n!=="nodes"&&n!=="edges"){Ye("An element must be of type `nodes` or `edges`; you specified `"+n+"`");return}this.length=1,this[0]=this;var i=this._private={cy:e,single:!0,data:r.data||{},position:r.position||{x:0,y:0},autoWidth:void 0,autoHeight:void 0,autoPadding:void 0,compoundBoundsClean:!1,listeners:[],group:n,style:{},rstyle:{},styleCxts:[],styleKeys:{},removed:!0,selected:!!r.selected,selectable:r.selectable===void 0?!0:!!r.selectable,locked:!!r.locked,grabbed:!1,grabbable:r.grabbable===void 0?!0:!!r.grabbable,pannable:r.pannable===void 0?n==="edges":!!r.pannable,active:!1,classes:new ma,animation:{current:[],queue:[]},rscratch:{},scratch:r.scratch||{},edges:[],children:[],parent:r.parent&&r.parent.isNode()?r.parent:null,traversalCache:{},backgrounding:!1,bbCache:null,bbCacheShift:{x:0,y:0},bodyBounds:null,overlayBounds:null,labelBounds:{all:null,source:null,target:null,main:null},arrowBounds:{source:null,target:null,"mid-source":null,"mid-target":null}};if(i.position.x==null&&(i.position.x=0),i.position.y==null&&(i.position.y=0),r.renderedPosition){var s=r.renderedPosition,o=e.pan(),l=e.zoom();i.position={x:(s.x-o.x)/l,y:(s.y-o.y)/l}}var u=[];Ue(r.classes)?u=r.classes:me(r.classes)&&(u=r.classes.split(/\s+/));for(var c=0,d=u.length;cm?1:0},c=function(g,m,b,w,E){var C;if(b==null&&(b=0),E==null&&(E=a),b<0)throw new Error("lo must be non-negative");for(w==null&&(w=g.length);bP;0<=P?k++:k--)T.push(k);return T}).apply(this).reverse(),x=[],w=0,E=C.length;wB;0<=B?++T:--T)D.push(s(g,b));return D},y=function(g,m,b,w){var E,C,x;for(w==null&&(w=a),E=g[b];b>m;){if(x=b-1>>1,C=g[x],w(E,C)<0){g[b]=C,b=x;continue}break}return g[b]=E},p=function(g,m,b){var w,E,C,x,T;for(b==null&&(b=a),E=g.length,T=m,C=g[m],w=2*m+1;w0;){var C=m.pop(),x=p(C),T=C.id();if(f[T]=x,x!==1/0)for(var k=C.neighborhood().intersect(v),P=0;P0)for(I.unshift(L);d[G];){var O=d[G];I.unshift(O.edge),I.unshift(O.node),H=O.node,G=H.id()}return o.spawn(I)}}}},tp={kruskal:function(e){e=e||function(b){return 1};for(var r=this.byGroup(),a=r.nodes,n=r.edges,i=a.length,s=new Array(i),o=a,l=function(w){for(var E=0;E0;){if(E(),x++,w===c){for(var T=[],k=i,P=c,B=g[P];T.unshift(k),B!=null&&T.unshift(B),k=p[P],k!=null;)P=k.id(),B=g[P];return{found:!0,distance:d[w],path:this.spawn(T),steps:x}}h[w]=!0;for(var D=b._private.edges,R=0;RB&&(v[P]=B,m[P]=k,b[P]=E),!i){var D=k*c+T;!i&&v[D]>B&&(v[D]=B,m[D]=T,b[D]=E)}}}for(var R=0;R1&&arguments[1]!==void 0?arguments[1]:s,ce=b(Ee),ye=[],pe=ce;;){if(pe==null)return r.spawn();var Se=m(pe),Ce=Se.edge,De=Se.pred;if(ye.unshift(pe[0]),pe.same(be)&&ye.length>0)break;Ce!=null&&ye.unshift(Ce),pe=De}return l.spawn(ye)},C=0;C=0;c--){var d=u[c],f=d[1],h=d[2];(r[f]===o&&r[h]===l||r[f]===l&&r[h]===o)&&u.splice(c,1)}for(var v=0;vn;){var i=Math.floor(Math.random()*r.length);r=up(i,e,r),a--}return r},cp={kargerStein:function(){var e=this,r=this.byGroup(),a=r.nodes,n=r.edges;n.unmergeBy(function(I){return I.isLoop()});var i=a.length,s=n.length,o=Math.ceil(Math.pow(Math.log(i)/Math.LN2,2)),l=Math.floor(i/lp);if(i<2){Ye("At least 2 nodes are required for Karger-Stein algorithm");return}for(var u=[],c=0;c1&&arguments[1]!==void 0?arguments[1]:0,a=arguments.length>2&&arguments[2]!==void 0?arguments[2]:e.length,n=1/0,i=r;i1&&arguments[1]!==void 0?arguments[1]:0,a=arguments.length>2&&arguments[2]!==void 0?arguments[2]:e.length,n=-1/0,i=r;i1&&arguments[1]!==void 0?arguments[1]:0,a=arguments.length>2&&arguments[2]!==void 0?arguments[2]:e.length,n=0,i=0,s=r;s1&&arguments[1]!==void 0?arguments[1]:0,a=arguments.length>2&&arguments[2]!==void 0?arguments[2]:e.length,n=arguments.length>3&&arguments[3]!==void 0?arguments[3]:!0,i=arguments.length>4&&arguments[4]!==void 0?arguments[4]:!0,s=arguments.length>5&&arguments[5]!==void 0?arguments[5]:!0;n?e=e.slice(r,a):(a0&&e.splice(0,r));for(var o=0,l=e.length-1;l>=0;l--){var u=e[l];s?isFinite(u)||(e[l]=-1/0,o++):e.splice(l,1)}i&&e.sort(function(f,h){return f-h});var c=e.length,d=Math.floor(c/2);return c%2!==0?e[d+1+o]:(e[d-1+o]+e[d+o])/2},gp=function(e){return Math.PI*e/180},pn=function(e,r){return Math.atan2(r,e)-Math.PI/2},Qo=Math.log2||function(t){return Math.log(t)/Math.log(2)},Jo=function(e){return e>0?1:e<0?-1:0},Kr=function(e,r){return Math.sqrt($r(e,r))},$r=function(e,r){var a=r.x-e.x,n=r.y-e.y;return a*a+n*n},yp=function(e){for(var r=e.length,a=0,n=0;n=e.x1&&e.y2>=e.y1)return{x1:e.x1,y1:e.y1,x2:e.x2,y2:e.y2,w:e.x2-e.x1,h:e.y2-e.y1};if(e.w!=null&&e.h!=null&&e.w>=0&&e.h>=0)return{x1:e.x1,y1:e.y1,x2:e.x1+e.w,y2:e.y1+e.h,w:e.w,h:e.h}}},bp=function(e){return{x1:e.x1,x2:e.x2,w:e.w,y1:e.y1,y2:e.y2,h:e.h}},wp=function(e){e.x1=1/0,e.y1=1/0,e.x2=-1/0,e.y2=-1/0,e.w=0,e.h=0},xp=function(e,r){e.x1=Math.min(e.x1,r.x1),e.x2=Math.max(e.x2,r.x2),e.w=e.x2-e.x1,e.y1=Math.min(e.y1,r.y1),e.y2=Math.max(e.y2,r.y2),e.h=e.y2-e.y1},Ld=function(e,r,a){e.x1=Math.min(e.x1,r),e.x2=Math.max(e.x2,r),e.w=e.x2-e.x1,e.y1=Math.min(e.y1,a),e.y2=Math.max(e.y2,a),e.h=e.y2-e.y1},Mn=function(e){var r=arguments.length>1&&arguments[1]!==void 0?arguments[1]:0;return e.x1-=r,e.x2+=r,e.y1-=r,e.y2+=r,e.w=e.x2-e.x1,e.h=e.y2-e.y1,e},_n=function(e){var r=arguments.length>1&&arguments[1]!==void 0?arguments[1]:[0],a,n,i,s;if(r.length===1)a=n=i=s=r[0];else if(r.length===2)a=i=r[0],s=n=r[1];else if(r.length===4){var o=at(r,4);a=o[0],n=o[1],i=o[2],s=o[3]}return e.x1-=s,e.x2+=n,e.y1-=a,e.y2+=i,e.w=e.x2-e.x1,e.h=e.y2-e.y1,e},Zl=function(e,r){e.x1=r.x1,e.y1=r.y1,e.x2=r.x2,e.y2=r.y2,e.w=e.x2-e.x1,e.h=e.y2-e.y1},el=function(e,r){return!(e.x1>r.x2||r.x1>e.x2||e.x2r.y2||r.y1>e.y2)},wr=function(e,r,a){return e.x1<=r&&r<=e.x2&&e.y1<=a&&a<=e.y2},Ql=function(e,r){return wr(e,r.x,r.y)},Md=function(e,r){return wr(e,r.x1,r.y1)&&wr(e,r.x2,r.y2)},Ep=(ji=Math.hypot)!==null&&ji!==void 0?ji:function(t,e){return Math.sqrt(t*t+e*e)};function Cp(t,e){if(t.length<3)throw new Error("Need at least 3 vertices");var r=function(T,k){return{x:T.x+k.x,y:T.y+k.y}},a=function(T,k){return{x:T.x-k.x,y:T.y-k.y}},n=function(T,k){return{x:T.x*k,y:T.y*k}},i=function(T,k){return T.x*k.y-T.y*k.x},s=function(T){var k=Ep(T.x,T.y);return k===0?{x:0,y:0}:{x:T.x/k,y:T.y/k}},o=function(T){for(var k=0,P=0;P7&&arguments[7]!==void 0?arguments[7]:"auto",u=l==="auto"?Pr(i,s):l,c=i/2,d=s/2;u=Math.min(u,c,d);var f=u!==c,h=u!==d,v;if(f){var y=a-c+u-o,p=n-d-o,g=a+c-u+o,m=p;if(v=xr(e,r,a,n,y,p,g,m,!1),v.length>0)return v}if(h){var b=a+c+o,w=n-d+u-o,E=b,C=n+d-u+o;if(v=xr(e,r,a,n,b,w,E,C,!1),v.length>0)return v}if(f){var x=a-c+u-o,T=n+d+o,k=a+c-u+o,P=T;if(v=xr(e,r,a,n,x,T,k,P,!1),v.length>0)return v}if(h){var B=a-c-o,D=n-d+u-o,R=B,A=n+d-u+o;if(v=xr(e,r,a,n,B,D,R,A,!1),v.length>0)return v}var M;{var _=a-c+u,L=n-d+u;if(M=Ia(e,r,a,n,_,L,u+o),M.length>0&&M[0]<=_&&M[1]<=L)return[M[0],M[1]]}{var I=a+c-u,H=n-d+u;if(M=Ia(e,r,a,n,I,H,u+o),M.length>0&&M[0]>=I&&M[1]<=H)return[M[0],M[1]]}{var G=a+c-u,O=n+d-u;if(M=Ia(e,r,a,n,G,O,u+o),M.length>0&&M[0]>=G&&M[1]>=O)return[M[0],M[1]]}{var $=a-c+u,Y=n+d-u;if(M=Ia(e,r,a,n,$,Y,u+o),M.length>0&&M[0]<=$&&M[1]>=Y)return[M[0],M[1]]}return[]},Sp=function(e,r,a,n,i,s,o){var l=o,u=Math.min(a,i),c=Math.max(a,i),d=Math.min(n,s),f=Math.max(n,s);return u-l<=e&&e<=c+l&&d-l<=r&&r<=f+l},kp=function(e,r,a,n,i,s,o,l,u){var c={x1:Math.min(a,o,i)-u,x2:Math.max(a,o,i)+u,y1:Math.min(n,l,s)-u,y2:Math.max(n,l,s)+u};return!(ec.x2||rc.y2)},Pp=function(e,r,a,n){a-=n;var i=r*r-4*e*a;if(i<0)return[];var s=Math.sqrt(i),o=2*e,l=(-r+s)/o,u=(-r-s)/o;return[l,u]},Bp=function(e,r,a,n,i){var s=1e-5;e===0&&(e=s),r/=e,a/=e,n/=e;var o,l,u,c,d,f,h,v;if(l=(3*a-r*r)/9,u=-(27*n)+r*(9*a-2*(r*r)),u/=54,o=l*l*l+u*u,i[1]=0,h=r/3,o>0){d=u+Math.sqrt(o),d=d<0?-Math.pow(-d,1/3):Math.pow(d,1/3),f=u-Math.sqrt(o),f=f<0?-Math.pow(-f,1/3):Math.pow(f,1/3),i[0]=-h+d+f,h+=(d+f)/2,i[4]=i[2]=-h,h=Math.sqrt(3)*(-f+d)/2,i[3]=h,i[5]=-h;return}if(i[5]=i[3]=0,o===0){v=u<0?-Math.pow(-u,1/3):Math.pow(u,1/3),i[0]=-h+2*v,i[4]=i[2]=-(v+h);return}l=-l,c=l*l*l,c=Math.acos(u/Math.sqrt(c)),v=2*Math.sqrt(l),i[0]=-h+v*Math.cos(c/3),i[2]=-h+v*Math.cos((c+2*Math.PI)/3),i[4]=-h+v*Math.cos((c+4*Math.PI)/3)},Dp=function(e,r,a,n,i,s,o,l){var u=1*a*a-4*a*i+2*a*o+4*i*i-4*i*o+o*o+n*n-4*n*s+2*n*l+4*s*s-4*s*l+l*l,c=9*a*i-3*a*a-3*a*o-6*i*i+3*i*o+9*n*s-3*n*n-3*n*l-6*s*s+3*s*l,d=3*a*a-6*a*i+a*o-a*e+2*i*i+2*i*e-o*e+3*n*n-6*n*s+n*l-n*r+2*s*s+2*s*r-l*r,f=1*a*i-a*a+a*e-i*e+n*s-n*n+n*r-s*r,h=[];Bp(u,c,d,f,h);for(var v=1e-7,y=[],p=0;p<6;p+=2)Math.abs(h[p+1])=0&&h[p]<=1&&y.push(h[p]);y.push(1),y.push(0);for(var g=-1,m,b,w,E=0;E=0?wu?(e-i)*(e-i)+(r-s)*(r-s):c-f},At=function(e,r,a){for(var n,i,s,o,l,u=0,c=0;c=e&&e>=s||n<=e&&e<=s)l=(e-n)/(s-n)*(o-i)+i,l>r&&u++;else continue;return u%2!==0},dr=function(e,r,a,n,i,s,o,l,u){var c=new Array(a.length),d;l[0]!=null?(d=Math.atan(l[1]/l[0]),l[0]<0?d=d+Math.PI/2:d=-d-Math.PI/2):d=l;for(var f=Math.cos(-d),h=Math.sin(-d),v=0;v0){var p=Yn(c,-u);y=jn(p)}else y=c;return At(e,r,y)},Ap=function(e,r,a,n,i,s,o,l){for(var u=new Array(a.length*2),c=0;c=0&&p<=1&&m.push(p),g>=0&&g<=1&&m.push(g),m.length===0)return[];var b=m[0]*l[0]+e,w=m[0]*l[1]+r;if(m.length>1){if(m[0]==m[1])return[b,w];var E=m[1]*l[0]+e,C=m[1]*l[1]+r;return[b,w,E,C]}else return[b,w]},Yi=function(e,r,a){return r<=e&&e<=a||a<=e&&e<=r?e:e<=r&&r<=a||a<=r&&r<=e?r:a},xr=function(e,r,a,n,i,s,o,l,u){var c=e-i,d=a-e,f=o-i,h=r-s,v=n-r,y=l-s,p=f*h-y*c,g=d*h-v*c,m=y*d-f*v;if(m!==0){var b=p/m,w=g/m,E=.001,C=0-E,x=1+E;return C<=b&&b<=x&&C<=w&&w<=x?[e+b*d,r+b*v]:u?[e+b*d,r+b*v]:[]}else return p===0||g===0?Yi(e,a,o)===o?[o,l]:Yi(e,a,i)===i?[i,s]:Yi(i,o,a)===a?[a,n]:[]:[]},Mp=function(e,r,a,n,i){var s=[],o=n/2,l=i/2,u=r,c=a;s.push({x:u+o*e[0],y:c+l*e[1]});for(var d=1;d0){var y=Yn(d,-l);h=jn(y)}else h=d}else h=a;for(var p,g,m,b,w=0;w2){for(var v=[c[0],c[1]],y=Math.pow(v[0]-e,2)+Math.pow(v[1]-r,2),p=1;pc&&(c=w)},get:function(b){return u[b]}},f=0;f0?M=A.edgesTo(R)[0]:M=R.edgesTo(A)[0];var _=n(M);R=R.id(),x[R]>x[B]+_&&(x[R]=x[B]+_,T.nodes.indexOf(R)<0?T.push(R):T.updateItem(R),C[R]=0,E[R]=[]),x[R]==x[B]+_&&(C[R]=C[R]+C[B],E[R].push(B))}else for(var L=0;L0;){for(var O=w.pop(),$=0;$0&&o.push(a[l]);o.length!==0&&i.push(n.collection(o))}return i},jp=function(e,r){for(var a=0;a5&&arguments[5]!==void 0?arguments[5]:Zp,o=n,l,u,c=0;c=2?Ba(e,r,a,0,au,Qp):Ba(e,r,a,0,ru)},squaredEuclidean:function(e,r,a){return Ba(e,r,a,0,au)},manhattan:function(e,r,a){return Ba(e,r,a,0,ru)},max:function(e,r,a){return Ba(e,r,a,-1/0,Jp)}};ga["squared-euclidean"]=ga.squaredEuclidean;ga.squaredeuclidean=ga.squaredEuclidean;function di(t,e,r,a,n,i){var s;return Ze(t)?s=t:s=ga[t]||ga.euclidean,e===0&&Ze(t)?s(n,i):s(e,r,a,n,i)}var eg=gt({k:2,m:2,sensitivityThreshold:1e-4,distance:"euclidean",maxIterations:10,attributes:[],testMode:!1,testCentroids:null}),rl=function(e){return eg(e)},Xn=function(e,r,a,n,i){var s=i!=="kMedoids",o=s?function(d){return a[d]}:function(d){return n[d](a)},l=function(f){return n[f](r)},u=a,c=r;return di(e,n.length,o,l,u,c)},Zi=function(e,r,a){for(var n=a.length,i=new Array(n),s=new Array(n),o=new Array(r),l=null,u=0;ua)return!1}return!0},ag=function(e,r,a){for(var n=0;no&&(o=r[u][c],l=c);i[l].push(e[u])}for(var d=0;d=i.threshold||i.mode==="dendrogram"&&e.length===1)return!1;var v=r[s],y=r[n[s]],p;i.mode==="dendrogram"?p={left:v,right:y,key:v.key}:p={value:v.value.concat(y.value),key:v.key},e[v.index]=p,e.splice(y.index,1),r[v.key]=p;for(var g=0;ga[y.key][m.key]&&(l=a[y.key][m.key])):i.linkage==="max"?(l=a[v.key][m.key],a[v.key][m.key]0&&n.push(i);return n},uu=function(e,r,a){for(var n=[],i=0;io&&(s=u,o=r[i*e+u])}s>0&&n.push(s)}for(var c=0;cu&&(l=c,u=d)}a[i]=s[l]}return n=uu(e,r,a),n},cu=function(e){for(var r=this.cy(),a=this.nodes(),n=pg(e),i={},s=0;s=B?(D=B,B=A,R=M):A>D&&(D=A);for(var _=0;_0?1:0;x[k%n.minIterations*o+$]=Y,O+=Y}if(O>0&&(k>=n.minIterations-1||k==n.maxIterations-1)){for(var te=0,J=0;J1||C>1)&&(o=!0),d[b]=[],m.outgoers().forEach(function(T){T.isEdge()&&d[b].push(T.id())})}else f[b]=[void 0,m.target().id()]}):s.forEach(function(m){var b=m.id();if(m.isNode()){var w=m.degree(!0);w%2&&(l?u?o=!0:u=b:l=b),d[b]=[],m.connectedEdges().forEach(function(E){return d[b].push(E.id())})}else f[b]=[m.source().id(),m.target().id()]});var h={found:!1,trail:void 0};if(o)return h;if(u&&l)if(i){if(c&&u!=c)return h;c=u}else{if(c&&u!=c&&l!=c)return h;c||(c=u)}else c||(c=s[0].id());var v=function(b){for(var w=b,E=[b],C,x,T;d[w].length;)C=d[w].shift(),x=f[C][0],T=f[C][1],w!=T?(d[T]=d[T].filter(function(k){return k!=C}),w=T):!i&&w!=x&&(d[x]=d[x].filter(function(k){return k!=C}),w=x),E.unshift(C),E.unshift(w);return E},y=[],p=[];for(p=v(c);p.length!=1;)d[p[0]].length==0?(y.unshift(s.getElementById(p.shift())),y.unshift(s.getElementById(p.shift()))):p=v(p.shift()).concat(p);y.unshift(s.getElementById(p.shift()));for(var g in d)if(d[g].length)return h;return h.found=!0,h.trail=this.spawn(y,!0),h}},yn=function(){var e=this,r={},a=0,n=0,i=[],s=[],o={},l=function(f,h){for(var v=s.length-1,y=[],p=e.spawn();s[v].x!=f||s[v].y!=h;)y.push(s.pop().edge),v--;y.push(s.pop().edge),y.forEach(function(g){var m=g.connectedNodes().intersection(e);p.merge(g),m.forEach(function(b){var w=b.id(),E=b.connectedEdges().intersection(e);p.merge(b),r[w].cutVertex?p.merge(E.filter(function(C){return C.isLoop()})):p.merge(E)})}),i.push(p)},u=function(f,h,v){f===v&&(n+=1),r[h]={id:a,low:a++,cutVertex:!1};var y=e.getElementById(h).connectedEdges().intersection(e);if(y.size()===0)i.push(e.spawn(e.getElementById(h)));else{var p,g,m,b;y.forEach(function(w){p=w.source().id(),g=w.target().id(),m=p===h?g:p,m!==v&&(b=w.id(),o[b]||(o[b]=!0,s.push({x:h,y:m,edge:w})),m in r?r[h].low=Math.min(r[h].low,r[m].id):(u(f,m,h),r[h].low=Math.min(r[h].low,r[m].low),r[h].id<=r[m].low&&(r[h].cutVertex=!0,l(h,m))))})}};e.forEach(function(d){if(d.isNode()){var f=d.id();f in r||(n=0,u(f,f),r[f].cutVertex=n>1)}});var c=Object.keys(r).filter(function(d){return r[d].cutVertex}).map(function(d){return e.getElementById(d)});return{cut:e.spawn(c),components:i}},Cg={hopcroftTarjanBiconnected:yn,htbc:yn,htb:yn,hopcroftTarjanBiconnectedComponents:yn},mn=function(){var e=this,r={},a=0,n=[],i=[],s=e.spawn(e),o=function(u){i.push(u),r[u]={index:a,low:a++,explored:!1};var c=e.getElementById(u).connectedEdges().intersection(e);if(c.forEach(function(y){var p=y.target().id();p!==u&&(p in r||o(p),r[p].explored||(r[u].low=Math.min(r[u].low,r[p].low)))}),r[u].index===r[u].low){for(var d=e.spawn();;){var f=i.pop();if(d.merge(e.getElementById(f)),r[f].low=r[u].index,r[f].explored=!0,f===u)break}var h=d.edgesWith(d),v=d.merge(h);n.push(v),s=s.difference(v)}};return e.forEach(function(l){if(l.isNode()){var u=l.id();u in r||o(u)}}),{cut:s,components:n}},Tg={tarjanStronglyConnected:mn,tsc:mn,tscc:mn,tarjanStronglyConnectedComponents:mn},qd={};[Ga,ep,tp,ap,ip,op,cp,Np,fa,va,yo,Xp,ug,vg,wg,Eg,Cg,Tg].forEach(function(t){xe(qd,t)});var Vd=0,Hd=1,Gd=2,Wt=function(e){if(!(this instanceof Wt))return new Wt(e);this.id="Thenable/1.0.7",this.state=Vd,this.fulfillValue=void 0,this.rejectReason=void 0,this.onFulfilled=[],this.onRejected=[],this.proxy={then:this.then.bind(this)},typeof e=="function"&&e.call(this,this.fulfill.bind(this),this.reject.bind(this))};Wt.prototype={fulfill:function(e){return du(this,Hd,"fulfillValue",e)},reject:function(e){return du(this,Gd,"rejectReason",e)},then:function(e,r){var a=this,n=new Wt;return a.onFulfilled.push(vu(e,n,"fulfill")),a.onRejected.push(vu(r,n,"reject")),Ud(a),n.proxy}};var du=function(e,r,a,n){return e.state===Vd&&(e.state=r,e[a]=n,Ud(e)),e},Ud=function(e){e.state===Hd?fu(e,"onFulfilled",e.fulfillValue):e.state===Gd&&fu(e,"onRejected",e.rejectReason)},fu=function(e,r,a){if(e[r].length!==0){var n=e[r];e[r]=[];var i=function(){for(var o=0;o0}},clearQueue:function(){return function(){var r=this,a=r.length!==void 0,n=a?r:[r],i=this._private.cy||this;if(!i.styleEnabled())return this;for(var s=0;s-1}return bs=e,bs}var ws,Ou;function Hg(){if(Ou)return ws;Ou=1;var t=hi();function e(r,a){var n=this.__data__,i=t(n,r);return i<0?(++this.size,n.push([r,a])):n[i][1]=a,this}return ws=e,ws}var xs,Nu;function Gg(){if(Nu)return xs;Nu=1;var t=$g(),e=Fg(),r=qg(),a=Vg(),n=Hg();function i(s){var o=-1,l=s==null?0:s.length;for(this.clear();++o-1&&a%1==0&&a0&&this.spawn(n).updateStyle().emit("class"),r},addClass:function(e){return this.toggleClass(e,!0)},hasClass:function(e){var r=this[0];return r!=null&&r._private.classes.has(e)},toggleClass:function(e,r){Ue(e)||(e=e.match(/\S+/g)||[]);for(var a=this,n=r===void 0,i=[],s=0,o=a.length;s0&&this.spawn(i).updateStyle().emit("class"),a},removeClass:function(e){return this.toggleClass(e,!1)},flashClass:function(e,r){var a=this;if(r==null)r=250;else if(r===0)return a;return a.addClass(e),setTimeout(function(){a.removeClass(e)},r),a}};In.className=In.classNames=In.classes;var Oe={metaChar:"[\\!\\\"\\#\\$\\%\\&\\'\\(\\)\\*\\+\\,\\.\\/\\:\\;\\<\\=\\>\\?\\@\\[\\]\\^\\`\\{\\|\\}\\~]",comparatorOp:"=|\\!=|>|>=|<|<=|\\$=|\\^=|\\*=",boolOp:"\\?|\\!|\\^",string:`"(?:\\\\"|[^"])*"|'(?:\\\\'|[^'])*'`,number:st,meta:"degree|indegree|outdegree",separator:"\\s*,\\s*",descendant:"\\s+",child:"\\s+>\\s+",subject:"\\$",group:"node|edge|\\*",directedEdge:"\\s+->\\s+",undirectedEdge:"\\s+<->\\s+"};Oe.variable="(?:[\\w-.]|(?:\\\\"+Oe.metaChar+"))+";Oe.className="(?:[\\w-]|(?:\\\\"+Oe.metaChar+"))+";Oe.value=Oe.string+"|"+Oe.number;Oe.id=Oe.variable;(function(){var t,e,r;for(t=Oe.comparatorOp.split("|"),r=0;r=0)&&e!=="="&&(Oe.comparatorOp+="|\\!"+e)})();var Ge=function(){return{checks:[]}},fe={GROUP:0,COLLECTION:1,FILTER:2,DATA_COMPARE:3,DATA_EXIST:4,DATA_BOOL:5,META_COMPARE:6,STATE:7,ID:8,CLASS:9,UNDIRECTED_EDGE:10,DIRECTED_EDGE:11,NODE_SOURCE:12,NODE_TARGET:13,NODE_NEIGHBOR:14,CHILD:15,DESCENDANT:16,PARENT:17,ANCESTOR:18,COMPOUND_SPLIT:19,TRUE:20},xo=[{selector:":selected",matches:function(e){return e.selected()}},{selector:":unselected",matches:function(e){return!e.selected()}},{selector:":selectable",matches:function(e){return e.selectable()}},{selector:":unselectable",matches:function(e){return!e.selectable()}},{selector:":locked",matches:function(e){return e.locked()}},{selector:":unlocked",matches:function(e){return!e.locked()}},{selector:":visible",matches:function(e){return e.visible()}},{selector:":hidden",matches:function(e){return!e.visible()}},{selector:":transparent",matches:function(e){return e.transparent()}},{selector:":grabbed",matches:function(e){return e.grabbed()}},{selector:":free",matches:function(e){return!e.grabbed()}},{selector:":removed",matches:function(e){return e.removed()}},{selector:":inside",matches:function(e){return!e.removed()}},{selector:":grabbable",matches:function(e){return e.grabbable()}},{selector:":ungrabbable",matches:function(e){return!e.grabbable()}},{selector:":animated",matches:function(e){return e.animated()}},{selector:":unanimated",matches:function(e){return!e.animated()}},{selector:":parent",matches:function(e){return e.isParent()}},{selector:":childless",matches:function(e){return e.isChildless()}},{selector:":child",matches:function(e){return e.isChild()}},{selector:":orphan",matches:function(e){return e.isOrphan()}},{selector:":nonorphan",matches:function(e){return e.isChild()}},{selector:":compound",matches:function(e){return e.isNode()?e.isParent():e.source().isParent()||e.target().isParent()}},{selector:":loop",matches:function(e){return e.isLoop()}},{selector:":simple",matches:function(e){return e.isSimple()}},{selector:":active",matches:function(e){return e.active()}},{selector:":inactive",matches:function(e){return!e.active()}},{selector:":backgrounding",matches:function(e){return e.backgrounding()}},{selector:":nonbackgrounding",matches:function(e){return!e.backgrounding()}}].sort(function(t,e){return mh(t.selector,e.selector)}),xy=(function(){for(var t={},e,r=0;r0&&c.edgeCount>0)return He("The selector `"+e+"` is invalid because it uses both a compound selector and an edge selector"),!1;if(c.edgeCount>1)return He("The selector `"+e+"` is invalid because it uses multiple edge selectors"),!1;c.edgeCount===1&&He("The selector `"+e+"` is deprecated. Edge selectors do not take effect on changes to source and target nodes after an edge is added, for performance reasons. Use a class or data selector on edges instead, updating the class or data of an edge when your app detects a change in source or target nodes.")}return!0},Py=function(){if(this.toStringCache!=null)return this.toStringCache;for(var e=function(c){return c??""},r=function(c){return me(c)?'"'+c+'"':e(c)},a=function(c){return" "+c+" "},n=function(c,d){var f=c.type,h=c.value;switch(f){case fe.GROUP:{var v=e(h);return v.substring(0,v.length-1)}case fe.DATA_COMPARE:{var y=c.field,p=c.operator;return"["+y+a(e(p))+r(h)+"]"}case fe.DATA_BOOL:{var g=c.operator,m=c.field;return"["+e(g)+m+"]"}case fe.DATA_EXIST:{var b=c.field;return"["+b+"]"}case fe.META_COMPARE:{var w=c.operator,E=c.field;return"[["+E+a(e(w))+r(h)+"]]"}case fe.STATE:return h;case fe.ID:return"#"+h;case fe.CLASS:return"."+h;case fe.PARENT:case fe.CHILD:return i(c.parent,d)+a(">")+i(c.child,d);case fe.ANCESTOR:case fe.DESCENDANT:return i(c.ancestor,d)+" "+i(c.descendant,d);case fe.COMPOUND_SPLIT:{var C=i(c.left,d),x=i(c.subject,d),T=i(c.right,d);return C+(C.length>0?" ":"")+x+T}case fe.TRUE:return""}},i=function(c,d){return c.checks.reduce(function(f,h,v){return f+(d===c&&v===0?"$":"")+n(h,d)},"")},s="",o=0;o1&&o=0&&(r=r.replace("!",""),d=!0),r.indexOf("@")>=0&&(r=r.replace("@",""),c=!0),(i||o||c)&&(l=!i&&!s?"":""+e,u=""+a),c&&(e=l=l.toLowerCase(),a=u=u.toLowerCase()),r){case"*=":n=l.indexOf(u)>=0;break;case"$=":n=l.indexOf(u,l.length-u.length)>=0;break;case"^=":n=l.indexOf(u)===0;break;case"=":n=e===a;break;case">":f=!0,n=e>a;break;case">=":f=!0,n=e>=a;break;case"<":f=!0,n=e0;){var c=n.shift();e(c),i.add(c.id()),o&&a(n,i,c)}return t}function Jd(t,e,r){if(r.isParent())for(var a=r._private.children,n=0;n1&&arguments[1]!==void 0?arguments[1]:!0;return sl(this,t,e,Jd)};function ef(t,e,r){if(r.isChild()){var a=r._private.parent;e.has(a.id())||t.push(a)}}ya.forEachUp=function(t){var e=arguments.length>1&&arguments[1]!==void 0?arguments[1]:!0;return sl(this,t,e,ef)};function Iy(t,e,r){ef(t,e,r),Jd(t,e,r)}ya.forEachUpAndDown=function(t){var e=arguments.length>1&&arguments[1]!==void 0?arguments[1]:!0;return sl(this,t,e,Iy)};ya.ancestors=ya.parents;var Wa,tf;Wa=tf={data:Ve.data({field:"data",bindingEvent:"data",allowBinding:!0,allowSetting:!0,settingEvent:"data",settingTriggersEvent:!0,triggerFnName:"trigger",allowGetting:!0,immutableKeys:{id:!0,source:!0,target:!0,parent:!0},updateStyle:!0}),removeData:Ve.removeData({field:"data",event:"data",triggerFnName:"trigger",triggerEvent:!0,immutableKeys:{id:!0,source:!0,target:!0,parent:!0},updateStyle:!0}),scratch:Ve.data({field:"scratch",bindingEvent:"scratch",allowBinding:!0,allowSetting:!0,settingEvent:"scratch",settingTriggersEvent:!0,triggerFnName:"trigger",allowGetting:!0,updateStyle:!0}),removeScratch:Ve.removeData({field:"scratch",event:"scratch",triggerFnName:"trigger",triggerEvent:!0,updateStyle:!0}),rscratch:Ve.data({field:"rscratch",allowBinding:!1,allowSetting:!0,settingTriggersEvent:!1,allowGetting:!0}),removeRscratch:Ve.removeData({field:"rscratch",triggerEvent:!1}),id:function(){var e=this[0];if(e)return e._private.data.id}};Wa.attr=Wa.data;Wa.removeAttr=Wa.removeData;var Oy=tf,gi={};function Ys(t){return function(e){var r=this;if(e===void 0&&(e=!0),r.length!==0)if(r.isNode()&&!r.removed()){for(var a=0,n=r[0],i=n._private.edges,s=0;se}),minIndegree:ea("indegree",function(t,e){return te}),minOutdegree:ea("outdegree",function(t,e){return te})});xe(gi,{totalDegree:function(e){for(var r=0,a=this.nodes(),n=0;n0,f=d;d&&(c=c[0]);var h=f?c.position():{x:0,y:0};r!==void 0?u.position(e,r+h[e]):i!==void 0&&u.position({x:i.x+h.x,y:i.y+h.y})}else{var v=a.position(),y=o?a.parent():null,p=y&&y.length>0,g=p;p&&(y=y[0]);var m=g?y.position():{x:0,y:0};return i={x:v.x-m.x,y:v.y-m.y},e===void 0?i:i[e]}else if(!s)return;return this}};Kt.modelPosition=Kt.point=Kt.position;Kt.modelPositions=Kt.points=Kt.positions;Kt.renderedPoint=Kt.renderedPosition;Kt.relativePoint=Kt.relativePosition;var Ny=rf,ha,Mr;ha=Mr={};Mr.renderedBoundingBox=function(t){var e=this.boundingBox(t),r=this.cy(),a=r.zoom(),n=r.pan(),i=e.x1*a+n.x,s=e.x2*a+n.x,o=e.y1*a+n.y,l=e.y2*a+n.y;return{x1:i,x2:s,y1:o,y2:l,w:s-i,h:l-o}};Mr.dirtyCompoundBoundsCache=function(){var t=arguments.length>0&&arguments[0]!==void 0?arguments[0]:!1,e=this.cy();return!e.styleEnabled()||!e.hasCompoundNodes()?this:(this.forEachUp(function(r){if(r.isParent()){var a=r._private;a.compoundBoundsClean=!1,a.bbCache=null,t||r.emitAndNotify("bounds")}}),this)};Mr.updateCompoundBounds=function(){var t=arguments.length>0&&arguments[0]!==void 0?arguments[0]:!1,e=this.cy();if(!e.styleEnabled()||!e.hasCompoundNodes())return this;if(!t&&e.batching())return this;function r(s){if(!s.isParent())return;var o=s._private,l=s.children(),u=s.pstyle("compound-sizing-wrt-labels").value==="include",c={width:{val:s.pstyle("min-width").pfValue,left:s.pstyle("min-width-bias-left"),right:s.pstyle("min-width-bias-right")},height:{val:s.pstyle("min-height").pfValue,top:s.pstyle("min-height-bias-top"),bottom:s.pstyle("min-height-bias-bottom")}},d=l.boundingBox({includeLabels:u,includeOverlays:!1,useCache:!1}),f=o.position;(d.w===0||d.h===0)&&(d={w:s.pstyle("width").pfValue,h:s.pstyle("height").pfValue},d.x1=f.x-d.w/2,d.x2=f.x+d.w/2,d.y1=f.y-d.h/2,d.y2=f.y+d.h/2);function h(k,P,B){var D=0,R=0,A=P+B;return k>0&&A>0&&(D=P/A*k,R=B/A*k),{biasDiff:D,biasComplementDiff:R}}function v(k,P,B,D){if(B.units==="%")switch(D){case"width":return k>0?B.pfValue*k:0;case"height":return P>0?B.pfValue*P:0;case"average":return k>0&&P>0?B.pfValue*(k+P)/2:0;case"min":return k>0&&P>0?k>P?B.pfValue*P:B.pfValue*k:0;case"max":return k>0&&P>0?k>P?B.pfValue*k:B.pfValue*P:0;default:return 0}else return B.units==="px"?B.pfValue:0}var y=c.width.left.value;c.width.left.units==="px"&&c.width.val>0&&(y=y*100/c.width.val);var p=c.width.right.value;c.width.right.units==="px"&&c.width.val>0&&(p=p*100/c.width.val);var g=c.height.top.value;c.height.top.units==="px"&&c.height.val>0&&(g=g*100/c.height.val);var m=c.height.bottom.value;c.height.bottom.units==="px"&&c.height.val>0&&(m=m*100/c.height.val);var b=h(c.width.val-d.w,y,p),w=b.biasDiff,E=b.biasComplementDiff,C=h(c.height.val-d.h,g,m),x=C.biasDiff,T=C.biasComplementDiff;o.autoPadding=v(d.w,d.h,s.pstyle("padding"),s.pstyle("padding-relative-to").value),o.autoWidth=Math.max(d.w,c.width.val),f.x=(-w+d.x1+d.x2+E)/2,o.autoHeight=Math.max(d.h,c.height.val),f.y=(-x+d.y1+d.y2+T)/2}for(var a=0;ae.x2?n:e.x2,e.y1=ae.y2?i:e.y2,e.w=e.x2-e.x1,e.h=e.y2-e.y1)},mr=function(e,r){return r==null?e:Ut(e,r.x1,r.y1,r.x2,r.y2)},Da=function(e,r,a){return Rt(e,r,a)},bn=function(e,r,a){if(!r.cy().headless()){var n=r._private,i=n.rstyle,s=i.arrowWidth/2,o=r.pstyle(a+"-arrow-shape").value,l,u;if(o!=="none"){a==="source"?(l=i.srcX,u=i.srcY):a==="target"?(l=i.tgtX,u=i.tgtY):(l=i.midX,u=i.midY);var c=n.arrowBounds=n.arrowBounds||{},d=c[a]=c[a]||{};d.x1=l-s,d.y1=u-s,d.x2=l+s,d.y2=u+s,d.w=d.x2-d.x1,d.h=d.y2-d.y1,Mn(d,1),Ut(e,d.x1,d.y1,d.x2,d.y2)}}},Xs=function(e,r,a){if(!r.cy().headless()){var n;a?n=a+"-":n="";var i=r._private,s=i.rstyle,o=r.pstyle(n+"label").strValue;if(o){var l=r.pstyle("text-halign"),u=r.pstyle("text-valign"),c=Da(s,"labelWidth",a),d=Da(s,"labelHeight",a),f=Da(s,"labelX",a),h=Da(s,"labelY",a),v=r.pstyle(n+"text-margin-x").pfValue,y=r.pstyle(n+"text-margin-y").pfValue,p=r.isEdge(),g=r.pstyle(n+"text-rotation"),m=r.pstyle("text-outline-width").pfValue,b=r.pstyle("text-border-width").pfValue,w=b/2,E=r.pstyle("text-background-padding").pfValue,C=2,x=d,T=c,k=T/2,P=x/2,B,D,R,A;if(p)B=f-k,D=f+k,R=h-P,A=h+P;else{switch(l.value){case"left":B=f-T,D=f;break;case"center":B=f-k,D=f+k;break;case"right":B=f,D=f+T;break}switch(u.value){case"top":R=h-x,A=h;break;case"center":R=h-P,A=h+P;break;case"bottom":R=h,A=h+x;break}}var M=v-Math.max(m,w)-E-C,_=v+Math.max(m,w)+E+C,L=y-Math.max(m,w)-E-C,I=y+Math.max(m,w)+E+C;B+=M,D+=_,R+=L,A+=I;var H=a||"main",G=i.labelBounds,O=G[H]=G[H]||{};O.x1=B,O.y1=R,O.x2=D,O.y2=A,O.w=D-B,O.h=A-R,O.leftPad=M,O.rightPad=_,O.topPad=L,O.botPad=I;var $=p&&g.strValue==="autorotate",Y=g.pfValue!=null&&g.pfValue!==0;if($||Y){var te=$?Da(i.rstyle,"labelAngle",a):g.pfValue,J=Math.cos(te),re=Math.sin(te),ne=(B+D)/2,oe=(R+A)/2;if(!p){switch(l.value){case"left":ne=D;break;case"right":ne=B;break}switch(u.value){case"top":oe=A;break;case"bottom":oe=R;break}}var ee=function(Te,Ee){return Te=Te-ne,Ee=Ee-oe,{x:Te*J-Ee*re+ne,y:Te*re+Ee*J+oe}},q=ee(B,R),K=ee(B,A),W=ee(D,R),Q=ee(D,A);B=Math.min(q.x,K.x,W.x,Q.x),D=Math.max(q.x,K.x,W.x,Q.x),R=Math.min(q.y,K.y,W.y,Q.y),A=Math.max(q.y,K.y,W.y,Q.y)}var ie=H+"Rot",ge=G[ie]=G[ie]||{};ge.x1=B,ge.y1=R,ge.x2=D,ge.y2=A,ge.w=D-B,ge.h=A-R,Ut(e,B,R,D,A),Ut(i.labelBounds.all,B,R,D,A)}return e}},fc=function(e,r){if(!r.cy().headless()){var a=r.pstyle("outline-opacity").value,n=r.pstyle("outline-width").value,i=r.pstyle("outline-offset").value,s=n+i;nf(e,r,a,s,"outside",s/2)}},nf=function(e,r,a,n,i,s){if(!(a===0||n<=0||i==="inside")){var o=r.cy(),l=o.renderer(),u=l.nodeShapes[l.getNodeShape(r)];if(u){var c=r.position(),d=c.x,f=c.y,h=r.width(),v=r.height();if(u.hasMiterBounds){i==="center"&&(n/=2);var y=u.miterBounds(d,f,h,v,n);mr(e,y)}else s!=null&&s>0&&_n(e,[s,s,s,s])}}},zy=function(e,r){if(!r.cy().headless()){var a=r.pstyle("border-opacity").value,n=r.pstyle("border-width").pfValue,i=r.pstyle("border-position").value;nf(e,r,a,n,i)}},$y=function(e,r){var a=e._private.cy,n=a.styleEnabled(),i=a.headless(),s=kt(),o=e._private,l=e.isNode(),u=e.isEdge(),c,d,f,h,v,y,p=o.rstyle,g=l&&n?e.pstyle("bounds-expansion").pfValue:[0],m=function(Me){return Me.pstyle("display").value!=="none"},b=!n||m(e)&&(!u||m(e.source())&&m(e.target()));if(b){var w=0,E=0;n&&r.includeOverlays&&(w=e.pstyle("overlay-opacity").value,w!==0&&(E=e.pstyle("overlay-padding").value));var C=0,x=0;n&&r.includeUnderlays&&(C=e.pstyle("underlay-opacity").value,C!==0&&(x=e.pstyle("underlay-padding").value));var T=Math.max(E,x),k=0,P=0;if(n&&(k=e.pstyle("width").pfValue,P=k/2),l&&r.includeNodes){var B=e.position();v=B.x,y=B.y;var D=e.outerWidth(),R=D/2,A=e.outerHeight(),M=A/2;c=v-R,d=v+R,f=y-M,h=y+M,Ut(s,c,f,d,h),n&&fc(s,e),n&&r.includeOutlines&&!i&&fc(s,e),n&&zy(s,e)}else if(u&&r.includeEdges)if(n&&!i){var _=e.pstyle("curve-style").strValue;if(c=Math.min(p.srcX,p.midX,p.tgtX),d=Math.max(p.srcX,p.midX,p.tgtX),f=Math.min(p.srcY,p.midY,p.tgtY),h=Math.max(p.srcY,p.midY,p.tgtY),c-=P,d+=P,f-=P,h+=P,Ut(s,c,f,d,h),_==="haystack"){var L=p.haystackPts;if(L&&L.length===2){if(c=L[0].x,f=L[0].y,d=L[1].x,h=L[1].y,c>d){var I=c;c=d,d=I}if(f>h){var H=f;f=h,h=H}Ut(s,c-P,f-P,d+P,h+P)}}else if(_==="bezier"||_==="unbundled-bezier"||br(_,"segments")||br(_,"taxi")){var G;switch(_){case"bezier":case"unbundled-bezier":G=p.bezierPts;break;case"segments":case"taxi":case"round-segments":case"round-taxi":G=p.linePts;break}if(G!=null)for(var O=0;Od){var ne=c;c=d,d=ne}if(f>h){var oe=f;f=h,h=oe}c-=P,d+=P,f-=P,h+=P,Ut(s,c,f,d,h)}if(n&&r.includeEdges&&u&&(bn(s,e,"mid-source"),bn(s,e,"mid-target"),bn(s,e,"source"),bn(s,e,"target")),n){var ee=e.pstyle("ghost").value==="yes";if(ee){var q=e.pstyle("ghost-offset-x").pfValue,K=e.pstyle("ghost-offset-y").pfValue;Ut(s,s.x1+q,s.y1+K,s.x2+q,s.y2+K)}}var W=o.bodyBounds=o.bodyBounds||{};Zl(W,s),_n(W,g),Mn(W,1),n&&(c=s.x1,d=s.x2,f=s.y1,h=s.y2,Ut(s,c-T,f-T,d+T,h+T));var Q=o.overlayBounds=o.overlayBounds||{};Zl(Q,s),_n(Q,g),Mn(Q,1);var ie=o.labelBounds=o.labelBounds||{};ie.all!=null?wp(ie.all):ie.all=kt(),n&&r.includeLabels&&(r.includeMainLabels&&Xs(s,e,null),u&&(r.includeSourceLabels&&Xs(s,e,"source"),r.includeTargetLabels&&Xs(s,e,"target")))}return s.x1=$t(s.x1),s.y1=$t(s.y1),s.x2=$t(s.x2),s.y2=$t(s.y2),s.w=$t(s.x2-s.x1),s.h=$t(s.y2-s.y1),s.w>0&&s.h>0&&b&&(_n(s,g),Mn(s,1)),s},sf=function(e){var r=0,a=function(s){return(s?1:0)<0&&arguments[0]!==void 0?arguments[0]:em,e=arguments.length>1?arguments[1]:void 0,r=0;r=0;o--)s(o);return this};Rr.removeAllListeners=function(){return this.removeListener("*")};Rr.emit=Rr.trigger=function(t,e,r){var a=this.listeners,n=a.length;return this.emitting++,Ue(e)||(e=[e]),tm(this,function(i,s){r!=null&&(a=[{event:s.event,type:s.type,namespace:s.namespace,callback:r}],n=a.length);for(var o=function(){var c=a[l];if(c.type===s.type&&(!c.namespace||c.namespace===s.namespace||c.namespace===Jy)&&i.eventMatches(i.context,c,s)){var d=[s];e!=null&&Uh(d,e),i.beforeEmit(i.context,c,s),c.conf&&c.conf.one&&(i.listeners=i.listeners.filter(function(v){return v!==c}));var f=i.callbackContext(i.context,c,s),h=c.callback.apply(f,d);i.afterEmit(i.context,c,s),h===!1&&(s.stopPropagation(),s.preventDefault())}},l=0;l1&&!s){var o=this.length-1,l=this[o],u=l._private.data.id;this[o]=void 0,this[e]=l,i.set(u,{ele:l,index:e})}return this.length--,this},unmergeOne:function(e){e=e[0];var r=this._private,a=e._private.data.id,n=r.map,i=n.get(a);if(!i)return this;var s=i.index;return this.unmergeAt(s),this},unmerge:function(e){var r=this._private.cy;if(!e)return this;if(e&&me(e)){var a=e;e=r.mutableElements().filter(a)}for(var n=0;n=0;r--){var a=this[r];e(a)&&this.unmergeAt(r)}return this},map:function(e,r){for(var a=[],n=this,i=0;ia&&(a=l,n=o)}return{value:a,ele:n}},min:function(e,r){for(var a=1/0,n,i=this,s=0;s=0&&i"u"?"undefined":ot(Symbol))!=e&&ot(Symbol.iterator)!=e;r&&(Zn[Symbol.iterator]=function(){var a=this,n={value:void 0,done:!1},i=0,s=this.length;return fd({next:function(){return i1&&arguments[1]!==void 0?arguments[1]:!0,a=this[0],n=a.cy();if(n.styleEnabled()&&a){a._private.styleDirty&&(a._private.styleDirty=!1,n.style().apply(a));var i=a._private.style[e];return i??(r?n.style().getDefaultProperty(e):null)}},numericStyle:function(e){var r=this[0];if(r.cy().styleEnabled()&&r){var a=r.pstyle(e);return a.pfValue!==void 0?a.pfValue:a.value}},numericStyleUnits:function(e){var r=this[0];if(r.cy().styleEnabled()&&r)return r.pstyle(e).units},renderedStyle:function(e){var r=this.cy();if(!r.styleEnabled())return this;var a=this[0];if(a)return r.style().getRenderedStyle(a,e)},style:function(e,r){var a=this.cy();if(!a.styleEnabled())return this;var n=!1,i=a.style();if(Ne(e)){var s=e;i.applyBypass(this,s,n),this.emitAndNotify("style")}else if(me(e))if(r===void 0){var o=this[0];return o?i.getStylePropertyValue(o,e):void 0}else i.applyBypass(this,e,r,n),this.emitAndNotify("style");else if(e===void 0){var l=this[0];return l?i.getRawStyle(l):void 0}return this},removeStyle:function(e){var r=this.cy();if(!r.styleEnabled())return this;var a=!1,n=r.style(),i=this;if(e===void 0)for(var s=0;s0&&e.push(c[0]),e.push(o[0])}return this.spawn(e,!0).filter(t)},"neighborhood"),closedNeighborhood:function(e){return this.neighborhood().add(this).filter(e)},openNeighborhood:function(e){return this.neighborhood(e)}});Et.neighbourhood=Et.neighborhood;Et.closedNeighbourhood=Et.closedNeighborhood;Et.openNeighbourhood=Et.openNeighborhood;xe(Et,{source:Ft(function(e){var r=this[0],a;return r&&(a=r._private.source||r.cy().collection()),a&&e?a.filter(e):a},"source"),target:Ft(function(e){var r=this[0],a;return r&&(a=r._private.target||r.cy().collection()),a&&e?a.filter(e):a},"target"),sources:Cc({attr:"source"}),targets:Cc({attr:"target"})});function Cc(t){return function(r){for(var a=[],n=0;n0);return s},component:function(){var e=this[0];return e.cy().mutableElements().components(e)[0]}});Et.componentsOf=Et.components;var pt=function(e,r){var a=arguments.length>2&&arguments[2]!==void 0?arguments[2]:!1,n=arguments.length>3&&arguments[3]!==void 0?arguments[3]:!1;if(e===void 0){Ye("A collection must have a reference to the core");return}var i=new or,s=!1;if(!r)r=[];else if(r.length>0&&Ne(r[0])&&!Ja(r[0])){s=!0;for(var o=[],l=new ma,u=0,c=r.length;u0&&arguments[0]!==void 0?arguments[0]:!0,e=arguments.length>1&&arguments[1]!==void 0?arguments[1]:!0,r=this,a=r.cy(),n=a._private,i=[],s=[],o,l=0,u=r.length;l0){for(var H=o.length===r.length?r:new pt(a,o),G=0;G0&&arguments[0]!==void 0?arguments[0]:!0,e=arguments.length>1&&arguments[1]!==void 0?arguments[1]:!0,r=this,a=[],n={},i=r._private.cy;function s(A){for(var M=A._private.edges,_=0;_0&&(t?B.emitAndNotify("remove"):e&&B.emit("remove"));for(var D=0;D0?D=A:B=A;while(Math.abs(R)>s&&++M=i?m(P,M):_===0?M:w(P,B,B+u)}var C=!1;function x(){C=!0,(t!==e||r!==a)&&b()}var T=function(B){return C||x(),t===e&&r===a?B:B===0?0:B===1?1:p(E(B),e,a)};T.getControlPoints=function(){return[{x:t,y:e},{x:r,y:a}]};var k="generateBezier("+[t,e,r,a]+")";return T.toString=function(){return k},T}var fm=(function(){function t(a){return-a.tension*a.x-a.friction*a.v}function e(a,n,i){var s={x:a.x+i.dx*n,v:a.v+i.dv*n,tension:a.tension,friction:a.friction};return{dx:s.v,dv:t(s)}}function r(a,n){var i={dx:a.v,dv:t(a)},s=e(a,n*.5,i),o=e(a,n*.5,s),l=e(a,n,o),u=1/6*(i.dx+2*(s.dx+o.dx)+l.dx),c=1/6*(i.dv+2*(s.dv+o.dv)+l.dv);return a.x=a.x+u*n,a.v=a.v+c*n,a}return function a(n,i,s){var o={x:-1,v:0,tension:null,friction:null},l=[0],u=0,c=1/1e4,d=16/1e3,f,h,v;for(n=parseFloat(n)||500,i=parseFloat(i)||20,s=s||null,o.tension=n,o.friction=i,f=s!==null,f?(u=a(n,i),h=u/s*d):h=d;v=r(v||o,h),l.push(1+v.x),u+=16,Math.abs(v.x)>c&&Math.abs(v.v)>c;);return f?function(y){return l[y*(l.length-1)|0]}:u}})(),Ke=function(e,r,a,n){var i=dm(e,r,a,n);return function(s,o,l){return s+(o-s)*i(l)}},Nn={linear:function(e,r,a){return e+(r-e)*a},ease:Ke(.25,.1,.25,1),"ease-in":Ke(.42,0,1,1),"ease-out":Ke(0,0,.58,1),"ease-in-out":Ke(.42,0,.58,1),"ease-in-sine":Ke(.47,0,.745,.715),"ease-out-sine":Ke(.39,.575,.565,1),"ease-in-out-sine":Ke(.445,.05,.55,.95),"ease-in-quad":Ke(.55,.085,.68,.53),"ease-out-quad":Ke(.25,.46,.45,.94),"ease-in-out-quad":Ke(.455,.03,.515,.955),"ease-in-cubic":Ke(.55,.055,.675,.19),"ease-out-cubic":Ke(.215,.61,.355,1),"ease-in-out-cubic":Ke(.645,.045,.355,1),"ease-in-quart":Ke(.895,.03,.685,.22),"ease-out-quart":Ke(.165,.84,.44,1),"ease-in-out-quart":Ke(.77,0,.175,1),"ease-in-quint":Ke(.755,.05,.855,.06),"ease-out-quint":Ke(.23,1,.32,1),"ease-in-out-quint":Ke(.86,0,.07,1),"ease-in-expo":Ke(.95,.05,.795,.035),"ease-out-expo":Ke(.19,1,.22,1),"ease-in-out-expo":Ke(1,0,0,1),"ease-in-circ":Ke(.6,.04,.98,.335),"ease-out-circ":Ke(.075,.82,.165,1),"ease-in-out-circ":Ke(.785,.135,.15,.86),spring:function(e,r,a){if(a===0)return Nn.linear;var n=fm(e,r,a);return function(i,s,o){return i+(s-i)*n(o)}},"cubic-bezier":Ke};function kc(t,e,r,a,n){if(a===1||e===r)return r;var i=n(e,r,a);return t==null||((t.roundValue||t.color)&&(i=Math.round(i)),t.min!==void 0&&(i=Math.max(i,t.min)),t.max!==void 0&&(i=Math.min(i,t.max))),i}function Pc(t,e){return t.pfValue!=null||t.value!=null?t.pfValue!=null&&(e==null||e.type.units!=="%")?t.pfValue:t.value:t}function ta(t,e,r,a,n){var i=n!=null?n.type:null;r<0?r=0:r>1&&(r=1);var s=Pc(t,n),o=Pc(e,n);if(se(s)&&se(o))return kc(i,s,o,r,a);if(Ue(s)&&Ue(o)){for(var l=[],u=0;u0?(h==="spring"&&v.push(s.duration),s.easingImpl=Nn[h].apply(null,v)):s.easingImpl=Nn[h]}var y=s.easingImpl,p;if(s.duration===0?p=1:p=(r-l)/s.duration,s.applying&&(p=s.progress),p<0?p=0:p>1&&(p=1),s.delay==null){var g=s.startPosition,m=s.position;if(m&&n&&!t.locked()){var b={};Aa(g.x,m.x)&&(b.x=ta(g.x,m.x,p,y)),Aa(g.y,m.y)&&(b.y=ta(g.y,m.y,p,y)),t.position(b)}var w=s.startPan,E=s.pan,C=i.pan,x=E!=null&&a;x&&(Aa(w.x,E.x)&&(C.x=ta(w.x,E.x,p,y)),Aa(w.y,E.y)&&(C.y=ta(w.y,E.y,p,y)),t.emit("pan"));var T=s.startZoom,k=s.zoom,P=k!=null&&a;P&&(Aa(T,k)&&(i.zoom=Ua(i.minZoom,ta(T,k,p,y),i.maxZoom)),t.emit("zoom")),(x||P)&&t.emit("viewport");var B=s.style;if(B&&B.length>0&&n){for(var D=0;D=0;x--){var T=C[x];T()}C.splice(0,C.length)},m=h.length-1;m>=0;m--){var b=h[m],w=b._private;if(w.stopped){h.splice(m,1),w.hooked=!1,w.playing=!1,w.started=!1,g(w.frames);continue}!w.playing&&!w.applying||(w.playing&&w.applying&&(w.applying=!1),w.started||hm(c,b,t),vm(c,b,t,d),w.applying&&(w.applying=!1),g(w.frames),w.step!=null&&w.step(t),b.completed()&&(h.splice(m,1),w.hooked=!1,w.playing=!1,w.started=!1,g(w.completes)),y=!0)}return!d&&h.length===0&&v.length===0&&a.push(c),y}for(var i=!1,s=0;s0?e.notify("draw",r):e.notify("draw")),r.unmerge(a),e.emit("step")}var pm={animate:Ve.animate(),animation:Ve.animation(),animated:Ve.animated(),clearQueue:Ve.clearQueue(),delay:Ve.delay(),delayAnimation:Ve.delayAnimation(),stop:Ve.stop(),addToAnimationPool:function(e){var r=this;r.styleEnabled()&&r._private.aniEles.merge(e)},stopAnimationLoop:function(){this._private.animationsRunning=!1},startAnimationLoop:function(){var e=this;if(e._private.animationsRunning=!0,!e.styleEnabled())return;function r(){e._private.animationsRunning&&Kn(function(i){Bc(i,e),r()})}var a=e.renderer();a&&a.beforeRender?a.beforeRender(function(i,s){Bc(s,e)},a.beforeRenderPriorities.animations):r()}},gm={qualifierCompare:function(e,r){return e==null||r==null?e==null&&r==null:e.sameText(r)},eventMatches:function(e,r,a){var n=r.qualifier;return n!=null?e!==a.target&&Ja(a.target)&&n.matches(a.target):!0},addEventFields:function(e,r){r.cy=e,r.target=e},callbackContext:function(e,r,a){return r.qualifier!=null?a.target:e}},En=function(e){return me(e)?new Br(e):e},yf={createEmitter:function(){var e=this._private;return e.emitter||(e.emitter=new yi(gm,this)),this},emitter:function(){return this._private.emitter},on:function(e,r,a){return this.emitter().on(e,En(r),a),this},removeListener:function(e,r,a){return this.emitter().removeListener(e,En(r),a),this},removeAllListeners:function(){return this.emitter().removeAllListeners(),this},one:function(e,r,a){return this.emitter().one(e,En(r),a),this},once:function(e,r,a){return this.emitter().one(e,En(r),a),this},emit:function(e,r){return this.emitter().emit(e,r),this},emitAndNotify:function(e,r){return this.emit(e),this.notify(e,r),this}};Ve.eventAliasesOn(yf);var Co={png:function(e){var r=this._private.renderer;return e=e||{},r.png(e)},jpg:function(e){var r=this._private.renderer;return e=e||{},e.bg=e.bg||"#fff",r.jpg(e)}};Co.jpeg=Co.jpg;var zn={layout:function(e){var r=this;if(e==null){Ye("Layout options must be specified to make a layout");return}if(e.name==null){Ye("A `name` must be specified to make a layout");return}var a=e.name,n=r.extension("layout",a);if(n==null){Ye("No such layout `"+a+"` found. Did you forget to import it and `cytoscape.use()` it?");return}var i;me(e.eles)?i=r.$(e.eles):i=e.eles!=null?e.eles:r.$();var s=new n(xe({},e,{cy:r,eles:i}));return s}};zn.createLayout=zn.makeLayout=zn.layout;var ym={notify:function(e,r){var a=this._private;if(this.batching()){a.batchNotifications=a.batchNotifications||{};var n=a.batchNotifications[e]=a.batchNotifications[e]||this.collection();r!=null&&n.merge(r);return}if(a.notificationsEnabled){var i=this.renderer();this.destroyed()||!i||i.notify(e,r)}},notifications:function(e){var r=this._private;return e===void 0?r.notificationsEnabled:(r.notificationsEnabled=!!e,this)},noNotifications:function(e){this.notifications(!1),e(),this.notifications(!0)},batching:function(){return this._private.batchCount>0},startBatch:function(){var e=this._private;return e.batchCount==null&&(e.batchCount=0),e.batchCount===0&&(e.batchStyleEles=this.collection(),e.batchNotifications={}),e.batchCount++,this},endBatch:function(){var e=this._private;if(e.batchCount===0)return this;if(e.batchCount--,e.batchCount===0){e.batchStyleEles.updateStyle();var r=this.renderer();Object.keys(e.batchNotifications).forEach(function(a){var n=e.batchNotifications[a];n.empty()?r.notify(a):r.notify(a,n)})}return this},batch:function(e){return this.startBatch(),e(),this.endBatch(),this},batchData:function(e){var r=this;return this.batch(function(){for(var a=Object.keys(e),n=0;n0;)r.removeChild(r.childNodes[0]);e._private.renderer=null,e.mutableElements().forEach(function(a){var n=a._private;n.rscratch={},n.rstyle={},n.animation.current=[],n.animation.queue=[]})},onRender:function(e){return this.on("render",e)},offRender:function(e){return this.off("render",e)}};To.invalidateDimensions=To.resize;var $n={collection:function(e,r){return me(e)?this.$(e):It(e)?e.collection():Ue(e)?(r||(r={}),new pt(this,e,r.unique,r.removed)):new pt(this)},nodes:function(e){var r=this.$(function(a){return a.isNode()});return e?r.filter(e):r},edges:function(e){var r=this.$(function(a){return a.isEdge()});return e?r.filter(e):r},$:function(e){var r=this._private.elements;return e?r.filter(e):r.spawnSelf()},mutableElements:function(){return this._private.elements}};$n.elements=$n.filter=$n.$;var ft={},za="t",bm="f";ft.apply=function(t){for(var e=this,r=e._private,a=r.cy,n=a.collection(),i=0;i0;if(f||d&&h){var v=void 0;f&&h||f?v=u.properties:h&&(v=u.mappedProperties);for(var y=0;y1&&(w=1),o.color){var C=a.valueMin[0],x=a.valueMax[0],T=a.valueMin[1],k=a.valueMax[1],P=a.valueMin[2],B=a.valueMax[2],D=a.valueMin[3]==null?1:a.valueMin[3],R=a.valueMax[3]==null?1:a.valueMax[3],A=[Math.round(C+(x-C)*w),Math.round(T+(k-T)*w),Math.round(P+(B-P)*w),Math.round(D+(R-D)*w)];i={bypass:a.bypass,name:a.name,value:A,strValue:"rgb("+A[0]+", "+A[1]+", "+A[2]+")"}}else if(o.number){var M=a.valueMin+(a.valueMax-a.valueMin)*w;i=this.parse(a.name,M,a.bypass,f)}else return!1;if(!i)return y(),!1;i.mapping=a,a=i;break}case s.data:{for(var _=a.field.split("."),L=d.data,I=0;I<_.length&&L;I++){var H=_[I];L=L[H]}if(L!=null&&(i=this.parse(a.name,L,a.bypass,f)),!i)return y(),!1;i.mapping=a,a=i;break}case s.fn:{var G=a.value,O=a.fnValue!=null?a.fnValue:G(t);if(a.prevFnValue=O,O==null)return He("Custom function mappers may not return null (i.e. `"+a.name+"` for ele `"+t.id()+"` is null)"),!1;if(i=this.parse(a.name,O,a.bypass,f),!i)return He("Custom function mappers may not return invalid values for the property type (i.e. `"+a.name+"` for ele `"+t.id()+"` is invalid)"),!1;i.mapping=Jt(a),a=i;break}case void 0:break;default:return!1}return l?(c?a.bypassed=u.bypassed:a.bypassed=u,n[a.name]=a):c?u.bypassed=a:n[a.name]=a,v(),!0};ft.cleanElements=function(t,e){for(var r=0;r0&&i>0){for(var o={},l=!1,u=0;u0?t.delayAnimation(s).play().promise().then(b):b()}).then(function(){return t.animation({style:o,duration:i,easing:t.pstyle("transition-timing-function").value,queue:!1}).play().promise()}).then(function(){r.removeBypasses(t,n),t.emitAndNotify("style"),a.transitioning=!1})}else a.transitioning&&(this.removeBypasses(t,n),t.emitAndNotify("style"),a.transitioning=!1)};ft.checkTrigger=function(t,e,r,a,n,i){var s=this.properties[e],o=n(s);t.removed()||o!=null&&o(r,a,t)&&i(s)};ft.checkZOrderTrigger=function(t,e,r,a){var n=this;this.checkTrigger(t,e,r,a,function(i){return i.triggersZOrder},function(){n._private.cy.notify("zorder",t)})};ft.checkBoundsTrigger=function(t,e,r,a){this.checkTrigger(t,e,r,a,function(n){return n.triggersBounds},function(n){t.dirtyCompoundBoundsCache(),t.dirtyBoundingBoxCache()})};ft.checkConnectedEdgesBoundsTrigger=function(t,e,r,a){this.checkTrigger(t,e,r,a,function(n){return n.triggersBoundsOfConnectedEdges},function(n){t.connectedEdges().forEach(function(i){i.dirtyBoundingBoxCache()})})};ft.checkParallelEdgesBoundsTrigger=function(t,e,r,a){this.checkTrigger(t,e,r,a,function(n){return n.triggersBoundsOfParallelEdges},function(n){t.parallelEdges().forEach(function(i){i.dirtyBoundingBoxCache()})})};ft.checkTriggers=function(t,e,r,a){t.dirtyStyleCache(),this.checkZOrderTrigger(t,e,r,a),this.checkBoundsTrigger(t,e,r,a),this.checkConnectedEdgesBoundsTrigger(t,e,r,a),this.checkParallelEdgesBoundsTrigger(t,e,r,a)};var on={};on.applyBypass=function(t,e,r,a){var n=this,i=[],s=!0;if(e==="*"||e==="**"){if(r!==void 0)for(var o=0;on.length?a=a.substr(n.length):a=""}function l(){i.length>s.length?i=i.substr(s.length):i=""}for(;;){var u=a.match(/^\s*$/);if(u)break;var c=a.match(/^\s*((?:.|\s)+?)\s*\{((?:.|\s)+?)\}/);if(!c){He("Halting stylesheet parsing: String stylesheet contains more to parse but no selector and block found in: "+a);break}n=c[0];var d=c[1];if(d!=="core"){var f=new Br(d);if(f.invalid){He("Skipping parsing of block: Invalid selector found in string stylesheet: "+d),o();continue}}var h=c[2],v=!1;i=h;for(var y=[];;){var p=i.match(/^\s*$/);if(p)break;var g=i.match(/^\s*(.+?)\s*:\s*(.+?)(?:\s*;|\s*$)/);if(!g){He("Skipping parsing of block: Invalid formatting of style property and value definitions found in:"+h),v=!0;break}s=g[0];var m=g[1],b=g[2],w=e.properties[m];if(!w){He("Skipping property: Invalid property name in: "+s),l();continue}var E=r.parse(m,b);if(!E){He("Skipping property: Invalid property definition in: "+s),l();continue}y.push({name:m,val:b}),l()}if(v){o();break}r.selector(d);for(var C=0;C=7&&e[0]==="d"&&(c=new RegExp(o.data.regex).exec(e))){if(r)return!1;var f=o.data;return{name:t,value:c,strValue:""+e,mapped:f,field:c[1],bypass:r}}else if(e.length>=10&&e[0]==="m"&&(d=new RegExp(o.mapData.regex).exec(e))){if(r||u.multiple)return!1;var h=o.mapData;if(!(u.color||u.number))return!1;var v=this.parse(t,d[4]);if(!v||v.mapped)return!1;var y=this.parse(t,d[5]);if(!y||y.mapped)return!1;if(v.pfValue===y.pfValue||v.strValue===y.strValue)return He("`"+t+": "+e+"` is not a valid mapper because the output range is zero; converting to `"+t+": "+v.strValue+"`"),this.parse(t,v.strValue);if(u.color){var p=v.value,g=y.value,m=p[0]===g[0]&&p[1]===g[1]&&p[2]===g[2]&&(p[3]===g[3]||(p[3]==null||p[3]===1)&&(g[3]==null||g[3]===1));if(m)return!1}return{name:t,value:d,strValue:""+e,mapped:h,field:d[1],fieldMin:parseFloat(d[2]),fieldMax:parseFloat(d[3]),valueMin:v.value,valueMax:y.value,bypass:r}}}if(u.multiple&&a!=="multiple"){var b;if(l?b=e.split(/\s+/):Ue(e)?b=e:b=[e],u.evenMultiple&&b.length%2!==0)return null;for(var w=[],E=[],C=[],x="",T=!1,k=0;k0?" ":"")+P.strValue}return u.validate&&!u.validate(w,E)?null:u.singleEnum&&T?w.length===1&&me(w[0])?{name:t,value:w[0],strValue:w[0],bypass:r}:null:{name:t,value:w,pfValue:C,strValue:x,bypass:r,units:E}}var B=function(){for(var ee=0;eeu.max||u.strictMax&&e===u.max))return null;var _={name:t,value:e,strValue:""+e+(D||""),units:D,bypass:r};return u.unitless||D!=="px"&&D!=="em"?_.pfValue=e:_.pfValue=D==="px"||!D?e:this.getEmSizeInPixels()*e,(D==="ms"||D==="s")&&(_.pfValue=D==="ms"?e:1e3*e),(D==="deg"||D==="rad")&&(_.pfValue=D==="rad"?e:gp(e)),D==="%"&&(_.pfValue=e/100),_}else if(u.propList){var L=[],I=""+e;if(I!=="none"){for(var H=I.split(/\s*,\s*|\s+/),G=0;G0&&o>0&&!isNaN(a.w)&&!isNaN(a.h)&&a.w>0&&a.h>0){l=Math.min((s-2*r)/a.w,(o-2*r)/a.h),l=l>this._private.maxZoom?this._private.maxZoom:l,l=l=a.minZoom&&(a.maxZoom=r),this},minZoom:function(e){return e===void 0?this._private.minZoom:this.zoomRange({min:e})},maxZoom:function(e){return e===void 0?this._private.maxZoom:this.zoomRange({max:e})},getZoomedViewport:function(e){var r=this._private,a=r.pan,n=r.zoom,i,s,o=!1;if(r.zoomingEnabled||(o=!0),se(e)?s=e:Ne(e)&&(s=e.level,e.position!=null?i=ci(e.position,n,a):e.renderedPosition!=null&&(i=e.renderedPosition),i!=null&&!r.panningEnabled&&(o=!0)),s=s>r.maxZoom?r.maxZoom:s,s=sr.maxZoom||!r.zoomingEnabled?s=!0:(r.zoom=l,i.push("zoom"))}if(n&&(!s||!e.cancelOnFailedZoom)&&r.panningEnabled){var u=e.pan;se(u.x)&&(r.pan.x=u.x,o=!1),se(u.y)&&(r.pan.y=u.y,o=!1),o||i.push("pan")}return i.length>0&&(i.push("viewport"),this.emit(i.join(" ")),this.notify("viewport")),this},center:function(e){var r=this.getCenterPan(e);return r&&(this._private.pan=r,this.emit("pan viewport"),this.notify("viewport")),this},getCenterPan:function(e,r){if(this._private.panningEnabled){if(me(e)){var a=e;e=this.mutableElements().filter(a)}else It(e)||(e=this.mutableElements());if(e.length!==0){var n=e.boundingBox(),i=this.width(),s=this.height();r=r===void 0?this._private.zoom:r;var o={x:(i-r*(n.x1+n.x2))/2,y:(s-r*(n.y1+n.y2))/2};return o}}},reset:function(){return!this._private.panningEnabled||!this._private.zoomingEnabled?this:(this.viewport({pan:{x:0,y:0},zoom:1}),this)},invalidateSize:function(){this._private.sizeCache=null},size:function(){var e=this._private,r=e.container,a=this;return e.sizeCache=e.sizeCache||(r?(function(){var n=a.window().getComputedStyle(r),i=function(o){return parseFloat(n.getPropertyValue(o))};return{width:r.clientWidth-i("padding-left")-i("padding-right"),height:r.clientHeight-i("padding-top")-i("padding-bottom")}})():{width:1,height:1})},width:function(){return this.size().width},height:function(){return this.size().height},extent:function(){var e=this._private.pan,r=this._private.zoom,a=this.renderedExtent(),n={x1:(a.x1-e.x)/r,x2:(a.x2-e.x)/r,y1:(a.y1-e.y)/r,y2:(a.y2-e.y)/r};return n.w=n.x2-n.x1,n.h=n.y2-n.y1,n},renderedExtent:function(){var e=this.width(),r=this.height();return{x1:0,y1:0,x2:e,y2:r,w:e,h:r}},multiClickDebounceTime:function(e){if(e)this._private.multiClickDebounceTime=e;else return this._private.multiClickDebounceTime;return this}};jr.centre=jr.center;jr.autolockNodes=jr.autolock;jr.autoungrabifyNodes=jr.autoungrabify;var Ya={data:Ve.data({field:"data",bindingEvent:"data",allowBinding:!0,allowSetting:!0,settingEvent:"data",settingTriggersEvent:!0,triggerFnName:"trigger",allowGetting:!0,updateStyle:!0}),removeData:Ve.removeData({field:"data",event:"data",triggerFnName:"trigger",triggerEvent:!0,updateStyle:!0}),scratch:Ve.data({field:"scratch",bindingEvent:"scratch",allowBinding:!0,allowSetting:!0,settingEvent:"scratch",settingTriggersEvent:!0,triggerFnName:"trigger",allowGetting:!0,updateStyle:!0}),removeScratch:Ve.removeData({field:"scratch",event:"scratch",triggerFnName:"trigger",triggerEvent:!0,updateStyle:!0})};Ya.attr=Ya.data;Ya.removeAttr=Ya.removeData;var Xa=function(e){var r=this;e=xe({},e);var a=e.container;a&&!Un(a)&&Un(a[0])&&(a=a[0]);var n=a?a._cyreg:null;n=n||{},n&&n.cy&&(n.cy.destroy(),n={});var i=n.readies=n.readies||[];a&&(a._cyreg=n),n.cy=r;var s=it!==void 0&&a!==void 0&&!e.headless,o=e;o.layout=xe({name:s?"grid":"null"},o.layout),o.renderer=xe({name:s?"canvas":"null"},o.renderer);var l=function(v,y,p){return y!==void 0?y:p!==void 0?p:v},u=this._private={container:a,ready:!1,options:o,elements:new pt(this),listeners:[],aniEles:new pt(this),data:o.data||{},scratch:{},layout:null,renderer:null,destroyed:!1,notificationsEnabled:!0,minZoom:1e-50,maxZoom:1e50,zoomingEnabled:l(!0,o.zoomingEnabled),userZoomingEnabled:l(!0,o.userZoomingEnabled),panningEnabled:l(!0,o.panningEnabled),userPanningEnabled:l(!0,o.userPanningEnabled),boxSelectionEnabled:l(!0,o.boxSelectionEnabled),autolock:l(!1,o.autolock,o.autolockNodes),autoungrabify:l(!1,o.autoungrabify,o.autoungrabifyNodes),autounselectify:l(!1,o.autounselectify),styleEnabled:o.styleEnabled===void 0?s:o.styleEnabled,zoom:se(o.zoom)?o.zoom:1,pan:{x:Ne(o.pan)&&se(o.pan.x)?o.pan.x:0,y:Ne(o.pan)&&se(o.pan.y)?o.pan.y:0},animation:{current:[],queue:[]},hasCompoundNodes:!1,multiClickDebounceTime:l(250,o.multiClickDebounceTime)};this.createEmitter(),this.selectionType(o.selectionType),this.zoomRange({min:o.minZoom,max:o.maxZoom});var c=function(v,y){var p=v.some(ch);if(p)return ba.all(v).then(y);y(v)};u.styleEnabled&&r.setStyle([]);var d=xe({},o,o.renderer);r.initRenderer(d);var f=function(v,y,p){r.notifications(!1);var g=r.mutableElements();g.length>0&&g.remove(),v!=null&&(Ne(v)||Ue(v))&&r.add(v),r.one("layoutready",function(b){r.notifications(!0),r.emit(b),r.one("load",y),r.emitAndNotify("load")}).one("layoutstop",function(){r.one("done",p),r.emit("done")});var m=xe({},r._private.options.layout);m.eles=r.elements(),r.layout(m).run()};c([o.style,o.elements],function(h){var v=h[0],y=h[1];u.styleEnabled&&r.style().append(v),f(y,function(){r.startAnimationLoop(),u.ready=!0,Ze(o.ready)&&r.on("ready",o.ready);for(var p=0;p0,o=!!t.boundingBox,l=kt(o?t.boundingBox:structuredClone(e.extent())),u;if(It(t.roots))u=t.roots;else if(Ue(t.roots)){for(var c=[],d=0;d0;){var A=R(),M=k(A,B);if(M)A.outgoers().filter(function(be){return be.isNode()&&r.has(be)}).forEach(D);else if(M===null){He("Detected double maximal shift for node `"+A.id()+"`. Bailing maximal adjustment due to cycle. Use `options.maximal: true` only on DAGs.");break}}}var _=0;if(t.avoidOverlap)for(var L=0;L0&&g[0].length<=3?Ce/2:0),Le=2*Math.PI/g[pe].length*Se;return pe===0&&g[0].length===1&&(De=1),{x:W.x+De*Math.cos(Le),y:W.y+De*Math.sin(Le)}}else{var qe=g[pe].length,ze=Math.max(qe===1?0:o?(l.w-t.padding*2-Q.w)/((t.grid?ge:qe)-1):(l.w-t.padding*2-Q.w)/((t.grid?ge:qe)+1),_),Ie={x:W.x+(Se+1-(qe+1)/2)*ze,y:W.y+(pe+1-(J+1)/2)*ie};return Ie}},Te={downward:0,leftward:90,upward:180,rightward:-90};Object.keys(Te).indexOf(t.direction)===-1&&Ye("Invalid direction '".concat(t.direction,"' specified for breadthfirst layout. Valid values are: ").concat(Object.keys(Te).join(", ")));var Ee=function(ce){return $h(Me(ce),l,Te[t.direction])};return r.nodes().layoutPositions(this,t,Ee),this};var Tm={fit:!0,padding:30,boundingBox:void 0,avoidOverlap:!0,nodeDimensionsIncludeLabels:!1,spacingFactor:void 0,radius:void 0,startAngle:3/2*Math.PI,sweep:void 0,clockwise:!0,sort:void 0,animate:!1,animationDuration:500,animationEasing:void 0,animateFilter:function(e,r){return!0},ready:void 0,stop:void 0,transform:function(e,r){return r}};function bf(t){this.options=xe({},Tm,t)}bf.prototype.run=function(){var t=this.options,e=t,r=t.cy,a=e.eles,n=e.counterclockwise!==void 0?!e.counterclockwise:e.clockwise,i=a.nodes().not(":parent");e.sort&&(i=i.sort(e.sort));for(var s=kt(e.boundingBox?e.boundingBox:{x1:0,y1:0,w:r.width(),h:r.height()}),o={x:s.x1+s.w/2,y:s.y1+s.h/2},l=e.sweep===void 0?2*Math.PI-2*Math.PI/i.length:e.sweep,u=l/Math.max(1,i.length-1),c,d=0,f=0;f1&&e.avoidOverlap){d*=1.75;var g=Math.cos(u)-Math.cos(0),m=Math.sin(u)-Math.sin(0),b=Math.sqrt(d*d/(g*g+m*m));c=Math.max(b,c)}var w=function(C,x){var T=e.startAngle+x*u*(n?1:-1),k=c*Math.cos(T),P=c*Math.sin(T),B={x:o.x+k,y:o.y+P};return B};return a.nodes().layoutPositions(this,e,w),this};var Sm={fit:!0,padding:30,startAngle:3/2*Math.PI,sweep:void 0,clockwise:!0,equidistant:!1,minNodeSpacing:10,boundingBox:void 0,avoidOverlap:!0,nodeDimensionsIncludeLabels:!1,height:void 0,width:void 0,spacingFactor:void 0,concentric:function(e){return e.degree()},levelWidth:function(e){return e.maxDegree()/4},animate:!1,animationDuration:500,animationEasing:void 0,animateFilter:function(e,r){return!0},ready:void 0,stop:void 0,transform:function(e,r){return r}};function wf(t){this.options=xe({},Sm,t)}wf.prototype.run=function(){for(var t=this.options,e=t,r=e.counterclockwise!==void 0?!e.counterclockwise:e.clockwise,a=t.cy,n=e.eles,i=n.nodes().not(":parent"),s=kt(e.boundingBox?e.boundingBox:{x1:0,y1:0,w:a.width(),h:a.height()}),o={x:s.x1+s.w/2,y:s.y1+s.h/2},l=[],u=0,c=0;c0){var E=Math.abs(m[0].value-w.value);E>=p&&(m=[],g.push(m))}m.push(w)}var C=u+e.minNodeSpacing;if(!e.avoidOverlap){var x=g.length>0&&g[0].length>1,T=Math.min(s.w,s.h)/2-C,k=T/(g.length+x?1:0);C=Math.min(C,k)}for(var P=0,B=0;B1&&e.avoidOverlap){var M=Math.cos(A)-Math.cos(0),_=Math.sin(A)-Math.sin(0),L=Math.sqrt(C*C/(M*M+_*_));P=Math.max(L,P)}D.r=P,P+=C}if(e.equidistant){for(var I=0,H=0,G=0;G=t.numIter||(Lm(a,t),a.temperature=a.temperature*t.coolingFactor,a.temperature=t.animationThreshold&&i(),Kn(c)}};c()}else{for(;u;)u=s(l),l++;Ac(a,t),o()}return this};Ei.prototype.stop=function(){return this.stopped=!0,this.thread&&this.thread.stop(),this.emit("layoutstop"),this};Ei.prototype.destroy=function(){return this.thread&&this.thread.stop(),this};var Pm=function(e,r,a){for(var n=a.eles.edges(),i=a.eles.nodes(),s=kt(a.boundingBox?a.boundingBox:{x1:0,y1:0,w:e.width(),h:e.height()}),o={isCompound:e.hasCompoundNodes(),layoutNodes:[],idToIndex:{},nodeSize:i.size(),graphSet:[],indexToGraph:[],layoutEdges:[],edgeSize:n.size(),temperature:a.initialTemp,clientWidth:s.w,clientHeight:s.h,boundingBox:s},l=a.eles.components(),u={},c=0;c0){o.graphSet.push(T);for(var c=0;cn.count?0:n.graph},xf=function(e,r,a,n){var i=n.graphSet[a];if(-10)var d=n.nodeOverlap*c,f=Math.sqrt(o*o+l*l),h=d*o/f,v=d*l/f;else var y=Jn(e,o,l),p=Jn(r,-1*o,-1*l),g=p.x-y.x,m=p.y-y.y,b=g*g+m*m,f=Math.sqrt(b),d=(e.nodeRepulsion+r.nodeRepulsion)/b,h=d*g/f,v=d*m/f;e.isLocked||(e.offsetX-=h,e.offsetY-=v),r.isLocked||(r.offsetX+=h,r.offsetY+=v)}},Im=function(e,r,a,n){if(a>0)var i=e.maxX-r.minX;else var i=r.maxX-e.minX;if(n>0)var s=e.maxY-r.minY;else var s=r.maxY-e.minY;return i>=0&&s>=0?Math.sqrt(i*i+s*s):0},Jn=function(e,r,a){var n=e.positionX,i=e.positionY,s=e.height||1,o=e.width||1,l=a/r,u=s/o,c={};return r===0&&0a?(c.x=n,c.y=i+s/2,c):0r&&-1*u<=l&&l<=u?(c.x=n-o/2,c.y=i-o*a/2/r,c):0=u)?(c.x=n+s*r/2/a,c.y=i+s/2,c):(0>a&&(l<=-1*u||l>=u)&&(c.x=n-s*r/2/a,c.y=i-s/2),c)},Om=function(e,r){for(var a=0;aa){var p=r.gravity*h/y,g=r.gravity*v/y;f.offsetX+=p,f.offsetY+=g}}}}},zm=function(e,r){var a=[],n=0,i=-1;for(a.push.apply(a,e.graphSet[0]),i+=e.graphSet[0].length;n<=i;){var s=a[n++],o=e.idToIndex[s],l=e.layoutNodes[o],u=l.children;if(0a)var i={x:a*e/n,y:a*r/n};else var i={x:e,y:r};return i},Cf=function(e,r){var a=e.parentId;if(a!=null){var n=r.layoutNodes[r.idToIndex[a]],i=!1;if((n.maxX==null||e.maxX+n.padRight>n.maxX)&&(n.maxX=e.maxX+n.padRight,i=!0),(n.minX==null||e.minX-n.padLeftn.maxY)&&(n.maxY=e.maxY+n.padBottom,i=!0),(n.minY==null||e.minY-n.padTopg&&(v+=p+r.componentSpacing,h=0,y=0,p=0)}}},qm={fit:!0,padding:30,boundingBox:void 0,avoidOverlap:!0,avoidOverlapPadding:10,nodeDimensionsIncludeLabels:!1,spacingFactor:void 0,condense:!1,rows:void 0,cols:void 0,position:function(e){},sort:void 0,animate:!1,animationDuration:500,animationEasing:void 0,animateFilter:function(e,r){return!0},ready:void 0,stop:void 0,transform:function(e,r){return r}};function Tf(t){this.options=xe({},qm,t)}Tf.prototype.run=function(){var t=this.options,e=t,r=t.cy,a=e.eles,n=a.nodes().not(":parent");e.sort&&(n=n.sort(e.sort));var i=kt(e.boundingBox?e.boundingBox:{x1:0,y1:0,w:r.width(),h:r.height()});if(i.h===0||i.w===0)a.nodes().layoutPositions(this,e,function(Y){return{x:i.x1,y:i.y1}});else{var s=n.size(),o=Math.sqrt(s*i.h/i.w),l=Math.round(o),u=Math.round(i.w/i.h*o),c=function(te){if(te==null)return Math.min(l,u);var J=Math.min(l,u);J==l?l=te:u=te},d=function(te){if(te==null)return Math.max(l,u);var J=Math.max(l,u);J==l?l=te:u=te},f=e.rows,h=e.cols!=null?e.cols:e.columns;if(f!=null&&h!=null)l=f,u=h;else if(f!=null&&h==null)l=f,u=Math.ceil(s/l);else if(f==null&&h!=null)u=h,l=Math.ceil(s/u);else if(u*l>s){var v=c(),y=d();(v-1)*y>=s?c(v-1):(y-1)*v>=s&&d(y-1)}else for(;u*l=s?d(g+1):c(p+1)}var m=i.w/u,b=i.h/l;if(e.condense&&(m=0,b=0),e.avoidOverlap)for(var w=0;w=u&&(M=0,A++)},L={},I=0;I(M=Rp(t,e,_[L],_[L+1],_[L+2],_[L+3])))return p(x,M),!0}else if(k.edgeType==="bezier"||k.edgeType==="multibezier"||k.edgeType==="self"||k.edgeType==="compound"){for(var _=k.allpts,L=0;L+5(M=Dp(t,e,_[L],_[L+1],_[L+2],_[L+3],_[L+4],_[L+5])))return p(x,M),!0}for(var I=I||T.source,H=H||T.target,G=n.getArrowWidth(P,B),O=[{name:"source",x:k.arrowStartX,y:k.arrowStartY,angle:k.srcArrowAngle},{name:"target",x:k.arrowEndX,y:k.arrowEndY,angle:k.tgtArrowAngle},{name:"mid-source",x:k.midX,y:k.midY,angle:k.midsrcArrowAngle},{name:"mid-target",x:k.midX,y:k.midY,angle:k.midtgtArrowAngle}],L=0;L0&&(g(I),g(H))}function b(x,T,k){return Rt(x,T,k)}function w(x,T){var k=x._private,P=f,B;T?B=T+"-":B="",x.boundingBox();var D=k.labelBounds[T||"main"],R=x.pstyle(B+"label").value,A=x.pstyle("text-events").strValue==="yes";if(!(!A||!R)){var M=b(k.rscratch,"labelX",T),_=b(k.rscratch,"labelY",T),L=b(k.rscratch,"labelAngle",T),I=x.pstyle(B+"text-margin-x").pfValue,H=x.pstyle(B+"text-margin-y").pfValue,G=D.x1-P-I,O=D.x2+P-I,$=D.y1-P-H,Y=D.y2+P-H;if(L){var te=Math.cos(L),J=Math.sin(L),re=function(Q,ie){return Q=Q-M,ie=ie-_,{x:Q*te-ie*J+M,y:Q*J+ie*te+_}},ne=re(G,$),oe=re(G,Y),ee=re(O,$),q=re(O,Y),K=[ne.x+I,ne.y+H,ee.x+I,ee.y+H,q.x+I,q.y+H,oe.x+I,oe.y+H];if(At(t,e,K))return p(x),!0}else if(wr(D,t,e))return p(x),!0}}for(var E=s.length-1;E>=0;E--){var C=s[E];C.isNode()?g(C)||w(C):m(C)||w(C)||w(C,"source")||w(C,"target")}return o};Zr.getAllInBox=function(t,e,r,a){var n=this.getCachedZSortedEles().interactive,i=this.cy.zoom(),s=2/i,o=[],l=Math.min(t,r),u=Math.max(t,r),c=Math.min(e,a),d=Math.max(e,a);t=l,r=u,e=c,a=d;var f=kt({x1:t,y1:e,x2:r,y2:a}),h=[{x:f.x1,y:f.y1},{x:f.x2,y:f.y1},{x:f.x2,y:f.y2},{x:f.x1,y:f.y2}],v=[[h[0],h[1]],[h[1],h[2]],[h[2],h[3]],[h[3],h[0]]];function y(Q,ie,ge){return Rt(Q,ie,ge)}function p(Q,ie){var ge=Q._private,Me=s,Te="";Q.boundingBox();var Ee=ge.labelBounds.main;if(!Ee)return null;var be=y(ge.rscratch,"labelX",ie),ce=y(ge.rscratch,"labelY",ie),ye=y(ge.rscratch,"labelAngle",ie),pe=Q.pstyle(Te+"text-margin-x").pfValue,Se=Q.pstyle(Te+"text-margin-y").pfValue,Ce=Ee.x1-Me-pe,De=Ee.x2+Me-pe,Le=Ee.y1-Me-Se,qe=Ee.y2+Me-Se;if(ye){var ze=Math.cos(ye),Ie=Math.sin(ye),Z=function(N,U){return N=N-be,U=U-ce,{x:N*ze-U*Ie+be,y:N*Ie+U*ze+ce}};return[Z(Ce,Le),Z(De,Le),Z(De,qe),Z(Ce,qe)]}else return[{x:Ce,y:Le},{x:De,y:Le},{x:De,y:qe},{x:Ce,y:qe}]}function g(Q,ie,ge,Me){function Te(Ee,be,ce){return(ce.y-Ee.y)*(be.x-Ee.x)>(be.y-Ee.y)*(ce.x-Ee.x)}return Te(Q,ge,Me)!==Te(ie,ge,Me)&&Te(Q,ie,ge)!==Te(Q,ie,Me)}for(var m=0;m0?-(Math.PI-e.ang):Math.PI+e.ang},Wm=function(e,r,a,n,i){if(e!==Oc?Nc(r,e,Zt):Km(zt,Zt),Nc(r,a,zt),_c=Zt.nx*zt.ny-Zt.ny*zt.nx,Ic=Zt.nx*zt.nx-Zt.ny*-zt.ny,nr=Math.asin(Math.max(-1,Math.min(1,_c))),Math.abs(nr)<1e-6){So=r.x,ko=r.y,Fr=aa=0;return}Hr=1,Fn=!1,Ic<0?nr<0?nr=Math.PI+nr:(nr=Math.PI-nr,Hr=-1,Fn=!0):nr>0&&(Hr=-1,Fn=!0),r.radius!==void 0?aa=r.radius:aa=n,Nr=nr/2,Cn=Math.min(Zt.len/2,zt.len/2),i?(Yt=Math.abs(Math.cos(Nr)*aa/Math.sin(Nr)),Yt>Cn?(Yt=Cn,Fr=Math.abs(Yt*Math.sin(Nr)/Math.cos(Nr))):Fr=aa):(Yt=Math.min(Cn,aa),Fr=Math.abs(Yt*Math.sin(Nr)/Math.cos(Nr))),Po=r.x+zt.nx*Yt,Bo=r.y+zt.ny*Yt,So=Po-zt.ny*Fr*Hr,ko=Bo+zt.nx*Fr*Hr,Bf=r.x+Zt.nx*Yt,Df=r.y+Zt.ny*Yt,Oc=r};function Rf(t,e){e.radius===0?t.lineTo(e.cx,e.cy):t.arc(e.cx,e.cy,e.radius,e.startAngle,e.endAngle,e.counterClockwise)}function fl(t,e,r,a){var n=arguments.length>4&&arguments[4]!==void 0?arguments[4]:!0;return a===0||e.radius===0?{cx:e.x,cy:e.y,radius:0,startX:e.x,startY:e.y,stopX:e.x,stopY:e.y,startAngle:void 0,endAngle:void 0,counterClockwise:void 0}:(Wm(t,e,r,a,n),{cx:So,cy:ko,radius:Fr,startX:Bf,startY:Df,stopX:Po,stopY:Bo,startAngle:Zt.ang+Math.PI/2*Hr,endAngle:zt.ang-Math.PI/2*Hr,counterClockwise:Fn})}var Za=.01,jm=Math.sqrt(2*Za),Tt={};Tt.findMidptPtsEtc=function(t,e){var r=e.posPts,a=e.intersectionPts,n=e.vectorNormInverse,i,s=t.pstyle("source-endpoint"),o=t.pstyle("target-endpoint"),l=s.units!=null&&o.units!=null,u=function(E,C,x,T){var k=T-C,P=x-E,B=Math.sqrt(P*P+k*k);return{x:-k/B,y:P/B}},c=t.pstyle("edge-distances").value;switch(c){case"node-position":i=r;break;case"intersection":i=a;break;case"endpoints":{if(l){var d=this.manualEndptToPx(t.source()[0],s),f=at(d,2),h=f[0],v=f[1],y=this.manualEndptToPx(t.target()[0],o),p=at(y,2),g=p[0],m=p[1],b={x1:h,y1:v,x2:g,y2:m};n=u(h,v,g,m),i=b}else He("Edge ".concat(t.id()," has edge-distances:endpoints specified without manual endpoints specified via source-endpoint and target-endpoint. Falling back on edge-distances:intersection (default).")),i=a;break}}return{midptPts:i,vectorNormInverse:n}};Tt.findHaystackPoints=function(t){for(var e=0;e0?Math.max(U-j,0):Math.min(U+j,0)},R=D(P,T),A=D(B,k),M=!1;m===u?g=Math.abs(R)>Math.abs(A)?n:a:m===l||m===o?(g=a,M=!0):(m===i||m===s)&&(g=n,M=!0);var _=g===a,L=_?A:R,I=_?B:P,H=Jo(I),G=!1;!(M&&(w||C))&&(m===o&&I<0||m===l&&I>0||m===i&&I>0||m===s&&I<0)&&(H*=-1,L=H*Math.abs(L),G=!0);var O;if(w){var $=E<0?1+E:E;O=$*L}else{var Y=E<0?L:0;O=Y+E*H}var te=function(U){return Math.abs(U)=Math.abs(L)},J=te(O),re=te(Math.abs(L)-Math.abs(O)),ne=J||re;if(ne&&!G)if(_){var oe=Math.abs(I)<=f/2,ee=Math.abs(P)<=h/2;if(oe){var q=(c.x1+c.x2)/2,K=c.y1,W=c.y2;r.segpts=[q,K,q,W]}else if(ee){var Q=(c.y1+c.y2)/2,ie=c.x1,ge=c.x2;r.segpts=[ie,Q,ge,Q]}else r.segpts=[c.x1,c.y2]}else{var Me=Math.abs(I)<=d/2,Te=Math.abs(B)<=v/2;if(Me){var Ee=(c.y1+c.y2)/2,be=c.x1,ce=c.x2;r.segpts=[be,Ee,ce,Ee]}else if(Te){var ye=(c.x1+c.x2)/2,pe=c.y1,Se=c.y2;r.segpts=[ye,pe,ye,Se]}else r.segpts=[c.x2,c.y1]}else if(_){var Ce=c.y1+O+(p?f/2*H:0),De=c.x1,Le=c.x2;r.segpts=[De,Ce,Le,Ce]}else{var qe=c.x1+O+(p?d/2*H:0),ze=c.y1,Ie=c.y2;r.segpts=[qe,ze,qe,Ie]}if(r.isRound){var Z=t.pstyle("taxi-radius").value,S=t.pstyle("radius-type").value[0]==="arc-radius";r.radii=new Array(r.segpts.length/2).fill(Z),r.isArcRadius=new Array(r.segpts.length/2).fill(S)}};Tt.tryToCorrectInvalidPoints=function(t,e){var r=t._private.rscratch;if(r.edgeType==="bezier"){var a=e.srcPos,n=e.tgtPos,i=e.srcW,s=e.srcH,o=e.tgtW,l=e.tgtH,u=e.srcShape,c=e.tgtShape,d=e.srcCornerRadius,f=e.tgtCornerRadius,h=e.srcRs,v=e.tgtRs,y=!se(r.startX)||!se(r.startY),p=!se(r.arrowStartX)||!se(r.arrowStartY),g=!se(r.endX)||!se(r.endY),m=!se(r.arrowEndX)||!se(r.arrowEndY),b=3,w=this.getArrowWidth(t.pstyle("width").pfValue,t.pstyle("arrow-scale").value)*this.arrowShapeWidth,E=b*w,C=Kr({x:r.ctrlpts[0],y:r.ctrlpts[1]},{x:r.startX,y:r.startY}),x=CI.poolIndex()){var H=L;L=I,I=H}var G=R.srcPos=L.position(),O=R.tgtPos=I.position(),$=R.srcW=L.outerWidth(),Y=R.srcH=L.outerHeight(),te=R.tgtW=I.outerWidth(),J=R.tgtH=I.outerHeight(),re=R.srcShape=r.nodeShapes[e.getNodeShape(L)],ne=R.tgtShape=r.nodeShapes[e.getNodeShape(I)],oe=R.srcCornerRadius=L.pstyle("corner-radius").value==="auto"?"auto":L.pstyle("corner-radius").pfValue,ee=R.tgtCornerRadius=I.pstyle("corner-radius").value==="auto"?"auto":I.pstyle("corner-radius").pfValue,q=R.tgtRs=I._private.rscratch,K=R.srcRs=L._private.rscratch;R.dirCounts={north:0,west:0,south:0,east:0,northwest:0,southwest:0,northeast:0,southeast:0};for(var W=0;W=jm||(Le=Math.sqrt(Math.max(De*De,Za)+Math.max(Ce*Ce,Za)));var qe=R.vector={x:De,y:Ce},ze=R.vectorNorm={x:qe.x/Le,y:qe.y/Le},Ie={x:-ze.y,y:ze.x};R.nodesOverlap=!se(Le)||ne.checkPoint(Ee[0],Ee[1],0,te,J,O.x,O.y,ee,q)||re.checkPoint(ce[0],ce[1],0,$,Y,G.x,G.y,oe,K),R.vectorNormInverse=Ie,A={nodesOverlap:R.nodesOverlap,dirCounts:R.dirCounts,calculatedIntersection:!0,hasBezier:R.hasBezier,hasUnbundled:R.hasUnbundled,eles:R.eles,srcPos:O,srcRs:q,tgtPos:G,tgtRs:K,srcW:te,srcH:J,tgtW:$,tgtH:Y,srcIntn:ye,tgtIntn:be,srcShape:ne,tgtShape:re,posPts:{x1:Se.x2,y1:Se.y2,x2:Se.x1,y2:Se.y1},intersectionPts:{x1:pe.x2,y1:pe.y2,x2:pe.x1,y2:pe.y1},vector:{x:-qe.x,y:-qe.y},vectorNorm:{x:-ze.x,y:-ze.y},vectorNormInverse:{x:-Ie.x,y:-Ie.y}}}var Z=Te?A:R;ie.nodesOverlap=Z.nodesOverlap,ie.srcIntn=Z.srcIntn,ie.tgtIntn=Z.tgtIntn,ie.isRound=ge.startsWith("round"),n&&(L.isParent()||L.isChild()||I.isParent()||I.isChild())&&(L.parents().anySame(I)||I.parents().anySame(L)||L.same(I)&&L.isParent())?e.findCompoundLoopPoints(Q,Z,W,Me):L===I?e.findLoopPoints(Q,Z,W,Me):ge.endsWith("segments")?e.findSegmentsPoints(Q,Z):ge.endsWith("taxi")?e.findTaxiPoints(Q,Z):ge==="straight"||!Me&&R.eles.length%2===1&&W===Math.floor(R.eles.length/2)?e.findStraightEdgePoints(Q):e.findBezierPoints(Q,Z,W,Me,Te),e.findEndpoints(Q),e.tryToCorrectInvalidPoints(Q,Z),e.checkForInvalidEdgeWarning(Q),e.storeAllpts(Q),e.storeEdgeProjections(Q),e.calculateArrowAngles(Q),e.recalculateEdgeLabelProjections(Q),e.calculateLabelAngles(Q)}},x=0;x0){var Ee=u,be=$r(Ee,la(s)),ce=$r(Ee,la(Te)),ye=be;if(ce2){var pe=$r(Ee,{x:Te[2],y:Te[3]});pe0){var X=c,ve=$r(X,la(s)),ae=$r(X,la(j)),le=ve;if(ae2){var de=$r(X,{x:j[2],y:j[3]});de=v||x){p={cp:w,segment:C};break}}if(p)break}var T=p.cp,k=p.segment,P=(v-g)/k.length,B=k.t1-k.t0,D=h?k.t0+B*P:k.t1-B*P;D=Ua(0,D,1),e=da(T.p0,T.p1,T.p2,D),f=Xm(T.p0,T.p1,T.p2,D);break}case"straight":case"segments":case"haystack":{for(var R=0,A,M,_,L,I=a.allpts.length,H=0;H+3=v));H+=2);var G=v-M,O=G/A;O=Ua(0,O,1),e=mp(_,L,O),f=Mf(_,L);break}}s("labelX",d,e.x),s("labelY",d,e.y),s("labelAutoAngle",d,f)}};u("source"),u("target"),this.applyLabelDimensions(t)}};tr.applyLabelDimensions=function(t){this.applyPrefixedLabelDimensions(t),t.isEdge()&&(this.applyPrefixedLabelDimensions(t,"source"),this.applyPrefixedLabelDimensions(t,"target"))};tr.applyPrefixedLabelDimensions=function(t,e){var r=t._private,a=this.getLabelText(t,e),n=Ur(a,t._private.labelDimsKey);if(Rt(r.rscratch,"prefixedLabelDimsKey",e)!==n){ir(r.rscratch,"prefixedLabelDimsKey",e,n);var i=this.calculateLabelDimensions(t,a),s=t.pstyle("line-height").pfValue,o=t.pstyle("text-wrap").strValue,l=Rt(r.rscratch,"labelWrapCachedLines",e)||[],u=o!=="wrap"?1:Math.max(l.length,1),c=i.height/u,d=c*s,f=i.width,h=i.height+(u-1)*(s-1)*c;ir(r.rstyle,"labelWidth",e,f),ir(r.rscratch,"labelWidth",e,f),ir(r.rstyle,"labelHeight",e,h),ir(r.rscratch,"labelHeight",e,h),ir(r.rscratch,"labelLineHeight",e,d)}};tr.getLabelText=function(t,e){var r=t._private,a=e?e+"-":"",n=t.pstyle(a+"label").strValue,i=t.pstyle("text-transform").value,s=function(Y,te){return te?(ir(r.rscratch,Y,e,te),te):Rt(r.rscratch,Y,e)};if(!n)return"";i=="none"||(i=="uppercase"?n=n.toUpperCase():i=="lowercase"&&(n=n.toLowerCase()));var o=t.pstyle("text-wrap").value;if(o==="wrap"){var l=s("labelKey");if(l!=null&&s("labelWrapKey")===l)return s("labelWrapCachedText");for(var u="​",c=n.split(` +`),d=t.pstyle("text-max-width").pfValue,f=t.pstyle("text-overflow-wrap").value,h=f==="anywhere",v=[],y=/[\s\u200b]+|$/g,p=0;pd){var E=g.matchAll(y),C="",x=0,T=Lt(E),k;try{for(T.s();!(k=T.n()).done;){var P=k.value,B=P[0],D=g.substring(x,P.index);x=P.index+B.length;var R=C.length===0?D:C+D+B,A=this.calculateLabelDimensions(t,R),M=A.width;M<=d?C+=D+B:(C&&v.push(C),C=D+B)}}catch($){T.e($)}finally{T.f()}C.match(/^[\s\u200b]+$/)||v.push(C)}else v.push(g)}s("labelWrapCachedLines",v),n=s("labelWrapCachedText",v.join(` +`)),s("labelWrapKey",l)}else if(o==="ellipsis"){var _=t.pstyle("text-max-width").pfValue,L="",I="…",H=!1;if(this.calculateLabelDimensions(t,n).width<_)return n;for(var G=0;G_)break;L+=n[G],G===n.length-1&&(H=!0)}return H||(L+=I),L}return n};tr.getLabelJustification=function(t){var e=t.pstyle("text-justification").strValue,r=t.pstyle("text-halign").strValue;if(e==="auto")if(t.isNode())switch(r){case"left":return"right";case"right":return"left";default:return"center"}else return"center";else return e};tr.calculateLabelDimensions=function(t,e){var r=this,a=r.cy.window(),n=a.document,i=0,s=t.pstyle("font-style").strValue,o=t.pstyle("font-size").pfValue,l=t.pstyle("font-family").strValue,u=t.pstyle("font-weight").strValue,c=this.labelCalcCanvas,d=this.labelCalcCanvasContext;if(!c){c=this.labelCalcCanvas=n.createElement("canvas"),d=this.labelCalcCanvasContext=c.getContext("2d");var f=c.style;f.position="absolute",f.left="-9999px",f.top="-9999px",f.zIndex="-1",f.visibility="hidden",f.pointerEvents="none"}d.font="".concat(s," ").concat(u," ").concat(o,"px ").concat(l);for(var h=0,v=0,y=e.split(` +`),p=0;p1&&arguments[1]!==void 0?arguments[1]:!0;if(e.merge(s),o)for(var l=0;l=t.desktopTapThreshold2}var vt=i(S);ut&&(t.hoverData.tapholdCancelled=!0);var jt=function(){var Nt=t.hoverData.dragDelta=t.hoverData.dragDelta||[];Nt.length===0?(Nt.push(Re[0]),Nt.push(Re[1])):(Nt[0]+=Re[0],Nt[1]+=Re[1])};U=!0,n(ke,["mousemove","vmousemove","tapdrag"],S,{x:ae[0],y:ae[1]});var Xe=function(Nt){return{originalEvent:S,type:Nt,position:{x:ae[0],y:ae[1]}}},ar=function(){t.data.bgActivePosistion=void 0,t.hoverData.selecting||j.emit(Xe("boxstart")),we[4]=1,t.hoverData.selecting=!0,t.redrawHint("select",!0),t.redraw()};if(t.hoverData.which===3){if(ut){var Vt=Xe("cxtdrag");he?he.emit(Vt):j.emit(Vt),t.hoverData.cxtDragged=!0,(!t.hoverData.cxtOver||ke!==t.hoverData.cxtOver)&&(t.hoverData.cxtOver&&t.hoverData.cxtOver.emit(Xe("cxtdragout")),t.hoverData.cxtOver=ke,ke&&ke.emit(Xe("cxtdragover")))}}else if(t.hoverData.dragging){if(U=!0,j.panningEnabled()&&j.userPanningEnabled()){var pr;if(t.hoverData.justStartedPan){var dn=t.hoverData.mdownPos;pr={x:(ae[0]-dn[0])*X,y:(ae[1]-dn[1])*X},t.hoverData.justStartedPan=!1}else pr={x:Re[0]*X,y:Re[1]*X};j.panBy(pr),j.emit(Xe("dragpan")),t.hoverData.dragged=!0}ae=t.projectIntoViewport(S.clientX,S.clientY)}else if(we[4]==1&&(he==null||he.pannable())){if(ut){if(!t.hoverData.dragging&&j.boxSelectionEnabled()&&(vt||!j.panningEnabled()||!j.userPanningEnabled()))ar();else if(!t.hoverData.selecting&&j.panningEnabled()&&j.userPanningEnabled()){var Or=s(he,t.hoverData.downs);Or&&(t.hoverData.dragging=!0,t.hoverData.justStartedPan=!0,we[4]=0,t.data.bgActivePosistion=la(le),t.redrawHint("select",!0),t.redraw())}he&&he.pannable()&&he.active()&&he.unactivate()}}else{if(he&&he.pannable()&&he.active()&&he.unactivate(),(!he||!he.grabbed())&&ke!=Pe&&(Pe&&n(Pe,["mouseout","tapdragout"],S,{x:ae[0],y:ae[1]}),ke&&n(ke,["mouseover","tapdragover"],S,{x:ae[0],y:ae[1]}),t.hoverData.last=ke),he)if(ut){if(j.boxSelectionEnabled()&&vt)he&&he.grabbed()&&(y(_e),he.emit(Xe("freeon")),_e.emit(Xe("free")),t.dragData.didDrag&&(he.emit(Xe("dragfreeon")),_e.emit(Xe("dragfree")))),ar();else if(he&&he.grabbed()&&t.nodeIsDraggable(he)){var Bt=!t.dragData.didDrag;Bt&&t.redrawHint("eles",!0),t.dragData.didDrag=!0,t.hoverData.draggingEles||h(_e,{inDragLayer:!0});var mt={x:0,y:0};if(se(Re[0])&&se(Re[1])&&(mt.x+=Re[0],mt.y+=Re[1],Bt)){var Dt=t.hoverData.dragDelta;Dt&&se(Dt[0])&&se(Dt[1])&&(mt.x+=Dt[0],mt.y+=Dt[1])}t.hoverData.draggingEles=!0,_e.silentShift(mt).emit(Xe("position")).emit(Xe("drag")),t.redrawHint("drag",!0),t.redraw()}}else jt();U=!0}if(we[2]=ae[0],we[3]=ae[1],U)return S.stopPropagation&&S.stopPropagation(),S.preventDefault&&S.preventDefault(),!1}},!1);var P,B,D;t.registerBinding(e,"mouseup",function(S){if(!(t.hoverData.which===1&&S.which!==1&&t.hoverData.capture)){var N=t.hoverData.capture;if(N){t.hoverData.capture=!1;var U=t.cy,j=t.projectIntoViewport(S.clientX,S.clientY),X=t.selection,ve=t.findNearestElement(j[0],j[1],!0,!1),ae=t.dragData.possibleDragElements,le=t.hoverData.down,de=i(S);t.data.bgActivePosistion&&(t.redrawHint("select",!0),t.redraw()),t.hoverData.tapholdCancelled=!0,t.data.bgActivePosistion=void 0,le&&le.unactivate();var we=function(Qe){return{originalEvent:S,type:Qe,position:{x:j[0],y:j[1]}}};if(t.hoverData.which===3){var ke=we("cxttapend");if(le?le.emit(ke):U.emit(ke),!t.hoverData.cxtDragged){var Pe=we("cxttap");le?le.emit(Pe):U.emit(Pe)}t.hoverData.cxtDragged=!1,t.hoverData.which=null}else if(t.hoverData.which===1){if(n(ve,["mouseup","tapend","vmouseup"],S,{x:j[0],y:j[1]}),!t.dragData.didDrag&&!t.hoverData.dragged&&!t.hoverData.selecting&&!t.hoverData.isOverThresholdDrag&&(n(le,["click","tap","vclick"],S,{x:j[0],y:j[1]}),B=!1,S.timeStamp-D<=U.multiClickDebounceTime()?(P&&clearTimeout(P),B=!0,D=null,n(le,["dblclick","dbltap","vdblclick"],S,{x:j[0],y:j[1]})):(P=setTimeout(function(){B||n(le,["oneclick","onetap","voneclick"],S,{x:j[0],y:j[1]})},U.multiClickDebounceTime()),D=S.timeStamp)),le==null&&!t.dragData.didDrag&&!t.hoverData.selecting&&!t.hoverData.dragged&&!i(S)&&(U.$(r).unselect(["tapunselect"]),ae.length>0&&t.redrawHint("eles",!0),t.dragData.possibleDragElements=ae=U.collection()),ve==le&&!t.dragData.didDrag&&!t.hoverData.selecting&&ve!=null&&ve._private.selectable&&(t.hoverData.dragging||(U.selectionType()==="additive"||de?ve.selected()?ve.unselect(["tapunselect"]):ve.select(["tapselect"]):de||(U.$(r).unmerge(ve).unselect(["tapunselect"]),ve.select(["tapselect"]))),t.redrawHint("eles",!0)),t.hoverData.selecting){var he=U.collection(t.getAllInBox(X[0],X[1],X[2],X[3]));t.redrawHint("select",!0),he.length>0&&t.redrawHint("eles",!0),U.emit(we("boxend"));var Re=function(Qe){return Qe.selectable()&&!Qe.selected()};U.selectionType()==="additive"||de||U.$(r).unmerge(he).unselect(),he.emit(we("box")).stdFilter(Re).select().emit(we("boxselect")),t.redraw()}if(t.hoverData.dragging&&(t.hoverData.dragging=!1,t.redrawHint("select",!0),t.redrawHint("eles",!0),t.redraw()),!X[4]){t.redrawHint("drag",!0),t.redrawHint("eles",!0);var _e=le&&le.grabbed();y(ae),_e&&(le.emit(we("freeon")),ae.emit(we("free")),t.dragData.didDrag&&(le.emit(we("dragfreeon")),ae.emit(we("dragfree"))))}}X[4]=0,t.hoverData.down=null,t.hoverData.cxtStarted=!1,t.hoverData.draggingEles=!1,t.hoverData.selecting=!1,t.hoverData.isOverThresholdDrag=!1,t.dragData.didDrag=!1,t.hoverData.dragged=!1,t.hoverData.dragDelta=[],t.hoverData.mdownPos=null,t.hoverData.mdownGPos=null,t.hoverData.which=null}}},!1);var R=[],A=4,M,_=1e5,L=function(S,N){for(var U=0;U=A){var j=R;if(M=L(j,5),!M){var X=Math.abs(j[0]);M=I(j)&&X>5}if(M)for(var ve=0;ve5&&(U=Jo(U)*5),Pe=U/-250,M&&(Pe/=_,Pe*=3),Pe=Pe*t.wheelSensitivity;var he=S.deltaMode===1;he&&(Pe*=33);var Re=ae.zoom()*Math.pow(10,Pe);S.type==="gesturechange"&&(Re=t.gestureStartZoom*S.scale),ae.zoom({level:Re,renderedPosition:{x:ke[0],y:ke[1]}}),ae.emit({type:S.type==="gesturechange"?"pinchzoom":"scrollzoom",originalEvent:S,position:{x:we[0],y:we[1]}})}}}};t.registerBinding(t.container,"wheel",H,!0),t.registerBinding(e,"scroll",function(S){t.scrollingPage=!0,clearTimeout(t.scrollingPageTimeout),t.scrollingPageTimeout=setTimeout(function(){t.scrollingPage=!1},250)},!0),t.registerBinding(t.container,"gesturestart",function(S){t.gestureStartZoom=t.cy.zoom(),t.hasTouchStarted||S.preventDefault()},!0),t.registerBinding(t.container,"gesturechange",function(Z){t.hasTouchStarted||H(Z)},!0),t.registerBinding(t.container,"mouseout",function(S){var N=t.projectIntoViewport(S.clientX,S.clientY);t.cy.emit({originalEvent:S,type:"mouseout",position:{x:N[0],y:N[1]}})},!1),t.registerBinding(t.container,"mouseover",function(S){var N=t.projectIntoViewport(S.clientX,S.clientY);t.cy.emit({originalEvent:S,type:"mouseover",position:{x:N[0],y:N[1]}})},!1);var G,O,$,Y,te,J,re,ne,oe,ee,q,K,W,Q=function(S,N,U,j){return Math.sqrt((U-S)*(U-S)+(j-N)*(j-N))},ie=function(S,N,U,j){return(U-S)*(U-S)+(j-N)*(j-N)},ge;t.registerBinding(t.container,"touchstart",ge=function(S){if(t.hasTouchStarted=!0,!!T(S)){g(),t.touchData.capture=!0,t.data.bgActivePosistion=void 0;var N=t.cy,U=t.touchData.now,j=t.touchData.earlier;if(S.touches[0]){var X=t.projectIntoViewport(S.touches[0].clientX,S.touches[0].clientY);U[0]=X[0],U[1]=X[1]}if(S.touches[1]){var X=t.projectIntoViewport(S.touches[1].clientX,S.touches[1].clientY);U[2]=X[0],U[3]=X[1]}if(S.touches[2]){var X=t.projectIntoViewport(S.touches[2].clientX,S.touches[2].clientY);U[4]=X[0],U[5]=X[1]}var ve=function(vt){return{originalEvent:S,type:vt,position:{x:U[0],y:U[1]}}};if(S.touches[1]){t.touchData.singleTouchMoved=!0,y(t.dragData.touchDragEles);var ae=t.findContainerClientCoords();oe=ae[0],ee=ae[1],q=ae[2],K=ae[3],G=S.touches[0].clientX-oe,O=S.touches[0].clientY-ee,$=S.touches[1].clientX-oe,Y=S.touches[1].clientY-ee,W=0<=G&&G<=q&&0<=$&&$<=q&&0<=O&&O<=K&&0<=Y&&Y<=K;var le=N.pan(),de=N.zoom();te=Q(G,O,$,Y),J=ie(G,O,$,Y),re=[(G+$)/2,(O+Y)/2],ne=[(re[0]-le.x)/de,(re[1]-le.y)/de];var we=200,ke=we*we;if(J=1){for(var qt=t.touchData.startPosition=[null,null,null,null,null,null],Je=0;Je=t.touchTapThreshold2}if(N&&t.touchData.cxt){S.preventDefault();var Je=S.touches[0].clientX-oe,tt=S.touches[0].clientY-ee,nt=S.touches[1].clientX-oe,vt=S.touches[1].clientY-ee,jt=ie(Je,tt,nt,vt),Xe=jt/J,ar=150,Vt=ar*ar,pr=1.5,dn=pr*pr;if(Xe>=dn||jt>=Vt){t.touchData.cxt=!1,t.data.bgActivePosistion=void 0,t.redrawHint("select",!0);var Or=de("cxttapend");t.touchData.start?(t.touchData.start.unactivate().emit(Or),t.touchData.start=null):j.emit(Or)}}if(N&&t.touchData.cxt){var Or=de("cxtdrag");t.data.bgActivePosistion=void 0,t.redrawHint("select",!0),t.touchData.start?t.touchData.start.emit(Or):j.emit(Or),t.touchData.start&&(t.touchData.start._private.grabbed=!1),t.touchData.cxtDragged=!0;var Bt=t.findNearestElement(X[0],X[1],!0,!0);(!t.touchData.cxtOver||Bt!==t.touchData.cxtOver)&&(t.touchData.cxtOver&&t.touchData.cxtOver.emit(de("cxtdragout")),t.touchData.cxtOver=Bt,Bt&&Bt.emit(de("cxtdragover")))}else if(N&&S.touches[2]&&j.boxSelectionEnabled())S.preventDefault(),t.data.bgActivePosistion=void 0,this.lastThreeTouch=+new Date,t.touchData.selecting||j.emit(de("boxstart")),t.touchData.selecting=!0,t.touchData.didSelect=!0,U[4]=1,!U||U.length===0||U[0]===void 0?(U[0]=(X[0]+X[2]+X[4])/3,U[1]=(X[1]+X[3]+X[5])/3,U[2]=(X[0]+X[2]+X[4])/3+1,U[3]=(X[1]+X[3]+X[5])/3+1):(U[2]=(X[0]+X[2]+X[4])/3,U[3]=(X[1]+X[3]+X[5])/3),t.redrawHint("select",!0),t.redraw();else if(N&&S.touches[1]&&!t.touchData.didSelect&&j.zoomingEnabled()&&j.panningEnabled()&&j.userZoomingEnabled()&&j.userPanningEnabled()){S.preventDefault(),t.data.bgActivePosistion=void 0,t.redrawHint("select",!0);var mt=t.dragData.touchDragEles;if(mt){t.redrawHint("drag",!0);for(var Dt=0;Dt0&&!t.hoverData.draggingEles&&!t.swipePanning&&t.data.bgActivePosistion!=null&&(t.data.bgActivePosistion=void 0,t.redrawHint("select",!0),t.redraw())}},!1);var Te;t.registerBinding(e,"touchcancel",Te=function(S){var N=t.touchData.start;t.touchData.capture=!1,N&&N.unactivate()});var Ee,be,ce,ye;if(t.registerBinding(e,"touchend",Ee=function(S){var N=t.touchData.start,U=t.touchData.capture;if(U)S.touches.length===0&&(t.touchData.capture=!1),S.preventDefault();else return;var j=t.selection;t.swipePanning=!1,t.hoverData.draggingEles=!1;var X=t.cy,ve=X.zoom(),ae=t.touchData.now,le=t.touchData.earlier;if(S.touches[0]){var de=t.projectIntoViewport(S.touches[0].clientX,S.touches[0].clientY);ae[0]=de[0],ae[1]=de[1]}if(S.touches[1]){var de=t.projectIntoViewport(S.touches[1].clientX,S.touches[1].clientY);ae[2]=de[0],ae[3]=de[1]}if(S.touches[2]){var de=t.projectIntoViewport(S.touches[2].clientX,S.touches[2].clientY);ae[4]=de[0],ae[5]=de[1]}var we=function(Vt){return{originalEvent:S,type:Vt,position:{x:ae[0],y:ae[1]}}};N&&N.unactivate();var ke;if(t.touchData.cxt){if(ke=we("cxttapend"),N?N.emit(ke):X.emit(ke),!t.touchData.cxtDragged){var Pe=we("cxttap");N?N.emit(Pe):X.emit(Pe)}t.touchData.start&&(t.touchData.start._private.grabbed=!1),t.touchData.cxt=!1,t.touchData.start=null,t.redraw();return}if(!S.touches[2]&&X.boxSelectionEnabled()&&t.touchData.selecting){t.touchData.selecting=!1;var he=X.collection(t.getAllInBox(j[0],j[1],j[2],j[3]));j[0]=void 0,j[1]=void 0,j[2]=void 0,j[3]=void 0,j[4]=0,t.redrawHint("select",!0),X.emit(we("boxend"));var Re=function(Vt){return Vt.selectable()&&!Vt.selected()};he.emit(we("box")).stdFilter(Re).select().emit(we("boxselect")),he.nonempty()&&t.redrawHint("eles",!0),t.redraw()}if(N?.unactivate(),S.touches[2])t.data.bgActivePosistion=void 0,t.redrawHint("select",!0);else if(!S.touches[1]){if(!S.touches[0]){if(!S.touches[0]){t.data.bgActivePosistion=void 0,t.redrawHint("select",!0);var _e=t.dragData.touchDragEles;if(N!=null){var ut=N._private.grabbed;y(_e),t.redrawHint("drag",!0),t.redrawHint("eles",!0),ut&&(N.emit(we("freeon")),_e.emit(we("free")),t.dragData.didDrag&&(N.emit(we("dragfreeon")),_e.emit(we("dragfree")))),n(N,["touchend","tapend","vmouseup","tapdragout"],S,{x:ae[0],y:ae[1]}),N.unactivate(),t.touchData.start=null}else{var Qe=t.findNearestElement(ae[0],ae[1],!0,!0);n(Qe,["touchend","tapend","vmouseup","tapdragout"],S,{x:ae[0],y:ae[1]})}var qt=t.touchData.startPosition[0]-ae[0],Je=qt*qt,tt=t.touchData.startPosition[1]-ae[1],nt=tt*tt,vt=Je+nt,jt=vt*ve*ve;t.touchData.singleTouchMoved||(N||X.$(":selected").unselect(["tapunselect"]),n(N,["tap","vclick"],S,{x:ae[0],y:ae[1]}),be=!1,S.timeStamp-ye<=X.multiClickDebounceTime()?(ce&&clearTimeout(ce),be=!0,ye=null,n(N,["dbltap","vdblclick"],S,{x:ae[0],y:ae[1]})):(ce=setTimeout(function(){be||n(N,["onetap","voneclick"],S,{x:ae[0],y:ae[1]})},X.multiClickDebounceTime()),ye=S.timeStamp)),N!=null&&!t.dragData.didDrag&&N._private.selectable&&jt"u"){var pe=[],Se=function(S){return{clientX:S.clientX,clientY:S.clientY,force:1,identifier:S.pointerId,pageX:S.pageX,pageY:S.pageY,radiusX:S.width/2,radiusY:S.height/2,screenX:S.screenX,screenY:S.screenY,target:S.target}},Ce=function(S){return{event:S,touch:Se(S)}},De=function(S){pe.push(Ce(S))},Le=function(S){for(var N=0;N0)return $[0]}return null},v=Object.keys(f),y=0;y0?h:_d(i,s,e,r,a,n,o,l)},checkPoint:function(e,r,a,n,i,s,o,l){l=l==="auto"?Pr(n,i):l;var u=2*l;if(dr(e,r,this.points,s,o,n,i-u,[0,-1],a)||dr(e,r,this.points,s,o,n-u,i,[0,-1],a))return!0;var c=n/2+2*a,d=i/2+2*a,f=[s-c,o-d,s-c,o,s+c,o,s+c,o-d];return!!(At(e,r,f)||Gr(e,r,u,u,s+n/2-l,o+i/2-l,a)||Gr(e,r,u,u,s-n/2+l,o+i/2-l,a))}}};vr.registerNodeShapes=function(){var t=this.nodeShapes={},e=this;this.generateEllipse(),this.generatePolygon("triangle",St(3,0)),this.generateRoundPolygon("round-triangle",St(3,0)),this.generatePolygon("rectangle",St(4,0)),t.square=t.rectangle,this.generateRoundRectangle(),this.generateCutRectangle(),this.generateBarrel(),this.generateBottomRoundrectangle();{var r=[0,1,1,0,0,-1,-1,0];this.generatePolygon("diamond",r),this.generateRoundPolygon("round-diamond",r)}this.generatePolygon("pentagon",St(5,0)),this.generateRoundPolygon("round-pentagon",St(5,0)),this.generatePolygon("hexagon",St(6,0)),this.generateRoundPolygon("round-hexagon",St(6,0)),this.generatePolygon("heptagon",St(7,0)),this.generateRoundPolygon("round-heptagon",St(7,0)),this.generatePolygon("octagon",St(8,0)),this.generateRoundPolygon("round-octagon",St(8,0));var a=new Array(20);{var n=po(5,0),i=po(5,Math.PI/5),s=.5*(3-Math.sqrt(5));s*=1.57;for(var o=0;o=e.deqFastCost*w)break}else if(u){if(m>=e.deqCost*h||m>=e.deqAvgCost*f)break}else if(b>=e.deqNoDrawCost*Js)break;var E=e.deq(a,p,y);if(E.length>0)for(var C=0;C0&&(e.onDeqd(a,v),!u&&e.shouldRedraw(a,v,p,y)&&i())},o=e.priority||Xo;n.beforeRender(s,o(a))}}}},e0=(function(){function t(e){var r=arguments.length>1&&arguments[1]!==void 0?arguments[1]:Wn;Ar(this,t),this.idsByKey=new or,this.keyForId=new or,this.cachesByLvl=new or,this.lvls=[],this.getKey=e,this.doesEleInvalidateKey=r}return Lr(t,[{key:"getIdsFor",value:function(r){r==null&&Ye("Can not get id list for null key");var a=this.idsByKey,n=this.idsByKey.get(r);return n||(n=new ma,a.set(r,n)),n}},{key:"addIdForKey",value:function(r,a){r!=null&&this.getIdsFor(r).add(a)}},{key:"deleteIdForKey",value:function(r,a){r!=null&&this.getIdsFor(r).delete(a)}},{key:"getNumberOfIdsForKey",value:function(r){return r==null?0:this.getIdsFor(r).size}},{key:"updateKeyMappingFor",value:function(r){var a=r.id(),n=this.keyForId.get(a),i=this.getKey(r);this.deleteIdForKey(n,a),this.addIdForKey(i,a),this.keyForId.set(a,i)}},{key:"deleteKeyMappingFor",value:function(r){var a=r.id(),n=this.keyForId.get(a);this.deleteIdForKey(n,a),this.keyForId.delete(a)}},{key:"keyHasChangedFor",value:function(r){var a=r.id(),n=this.keyForId.get(a),i=this.getKey(r);return n!==i}},{key:"isInvalid",value:function(r){return this.keyHasChangedFor(r)||this.doesEleInvalidateKey(r)}},{key:"getCachesAt",value:function(r){var a=this.cachesByLvl,n=this.lvls,i=a.get(r);return i||(i=new or,a.set(r,i),n.push(r)),i}},{key:"getCache",value:function(r,a){return this.getCachesAt(a).get(r)}},{key:"get",value:function(r,a){var n=this.getKey(r),i=this.getCache(n,a);return i!=null&&this.updateKeyMappingFor(r),i}},{key:"getForCachedKey",value:function(r,a){var n=this.keyForId.get(r.id()),i=this.getCache(n,a);return i}},{key:"hasCache",value:function(r,a){return this.getCachesAt(a).has(r)}},{key:"has",value:function(r,a){var n=this.getKey(r);return this.hasCache(n,a)}},{key:"setCache",value:function(r,a,n){n.key=r,this.getCachesAt(a).set(r,n)}},{key:"set",value:function(r,a,n){var i=this.getKey(r);this.setCache(i,a,n),this.updateKeyMappingFor(r)}},{key:"deleteCache",value:function(r,a){this.getCachesAt(a).delete(r)}},{key:"delete",value:function(r,a){var n=this.getKey(r);this.deleteCache(n,a)}},{key:"invalidateKey",value:function(r){var a=this;this.lvls.forEach(function(n){return a.deleteCache(r,n)})}},{key:"invalidate",value:function(r){var a=r.id(),n=this.keyForId.get(a);this.deleteKeyMappingFor(r);var i=this.doesEleInvalidateKey(r);return i&&this.invalidateKey(n),i||this.getNumberOfIdsForKey(n)===0}}])})(),qc=25,Tn=50,qn=-4,Do=3,Ff=7.99,t0=8,r0=1024,a0=1024,n0=1024,i0=.2,s0=.8,o0=10,l0=.15,u0=.1,c0=.9,d0=.9,f0=100,v0=1,ca={dequeue:"dequeue",downscale:"downscale",highQuality:"highQuality"},h0=gt({getKey:null,doesEleInvalidateKey:Wn,drawElement:null,getBoundingBox:null,getRotationPoint:null,getRotationOffset:null,isVisible:Pd,allowEdgeTxrCaching:!0,allowParentTxrCaching:!0}),Na=function(e,r){var a=this;a.renderer=e,a.onDequeues=[];var n=h0(r);xe(a,n),a.lookup=new e0(n.getKey,n.doesEleInvalidateKey),a.setupDequeueing()},lt=Na.prototype;lt.reasons=ca;lt.getTextureQueue=function(t){var e=this;return e.eleImgCaches=e.eleImgCaches||{},e.eleImgCaches[t]=e.eleImgCaches[t]||[]};lt.getRetiredTextureQueue=function(t){var e=this,r=e.eleImgCaches.retired=e.eleImgCaches.retired||{},a=r[t]=r[t]||[];return a};lt.getElementQueue=function(){var t=this,e=t.eleCacheQueue=t.eleCacheQueue||new nn(function(r,a){return a.reqs-r.reqs});return e};lt.getElementKeyToQueue=function(){var t=this,e=t.eleKeyToCacheQueue=t.eleKeyToCacheQueue||{};return e};lt.getElement=function(t,e,r,a,n){var i=this,s=this.renderer,o=s.cy.zoom(),l=this.lookup;if(!e||e.w===0||e.h===0||isNaN(e.w)||isNaN(e.h)||!t.visible()||t.removed()||!i.allowEdgeTxrCaching&&t.isEdge()||!i.allowParentTxrCaching&&t.isParent())return null;if(a==null&&(a=Math.ceil(Qo(o*r))),a=Ff||a>Do)return null;var u=Math.pow(2,a),c=e.h*u,d=e.w*u,f=s.eleTextBiggerThanMin(t,u);if(!this.isVisible(t,f))return null;var h=l.get(t,a);if(h&&h.invalidated&&(h.invalidated=!1,h.texture.invalidatedWidth-=h.width),h)return h;var v;if(c<=qc?v=qc:c<=Tn?v=Tn:v=Math.ceil(c/Tn)*Tn,c>n0||d>a0)return null;var y=i.getTextureQueue(v),p=y[y.length-2],g=function(){return i.recycleTexture(v,d)||i.addTexture(v,d)};p||(p=y[y.length-1]),p||(p=g()),p.width-p.usedWidtha;B--)k=i.getElement(t,e,r,B,ca.downscale);P()}else return i.queueElement(t,C.level-1),C;else{var D;if(!b&&!w&&!E)for(var R=a-1;R>=qn;R--){var A=l.get(t,R);if(A){D=A;break}}if(m(D))return i.queueElement(t,a),D;p.context.translate(p.usedWidth,0),p.context.scale(u,u),this.drawElement(p.context,t,e,f,!1),p.context.scale(1/u,1/u),p.context.translate(-p.usedWidth,0)}return h={x:p.usedWidth,texture:p,level:a,scale:u,width:d,height:c,scaledLabelShown:f},p.usedWidth+=Math.ceil(d+t0),p.eleCaches.push(h),l.set(t,a,h),i.checkTextureFullness(p),h};lt.invalidateElements=function(t){for(var e=0;e=i0*t.width&&this.retireTexture(t)};lt.checkTextureFullness=function(t){var e=this,r=e.getTextureQueue(t.height);t.usedWidth/t.width>s0&&t.fullnessChecks>=o0?kr(r,t):t.fullnessChecks++};lt.retireTexture=function(t){var e=this,r=t.height,a=e.getTextureQueue(r),n=this.lookup;kr(a,t),t.retired=!0;for(var i=t.eleCaches,s=0;s=e)return s.retired=!1,s.usedWidth=0,s.invalidatedWidth=0,s.fullnessChecks=0,Zo(s.eleCaches),s.context.setTransform(1,0,0,1,0,0),s.context.clearRect(0,0,s.width,s.height),kr(n,s),a.push(s),s}};lt.queueElement=function(t,e){var r=this,a=r.getElementQueue(),n=r.getElementKeyToQueue(),i=this.getKey(t),s=n[i];if(s)s.level=Math.max(s.level,e),s.eles.merge(t),s.reqs++,a.updateItem(s);else{var o={eles:t.spawn().merge(t),level:e,reqs:1,key:i};a.push(o),n[i]=o}};lt.dequeue=function(t){for(var e=this,r=e.getElementQueue(),a=e.getElementKeyToQueue(),n=[],i=e.lookup,s=0;s0;s++){var o=r.pop(),l=o.key,u=o.eles[0],c=i.hasCache(u,o.level);if(a[l]=null,c)continue;n.push(o);var d=e.getBoundingBox(u);e.getElement(u,d,t,o.level,ca.dequeue)}return n};lt.removeFromQueue=function(t){var e=this,r=e.getElementQueue(),a=e.getElementKeyToQueue(),n=this.getKey(t),i=a[n];i!=null&&(i.eles.length===1?(i.reqs=Yo,r.updateItem(i),r.pop(),a[n]=null):i.eles.unmerge(t))};lt.onDequeue=function(t){this.onDequeues.push(t)};lt.offDequeue=function(t){kr(this.onDequeues,t)};lt.setupDequeueing=$f.setupDequeueing({deqRedrawThreshold:f0,deqCost:l0,deqAvgCost:u0,deqNoDrawCost:c0,deqFastCost:d0,deq:function(e,r,a){return e.dequeue(r,a)},onDeqd:function(e,r){for(var a=0;a=g0||r>ti)return null}a.validateLayersElesOrdering(r,t);var l=a.layersByLevel,u=Math.pow(2,r),c=l[r]=l[r]||[],d,f=a.levelIsComplete(r,t),h,v=function(){var P=function(M){if(a.validateLayersElesOrdering(M,t),a.levelIsComplete(M,t))return h=l[M],!0},B=function(M){if(!h)for(var _=r+M;$a<=_&&_<=ti&&!P(_);_+=M);};B(1),B(-1);for(var D=c.length-1;D>=0;D--){var R=c[D];R.invalid&&kr(c,R)}};if(!f)v();else return c;var y=function(){if(!d){d=kt();for(var P=0;PHc||R>Hc)return null;var A=D*R;if(A>T0)return null;var M=a.makeLayer(d,r);if(B!=null){var _=c.indexOf(B)+1;c.splice(_,0,M)}else(P.insert===void 0||P.insert)&&c.unshift(M);return M};if(a.skipping&&!o)return null;for(var g=null,m=t.length/p0,b=!o,w=0;w=m||!Md(g.bb,E.boundingBox()))&&(g=p({insert:!0,after:g}),!g))return null;h||b?a.queueLayer(g,E):a.drawEleInLayer(g,E,r,e),g.eles.push(E),x[r]=g}return h||(b?null:c)};yt.getEleLevelForLayerLevel=function(t,e){return t};yt.drawEleInLayer=function(t,e,r,a){var n=this,i=this.renderer,s=t.context,o=e.boundingBox();o.w===0||o.h===0||!e.visible()||(r=n.getEleLevelForLayerLevel(r,a),i.setImgSmoothing(s,!1),i.drawCachedElement(s,e,null,null,r,S0),i.setImgSmoothing(s,!0))};yt.levelIsComplete=function(t,e){var r=this,a=r.layersByLevel[t];if(!a||a.length===0)return!1;for(var n=0,i=0;i0||s.invalid)return!1;n+=s.eles.length}return n===e.length};yt.validateLayersElesOrdering=function(t,e){var r=this.layersByLevel[t];if(r)for(var a=0;a0){e=!0;break}}return e};yt.invalidateElements=function(t){var e=this;t.length!==0&&(e.lastInvalidationTime=cr(),!(t.length===0||!e.haveLayers())&&e.updateElementsInLayers(t,function(a,n,i){e.invalidateLayer(a)}))};yt.invalidateLayer=function(t){if(this.lastInvalidationTime=cr(),!t.invalid){var e=t.level,r=t.eles,a=this.layersByLevel[e];kr(a,t),t.elesQueue=[],t.invalid=!0,t.replacement&&(t.replacement.invalid=!0);for(var n=0;n3&&arguments[3]!==void 0?arguments[3]:!0,n=arguments.length>4&&arguments[4]!==void 0?arguments[4]:!0,i=arguments.length>5&&arguments[5]!==void 0?arguments[5]:!0,s=this,o=e._private.rscratch;if(!(i&&!e.visible())&&!(o.badLine||o.allpts==null||isNaN(o.allpts[0]))){var l;r&&(l=r,t.translate(-l.x1,-l.y1));var u=i?e.pstyle("opacity").value:1,c=i?e.pstyle("line-opacity").value:1,d=e.pstyle("curve-style").value,f=e.pstyle("line-style").value,h=e.pstyle("width").pfValue,v=e.pstyle("line-cap").value,y=e.pstyle("line-outline-width").value,p=e.pstyle("line-outline-color").value,g=u*c,m=u*c,b=function(){var M=arguments.length>0&&arguments[0]!==void 0?arguments[0]:g;d==="straight-triangle"?(s.eleStrokeStyle(t,e,M),s.drawEdgeTrianglePath(e,t,o.allpts)):(t.lineWidth=h,t.lineCap=v,s.eleStrokeStyle(t,e,M),s.drawEdgePath(e,t,o.allpts,f),t.lineCap="butt")},w=function(){var M=arguments.length>0&&arguments[0]!==void 0?arguments[0]:g;if(t.lineWidth=h+y,t.lineCap=v,y>0)s.colorStrokeStyle(t,p[0],p[1],p[2],M);else{t.lineCap="butt";return}d==="straight-triangle"?s.drawEdgeTrianglePath(e,t,o.allpts):(s.drawEdgePath(e,t,o.allpts,f),t.lineCap="butt")},E=function(){n&&s.drawEdgeOverlay(t,e)},C=function(){n&&s.drawEdgeUnderlay(t,e)},x=function(){var M=arguments.length>0&&arguments[0]!==void 0?arguments[0]:m;s.drawArrowheads(t,e,M)},T=function(){s.drawElementText(t,e,null,a)};t.lineJoin="round";var k=e.pstyle("ghost").value==="yes";if(k){var P=e.pstyle("ghost-offset-x").pfValue,B=e.pstyle("ghost-offset-y").pfValue,D=e.pstyle("ghost-opacity").value,R=g*D;t.translate(P,B),b(R),x(R),t.translate(-P,-B)}else w();C(),b(),x(),E(),T(),r&&t.translate(l.x1,l.y1)}};var Hf=function(e){if(!["overlay","underlay"].includes(e))throw new Error("Invalid state");return function(r,a){if(a.visible()){var n=a.pstyle("".concat(e,"-opacity")).value;if(n!==0){var i=this,s=i.usePaths(),o=a._private.rscratch,l=a.pstyle("".concat(e,"-padding")).pfValue,u=2*l,c=a.pstyle("".concat(e,"-color")).value;r.lineWidth=u,o.edgeType==="self"&&!s?r.lineCap="butt":r.lineCap="round",i.colorStrokeStyle(r,c[0],c[1],c[2],n),i.drawEdgePath(a,r,o.allpts,"solid")}}}};hr.drawEdgeOverlay=Hf("overlay");hr.drawEdgeUnderlay=Hf("underlay");hr.drawEdgePath=function(t,e,r,a){var n=t._private.rscratch,i=e,s,o=!1,l=this.usePaths(),u=t.pstyle("line-dash-pattern").pfValue,c=t.pstyle("line-dash-offset").pfValue;if(l){var d=r.join("$"),f=n.pathCacheKey&&n.pathCacheKey===d;f?(s=e=n.pathCache,o=!0):(s=e=new Path2D,n.pathCacheKey=d,n.pathCache=s)}if(i.setLineDash)switch(a){case"dotted":i.setLineDash([1,1]);break;case"dashed":i.setLineDash(u),i.lineDashOffset=c;break;case"solid":i.setLineDash([]);break}if(!o&&!n.badLine)switch(e.beginPath&&e.beginPath(),e.moveTo(r[0],r[1]),n.edgeType){case"bezier":case"self":case"compound":case"multibezier":for(var h=2;h+35&&arguments[5]!==void 0?arguments[5]:!0,s=this;if(a==null){if(i&&!s.eleTextBiggerThanMin(e))return}else if(a===!1)return;if(e.isNode()){var o=e.pstyle("label");if(!o||!o.value)return;var l=s.getLabelJustification(e);t.textAlign=l,t.textBaseline="bottom"}else{var u=e.element()._private.rscratch.badLine,c=e.pstyle("label"),d=e.pstyle("source-label"),f=e.pstyle("target-label");if(u||(!c||!c.value)&&(!d||!d.value)&&(!f||!f.value))return;t.textAlign="center",t.textBaseline="bottom"}var h=!r,v;r&&(v=r,t.translate(-v.x1,-v.y1)),n==null?(s.drawText(t,e,null,h,i),e.isEdge()&&(s.drawText(t,e,"source",h,i),s.drawText(t,e,"target",h,i))):s.drawText(t,e,n,h,i),r&&t.translate(v.x1,v.y1)};Qr.getFontCache=function(t){var e;this.fontCaches=this.fontCaches||[];for(var r=0;r2&&arguments[2]!==void 0?arguments[2]:!0,a=e.pstyle("font-style").strValue,n=e.pstyle("font-size").pfValue+"px",i=e.pstyle("font-family").strValue,s=e.pstyle("font-weight").strValue,o=r?e.effectiveOpacity()*e.pstyle("text-opacity").value:1,l=e.pstyle("text-outline-opacity").value*o,u=e.pstyle("color").value,c=e.pstyle("text-outline-color").value;t.font=a+" "+s+" "+n+" "+i,t.lineJoin="round",this.colorFillStyle(t,u[0],u[1],u[2],o),this.colorStrokeStyle(t,c[0],c[1],c[2],l)};function O0(t,e,r,a,n){var i=Math.min(a,n),s=i/2,o=e+a/2,l=r+n/2;t.beginPath(),t.arc(o,l,s,0,Math.PI*2),t.closePath()}function Wc(t,e,r,a,n){var i=arguments.length>5&&arguments[5]!==void 0?arguments[5]:5,s=Math.min(i,a/2,n/2);t.beginPath(),t.moveTo(e+s,r),t.lineTo(e+a-s,r),t.quadraticCurveTo(e+a,r,e+a,r+s),t.lineTo(e+a,r+n-s),t.quadraticCurveTo(e+a,r+n,e+a-s,r+n),t.lineTo(e+s,r+n),t.quadraticCurveTo(e,r+n,e,r+n-s),t.lineTo(e,r+s),t.quadraticCurveTo(e,r,e+s,r),t.closePath()}Qr.getTextAngle=function(t,e){var r,a=t._private,n=a.rscratch,i=e?e+"-":"",s=t.pstyle(i+"text-rotation");if(s.strValue==="autorotate"){var o=Rt(n,"labelAngle",e);r=t.isEdge()?o:0}else s.strValue==="none"?r=0:r=s.pfValue;return r};Qr.drawText=function(t,e,r){var a=arguments.length>3&&arguments[3]!==void 0?arguments[3]:!0,n=arguments.length>4&&arguments[4]!==void 0?arguments[4]:!0,i=e._private,s=i.rscratch,o=n?e.effectiveOpacity():1;if(!(n&&(o===0||e.pstyle("text-opacity").value===0))){r==="main"&&(r=null);var l=Rt(s,"labelX",r),u=Rt(s,"labelY",r),c,d,f=this.getLabelText(e,r);if(f!=null&&f!==""&&!isNaN(l)&&!isNaN(u)){this.setupTextStyle(t,e,n);var h=r?r+"-":"",v=Rt(s,"labelWidth",r),y=Rt(s,"labelHeight",r),p=e.pstyle(h+"text-margin-x").pfValue,g=e.pstyle(h+"text-margin-y").pfValue,m=e.isEdge(),b=e.pstyle("text-halign").value,w=e.pstyle("text-valign").value;m&&(b="center",w="center"),l+=p,u+=g;var E;switch(a?E=this.getTextAngle(e,r):E=0,E!==0&&(c=l,d=u,t.translate(c,d),t.rotate(E),l=0,u=0),w){case"top":break;case"center":u+=y/2;break;case"bottom":u+=y;break}var C=e.pstyle("text-background-opacity").value,x=e.pstyle("text-border-opacity").value,T=e.pstyle("text-border-width").pfValue,k=e.pstyle("text-background-padding").pfValue,P=e.pstyle("text-background-shape").strValue,B=P==="round-rectangle"||P==="roundrectangle",D=P==="circle",R=2;if(C>0||T>0&&x>0){var A=t.fillStyle,M=t.strokeStyle,_=t.lineWidth,L=e.pstyle("text-background-color").value,I=e.pstyle("text-border-color").value,H=e.pstyle("text-border-style").value,G=C>0,O=T>0&&x>0,$=l-k;switch(b){case"left":$-=v;break;case"center":$-=v/2;break}var Y=u-y-k,te=v+2*k,J=y+2*k;if(G&&(t.fillStyle="rgba(".concat(L[0],",").concat(L[1],",").concat(L[2],",").concat(C*o,")")),O&&(t.strokeStyle="rgba(".concat(I[0],",").concat(I[1],",").concat(I[2],",").concat(x*o,")"),t.lineWidth=T,t.setLineDash))switch(H){case"dotted":t.setLineDash([1,1]);break;case"dashed":t.setLineDash([4,2]);break;case"double":t.lineWidth=T/4,t.setLineDash([]);break;default:t.setLineDash([]);break}if(B?(t.beginPath(),Wc(t,$,Y,te,J,R)):D?(t.beginPath(),O0(t,$,Y,te,J)):(t.beginPath(),t.rect($,Y,te,J)),G&&t.fill(),O&&t.stroke(),O&&H==="double"){var re=T/2;t.beginPath(),B?Wc(t,$+re,Y+re,te-2*re,J-2*re,R):t.rect($+re,Y+re,te-2*re,J-2*re),t.stroke()}t.fillStyle=A,t.strokeStyle=M,t.lineWidth=_,t.setLineDash&&t.setLineDash([])}var ne=2*e.pstyle("text-outline-width").pfValue;if(ne>0&&(t.lineWidth=ne),e.pstyle("text-wrap").value==="wrap"){var oe=Rt(s,"labelWrapCachedLines",r),ee=Rt(s,"labelLineHeight",r),q=v/2,K=this.getLabelJustification(e);switch(K==="auto"||(b==="left"?K==="left"?l+=-v:K==="center"&&(l+=-q):b==="center"?K==="left"?l+=-q:K==="right"&&(l+=q):b==="right"&&(K==="center"?l+=q:K==="right"&&(l+=v))),w){case"top":u-=(oe.length-1)*ee;break;case"center":case"bottom":u-=(oe.length-1)*ee;break}for(var W=0;W0&&t.strokeText(oe[W],l,u),t.fillText(oe[W],l,u),u+=ee}else ne>0&&t.strokeText(f,l,u),t.fillText(f,l,u);E!==0&&(t.rotate(-E),t.translate(-c,-d))}}};var _r={};_r.drawNode=function(t,e,r){var a=arguments.length>3&&arguments[3]!==void 0?arguments[3]:!0,n=arguments.length>4&&arguments[4]!==void 0?arguments[4]:!0,i=arguments.length>5&&arguments[5]!==void 0?arguments[5]:!0,s=this,o,l,u=e._private,c=u.rscratch,d=e.position();if(!(!se(d.x)||!se(d.y))&&!(i&&!e.visible())){var f=i?e.effectiveOpacity():1,h=s.usePaths(),v,y=!1,p=e.padding();o=e.width()+2*p,l=e.height()+2*p;var g;r&&(g=r,t.translate(-g.x1,-g.y1));for(var m=e.pstyle("background-image"),b=m.value,w=new Array(b.length),E=new Array(b.length),C=0,x=0;x0&&arguments[0]!==void 0?arguments[0]:R;s.eleFillStyle(t,e,S)},ee=function(){var S=arguments.length>0&&arguments[0]!==void 0?arguments[0]:O;s.colorStrokeStyle(t,A[0],A[1],A[2],S)},q=function(){var S=arguments.length>0&&arguments[0]!==void 0?arguments[0]:J;s.colorStrokeStyle(t,Y[0],Y[1],Y[2],S)},K=function(S,N,U,j){var X=s.nodePathCache=s.nodePathCache||[],ve=kd(U==="polygon"?U+","+j.join(","):U,""+N,""+S,""+ne),ae=X[ve],le,de=!1;return ae!=null?(le=ae,de=!0,c.pathCache=le):(le=new Path2D,X[ve]=c.pathCache=le),{path:le,cacheHit:de}},W=e.pstyle("shape").strValue,Q=e.pstyle("shape-polygon-points").pfValue;if(h){t.translate(d.x,d.y);var ie=K(o,l,W,Q);v=ie.path,y=ie.cacheHit}var ge=function(){if(!y){var S=d;h&&(S={x:0,y:0}),s.nodeShapes[s.getNodeShape(e)].draw(v||t,S.x,S.y,o,l,ne,c)}h?t.fill(v):t.fill()},Me=function(){for(var S=arguments.length>0&&arguments[0]!==void 0?arguments[0]:f,N=arguments.length>1&&arguments[1]!==void 0?arguments[1]:!0,U=u.backgrounding,j=0,X=0;X0&&arguments[0]!==void 0?arguments[0]:!1,N=arguments.length>1&&arguments[1]!==void 0?arguments[1]:f;s.hasPie(e)&&(s.drawPie(t,e,N),S&&(h||s.nodeShapes[s.getNodeShape(e)].draw(t,d.x,d.y,o,l,ne,c)))},Ee=function(){var S=arguments.length>0&&arguments[0]!==void 0?arguments[0]:!1,N=arguments.length>1&&arguments[1]!==void 0?arguments[1]:f;s.hasStripe(e)&&(t.save(),h?t.clip(c.pathCache):(s.nodeShapes[s.getNodeShape(e)].draw(t,d.x,d.y,o,l,ne,c),t.clip()),s.drawStripe(t,e,N),t.restore(),S&&(h||s.nodeShapes[s.getNodeShape(e)].draw(t,d.x,d.y,o,l,ne,c)))},be=function(){var S=arguments.length>0&&arguments[0]!==void 0?arguments[0]:f,N=(B>0?B:-B)*S,U=B>0?0:255;B!==0&&(s.colorFillStyle(t,U,U,U,N),h?t.fill(v):t.fill())},ce=function(){if(D>0){if(t.lineWidth=D,t.lineCap=L,t.lineJoin=_,t.setLineDash)switch(M){case"dotted":t.setLineDash([1,1]);break;case"dashed":t.setLineDash(H),t.lineDashOffset=G;break;case"solid":case"double":t.setLineDash([]);break}if(I!=="center"){if(t.save(),t.lineWidth*=2,I==="inside")h?t.clip(v):t.clip();else{var S=new Path2D;S.rect(-o/2-D,-l/2-D,o+2*D,l+2*D),S.addPath(v),t.clip(S,"evenodd")}h?t.stroke(v):t.stroke(),t.restore()}else h?t.stroke(v):t.stroke();if(M==="double"){t.lineWidth=D/3;var N=t.globalCompositeOperation;t.globalCompositeOperation="destination-out",h?t.stroke(v):t.stroke(),t.globalCompositeOperation=N}t.setLineDash&&t.setLineDash([])}},ye=function(){if($>0){if(t.lineWidth=$,t.lineCap="butt",t.setLineDash)switch(te){case"dotted":t.setLineDash([1,1]);break;case"dashed":t.setLineDash([4,2]);break;case"solid":case"double":t.setLineDash([]);break}var S=d;h&&(S={x:0,y:0});var N=s.getNodeShape(e),U=D;I==="inside"&&(U=0),I==="outside"&&(U*=2);var j=(o+U+($+re))/o,X=(l+U+($+re))/l,ve=o*j,ae=l*X,le=s.nodeShapes[N].points,de;if(h){var we=K(ve,ae,N,le);de=we.path}if(N==="ellipse")s.drawEllipsePath(de||t,S.x,S.y,ve,ae);else if(["round-diamond","round-heptagon","round-hexagon","round-octagon","round-pentagon","round-polygon","round-triangle","round-tag"].includes(N)){var ke=0,Pe=0,he=0;N==="round-diamond"?ke=(U+re+$)*1.4:N==="round-heptagon"?(ke=(U+re+$)*1.075,he=-(U/2+re+$)/35):N==="round-hexagon"?ke=(U+re+$)*1.12:N==="round-pentagon"?(ke=(U+re+$)*1.13,he=-(U/2+re+$)/15):N==="round-tag"?(ke=(U+re+$)*1.12,Pe=(U/2+$+re)*.07):N==="round-triangle"&&(ke=(U+re+$)*(Math.PI/2),he=-(U+re/2+$)/Math.PI),ke!==0&&(j=(o+ke)/o,ve=o*j,["round-hexagon","round-tag"].includes(N)||(X=(l+ke)/l,ae=l*X)),ne=ne==="auto"?Od(ve,ae):ne;for(var Re=ve/2,_e=ae/2,ut=ne+(U+$+re)/2,Qe=new Array(le.length/2),qt=new Array(le.length/2),Je=0;Je0){if(n=n||a.position(),i==null||s==null){var h=a.padding();i=a.width()+2*h,s=a.height()+2*h}o.colorFillStyle(r,c[0],c[1],c[2],u),o.nodeShapes[d].draw(r,n.x,n.y,i+l*2,s+l*2,f),r.fill()}}}};_r.drawNodeOverlay=Gf("overlay");_r.drawNodeUnderlay=Gf("underlay");_r.hasPie=function(t){return t=t[0],t._private.hasPie};_r.hasStripe=function(t){return t=t[0],t._private.hasStripe};_r.drawPie=function(t,e,r,a){e=e[0],a=a||e.position();var n=e.cy().style(),i=e.pstyle("pie-size"),s=e.pstyle("pie-hole"),o=e.pstyle("pie-start-angle").pfValue,l=a.x,u=a.y,c=e.width(),d=e.height(),f=Math.min(c,d)/2,h,v=0,y=this.usePaths();if(y&&(l=0,u=0),i.units==="%"?f=f*i.pfValue:i.pfValue!==void 0&&(f=i.pfValue/2),s.units==="%"?h=f*s.pfValue:s.pfValue!==void 0&&(h=s.pfValue/2),!(h>=f))for(var p=1;p<=n.pieBackgroundN;p++){var g=e.pstyle("pie-"+p+"-background-size").value,m=e.pstyle("pie-"+p+"-background-color").value,b=e.pstyle("pie-"+p+"-background-opacity").value*r,w=g/100;w+v>1&&(w=1-v);var E=1.5*Math.PI+2*Math.PI*v;E+=o;var C=2*Math.PI*w,x=E+C;g===0||v>=1||v+w>1||(h===0?(t.beginPath(),t.moveTo(l,u),t.arc(l,u,f,E,x),t.closePath()):(t.beginPath(),t.arc(l,u,f,E,x),t.arc(l,u,h,x,E,!0),t.closePath()),this.colorFillStyle(t,m[0],m[1],m[2],b),t.fill(),v+=w)}};_r.drawStripe=function(t,e,r,a){e=e[0],a=a||e.position();var n=e.cy().style(),i=a.x,s=a.y,o=e.width(),l=e.height(),u=0,c=this.usePaths();t.save();var d=e.pstyle("stripe-direction").value,f=e.pstyle("stripe-size");switch(d){case"vertical":break;case"righward":t.rotate(-Math.PI/2);break}var h=o,v=l;f.units==="%"?(h=h*f.pfValue,v=v*f.pfValue):f.pfValue!==void 0&&(h=f.pfValue,v=f.pfValue),c&&(i=0,s=0),s-=h/2,i-=v/2;for(var y=1;y<=n.stripeBackgroundN;y++){var p=e.pstyle("stripe-"+y+"-background-size").value,g=e.pstyle("stripe-"+y+"-background-color").value,m=e.pstyle("stripe-"+y+"-background-opacity").value*r,b=p/100;b+u>1&&(b=1-u),!(p===0||u>=1||u+b>1)&&(t.beginPath(),t.rect(i,s+v*u,h,v*b),t.closePath(),this.colorFillStyle(t,g[0],g[1],g[2],m),t.fill(),u+=b)}t.restore()};var Pt={},N0=100;Pt.getPixelRatio=function(){var t=this.data.contexts[0];if(this.forcedPixelRatio!=null)return this.forcedPixelRatio;var e=this.cy.window(),r=t.backingStorePixelRatio||t.webkitBackingStorePixelRatio||t.mozBackingStorePixelRatio||t.msBackingStorePixelRatio||t.oBackingStorePixelRatio||t.backingStorePixelRatio||1;return(e.devicePixelRatio||1)/r};Pt.paintCache=function(t){for(var e=this.paintCaches=this.paintCaches||[],r=!0,a,n=0;ne.minMbLowQualFrames&&(e.motionBlurPxRatio=e.mbPxRBlurry)),e.clearingMotionBlur&&(e.motionBlurPxRatio=1),e.textureDrawLastFrame&&!d&&(c[e.NODE]=!0,c[e.SELECT_BOX]=!0);var m=r.style(),b=r.zoom(),w=s!==void 0?s:b,E=r.pan(),C={x:E.x,y:E.y},x={zoom:b,pan:{x:E.x,y:E.y}},T=e.prevViewport,k=T===void 0||x.zoom!==T.zoom||x.pan.x!==T.pan.x||x.pan.y!==T.pan.y;!k&&!(y&&!v)&&(e.motionBlurPxRatio=1),o&&(C=o),w*=l,C.x*=l,C.y*=l;var P=e.getCachedZSortedEles();function B(ee,q,K,W,Q){var ie=ee.globalCompositeOperation;ee.globalCompositeOperation="destination-out",e.colorFillStyle(ee,255,255,255,e.motionBlurTransparency),ee.fillRect(q,K,W,Q),ee.globalCompositeOperation=ie}function D(ee,q){var K,W,Q,ie;!e.clearingMotionBlur&&(ee===u.bufferContexts[e.MOTIONBLUR_BUFFER_NODE]||ee===u.bufferContexts[e.MOTIONBLUR_BUFFER_DRAG])?(K={x:E.x*h,y:E.y*h},W=b*h,Q=e.canvasWidth*h,ie=e.canvasHeight*h):(K=C,W=w,Q=e.canvasWidth,ie=e.canvasHeight),ee.setTransform(1,0,0,1,0,0),q==="motionBlur"?B(ee,0,0,Q,ie):!a&&(q===void 0||q)&&ee.clearRect(0,0,Q,ie),n||(ee.translate(K.x,K.y),ee.scale(W,W)),o&&ee.translate(o.x,o.y),s&&ee.scale(s,s)}if(d||(e.textureDrawLastFrame=!1),d){if(e.textureDrawLastFrame=!0,!e.textureCache){e.textureCache={},e.textureCache.bb=r.mutableElements().boundingBox(),e.textureCache.texture=e.data.bufferCanvases[e.TEXTURE_BUFFER];var R=e.data.bufferContexts[e.TEXTURE_BUFFER];R.setTransform(1,0,0,1,0,0),R.clearRect(0,0,e.canvasWidth*e.textureMult,e.canvasHeight*e.textureMult),e.render({forcedContext:R,drawOnlyNodeLayer:!0,forcedPxRatio:l*e.textureMult});var x=e.textureCache.viewport={zoom:r.zoom(),pan:r.pan(),width:e.canvasWidth,height:e.canvasHeight};x.mpan={x:(0-x.pan.x)/x.zoom,y:(0-x.pan.y)/x.zoom}}c[e.DRAG]=!1,c[e.NODE]=!1;var A=u.contexts[e.NODE],M=e.textureCache.texture,x=e.textureCache.viewport;A.setTransform(1,0,0,1,0,0),f?B(A,0,0,x.width,x.height):A.clearRect(0,0,x.width,x.height);var _=m.core("outside-texture-bg-color").value,L=m.core("outside-texture-bg-opacity").value;e.colorFillStyle(A,_[0],_[1],_[2],L),A.fillRect(0,0,x.width,x.height);var b=r.zoom();D(A,!1),A.clearRect(x.mpan.x,x.mpan.y,x.width/x.zoom/l,x.height/x.zoom/l),A.drawImage(M,x.mpan.x,x.mpan.y,x.width/x.zoom/l,x.height/x.zoom/l)}else e.textureOnViewport&&!a&&(e.textureCache=null);var I=r.extent(),H=e.pinching||e.hoverData.dragging||e.swipePanning||e.data.wheelZooming||e.hoverData.draggingEles||e.cy.animated(),G=e.hideEdgesOnViewport&&H,O=[];if(O[e.NODE]=!c[e.NODE]&&f&&!e.clearedForMotionBlur[e.NODE]||e.clearingMotionBlur,O[e.NODE]&&(e.clearedForMotionBlur[e.NODE]=!0),O[e.DRAG]=!c[e.DRAG]&&f&&!e.clearedForMotionBlur[e.DRAG]||e.clearingMotionBlur,O[e.DRAG]&&(e.clearedForMotionBlur[e.DRAG]=!0),c[e.NODE]||n||i||O[e.NODE]){var $=f&&!O[e.NODE]&&h!==1,A=a||($?e.data.bufferContexts[e.MOTIONBLUR_BUFFER_NODE]:u.contexts[e.NODE]),Y=f&&!$?"motionBlur":void 0;D(A,Y),G?e.drawCachedNodes(A,P.nondrag,l,I):e.drawLayeredElements(A,P.nondrag,l,I),e.debug&&e.drawDebugPoints(A,P.nondrag),!n&&!f&&(c[e.NODE]=!1)}if(!i&&(c[e.DRAG]||n||O[e.DRAG])){var $=f&&!O[e.DRAG]&&h!==1,A=a||($?e.data.bufferContexts[e.MOTIONBLUR_BUFFER_DRAG]:u.contexts[e.DRAG]);D(A,f&&!$?"motionBlur":void 0),G?e.drawCachedNodes(A,P.drag,l,I):e.drawCachedElements(A,P.drag,l,I),e.debug&&e.drawDebugPoints(A,P.drag),!n&&!f&&(c[e.DRAG]=!1)}if(this.drawSelectionRectangle(t,D),f&&h!==1){var te=u.contexts[e.NODE],J=e.data.bufferCanvases[e.MOTIONBLUR_BUFFER_NODE],re=u.contexts[e.DRAG],ne=e.data.bufferCanvases[e.MOTIONBLUR_BUFFER_DRAG],oe=function(q,K,W){q.setTransform(1,0,0,1,0,0),W||!g?q.clearRect(0,0,e.canvasWidth,e.canvasHeight):B(q,0,0,e.canvasWidth,e.canvasHeight);var Q=h;q.drawImage(K,0,0,e.canvasWidth*Q,e.canvasHeight*Q,0,0,e.canvasWidth,e.canvasHeight)};(c[e.NODE]||O[e.NODE])&&(oe(te,J,O[e.NODE]),c[e.NODE]=!1),(c[e.DRAG]||O[e.DRAG])&&(oe(re,ne,O[e.DRAG]),c[e.DRAG]=!1)}e.prevViewport=x,e.clearingMotionBlur&&(e.clearingMotionBlur=!1,e.motionBlurCleared=!0,e.motionBlur=!0),f&&(e.motionBlurTimeout=setTimeout(function(){e.motionBlurTimeout=null,e.clearedForMotionBlur[e.NODE]=!1,e.clearedForMotionBlur[e.DRAG]=!1,e.motionBlur=!1,e.clearingMotionBlur=!d,e.mbFrames=0,c[e.NODE]=!0,c[e.DRAG]=!0,e.redraw()},N0)),a||r.emit("render")};var La;Pt.drawSelectionRectangle=function(t,e){var r=this,a=r.cy,n=r.data,i=a.style(),s=t.drawOnlyNodeLayer,o=t.drawAllLayers,l=n.canvasNeedsRedraw,u=t.forcedContext;if(r.showFps||!s&&l[r.SELECT_BOX]&&!o){var c=u||n.contexts[r.SELECT_BOX];if(e(c),r.selection[4]==1&&(r.hoverData.selecting||r.touchData.selecting)){var d=r.cy.zoom(),f=i.core("selection-box-border-width").value/d;c.lineWidth=f,c.fillStyle="rgba("+i.core("selection-box-color").value[0]+","+i.core("selection-box-color").value[1]+","+i.core("selection-box-color").value[2]+","+i.core("selection-box-opacity").value+")",c.fillRect(r.selection[0],r.selection[1],r.selection[2]-r.selection[0],r.selection[3]-r.selection[1]),f>0&&(c.strokeStyle="rgba("+i.core("selection-box-border-color").value[0]+","+i.core("selection-box-border-color").value[1]+","+i.core("selection-box-border-color").value[2]+","+i.core("selection-box-opacity").value+")",c.strokeRect(r.selection[0],r.selection[1],r.selection[2]-r.selection[0],r.selection[3]-r.selection[1]))}if(n.bgActivePosistion&&!r.hoverData.selecting){var d=r.cy.zoom(),h=n.bgActivePosistion;c.fillStyle="rgba("+i.core("active-bg-color").value[0]+","+i.core("active-bg-color").value[1]+","+i.core("active-bg-color").value[2]+","+i.core("active-bg-opacity").value+")",c.beginPath(),c.arc(h.x,h.y,i.core("active-bg-size").pfValue/d,0,2*Math.PI),c.fill()}var v=r.lastRedrawTime;if(r.showFps&&v){v=Math.round(v);var y=Math.round(1e3/v),p="1 frame = "+v+" ms = "+y+" fps";if(c.setTransform(1,0,0,1,0,0),c.fillStyle="rgba(255, 0, 0, 0.75)",c.strokeStyle="rgba(255, 0, 0, 0.75)",c.font="30px Arial",!La){var g=c.measureText(p);La=g.actualBoundingBoxAscent}c.fillText(p,0,La);var m=60;c.strokeRect(0,La+10,250,20),c.fillRect(0,La+10,250*Math.min(y/m,1),20)}o||(l[r.SELECT_BOX]=!1)}};function jc(t,e,r){var a=t.createShader(e);if(t.shaderSource(a,r),t.compileShader(a),!t.getShaderParameter(a,t.COMPILE_STATUS))throw new Error(t.getShaderInfoLog(a));return a}function z0(t,e,r){var a=jc(t,t.VERTEX_SHADER,e),n=jc(t,t.FRAGMENT_SHADER,r),i=t.createProgram();if(t.attachShader(i,a),t.attachShader(i,n),t.linkProgram(i),!t.getProgramParameter(i,t.LINK_STATUS))throw new Error("Could not initialize shaders");return i}function $0(t,e,r){r===void 0&&(r=e);var a=t.makeOffscreenCanvas(e,r),n=a.context=a.getContext("2d");return a.clear=function(){return n.clearRect(0,0,a.width,a.height)},a.clear(),a}function pl(t){var e=t.pixelRatio,r=t.cy.zoom(),a=t.cy.pan();return{zoom:r*e,pan:{x:a.x*e,y:a.y*e}}}function F0(t){var e=t.pixelRatio,r=t.cy.zoom();return r*e}function q0(t,e,r,a,n){var i=a*r+e.x,s=n*r+e.y;return s=Math.round(t.canvasHeight-s),[i,s]}function V0(t,e){return e.picking?!0:t.pstyle("background-fill").value!=="solid"||t.pstyle("background-image").strValue!=="none"?!1:t.pstyle("border-width").value===0||t.pstyle("border-opacity").value===0?!0:t.pstyle("border-style").value==="solid"}function H0(t,e){if(t.length!==e.length)return!1;for(var r=0;r>0&255)/255,r[1]=(t>>8&255)/255,r[2]=(t>>16&255)/255,r[3]=(t>>24&255)/255,r}function G0(t){return t[0]+(t[1]<<8)+(t[2]<<16)+(t[3]<<24)}function U0(t,e){var r=t.createTexture();return r.buffer=function(a){t.bindTexture(t.TEXTURE_2D,r),t.texParameteri(t.TEXTURE_2D,t.TEXTURE_WRAP_S,t.CLAMP_TO_EDGE),t.texParameteri(t.TEXTURE_2D,t.TEXTURE_WRAP_T,t.CLAMP_TO_EDGE),t.texParameteri(t.TEXTURE_2D,t.TEXTURE_MAG_FILTER,t.LINEAR),t.texParameteri(t.TEXTURE_2D,t.TEXTURE_MIN_FILTER,t.LINEAR_MIPMAP_NEAREST),t.pixelStorei(t.UNPACK_PREMULTIPLY_ALPHA_WEBGL,!0),t.texImage2D(t.TEXTURE_2D,0,t.RGBA,t.RGBA,t.UNSIGNED_BYTE,a),t.generateMipmap(t.TEXTURE_2D),t.bindTexture(t.TEXTURE_2D,null)},r.deleteTexture=function(){t.deleteTexture(r)},r}function Uf(t,e){switch(e){case"float":return[1,t.FLOAT,4];case"vec2":return[2,t.FLOAT,4];case"vec3":return[3,t.FLOAT,4];case"vec4":return[4,t.FLOAT,4];case"int":return[1,t.INT,4];case"ivec2":return[2,t.INT,4]}}function Kf(t,e,r){switch(e){case t.FLOAT:return new Float32Array(r);case t.INT:return new Int32Array(r)}}function K0(t,e,r,a,n,i){switch(e){case t.FLOAT:return new Float32Array(r.buffer,i*a,n);case t.INT:return new Int32Array(r.buffer,i*a,n)}}function W0(t,e,r,a){var n=Uf(t,e),i=at(n,2),s=i[0],o=i[1],l=Kf(t,o,a),u=t.createBuffer();return t.bindBuffer(t.ARRAY_BUFFER,u),t.bufferData(t.ARRAY_BUFFER,l,t.STATIC_DRAW),o===t.FLOAT?t.vertexAttribPointer(r,s,o,!1,0,0):o===t.INT&&t.vertexAttribIPointer(r,s,o,0,0),t.enableVertexAttribArray(r),t.bindBuffer(t.ARRAY_BUFFER,null),u}function Xt(t,e,r,a){var n=Uf(t,r),i=at(n,3),s=i[0],o=i[1],l=i[2],u=Kf(t,o,e*s),c=s*l,d=t.createBuffer();t.bindBuffer(t.ARRAY_BUFFER,d),t.bufferData(t.ARRAY_BUFFER,e*c,t.DYNAMIC_DRAW),t.enableVertexAttribArray(a),o===t.FLOAT?t.vertexAttribPointer(a,s,o,!1,c,0):o===t.INT&&t.vertexAttribIPointer(a,s,o,c,0),t.vertexAttribDivisor(a,1),t.bindBuffer(t.ARRAY_BUFFER,null);for(var f=new Array(e),h=0;hs&&(o=s/a,l=a*o,u=n*o),{scale:o,texW:l,texH:u}}},{key:"draw",value:function(r,a,n){var i=this;if(this.locked)throw new Error("can't draw, atlas is locked");var s=this.texSize,o=this.texRows,l=this.texHeight,u=this.getScale(a),c=u.scale,d=u.texW,f=u.texH,h=function(b,w){if(n&&w){var E=w.context,C=b.x,x=b.row,T=C,k=l*x;E.save(),E.translate(T,k),E.scale(c,c),n(E,a),E.restore()}},v=[null,null],y=function(){h(i.freePointer,i.canvas),v[0]={x:i.freePointer.x,y:i.freePointer.row*l,w:d,h:f},v[1]={x:i.freePointer.x+d,y:i.freePointer.row*l,w:0,h:f},i.freePointer.x+=d,i.freePointer.x==s&&(i.freePointer.x=0,i.freePointer.row++)},p=function(){var b=i.scratch,w=i.canvas;b.clear(),h({x:0,row:0},b);var E=s-i.freePointer.x,C=d-E,x=l;{var T=i.freePointer.x,k=i.freePointer.row*l,P=E;w.context.drawImage(b,0,0,P,x,T,k,P,x),v[0]={x:T,y:k,w:P,h:f}}{var B=E,D=(i.freePointer.row+1)*l,R=C;w&&w.context.drawImage(b,B,0,R,x,0,D,R,x),v[1]={x:0,y:D,w:R,h:f}}i.freePointer.x=C,i.freePointer.row++},g=function(){i.freePointer.x=0,i.freePointer.row++};if(this.freePointer.x+d<=s)y();else{if(this.freePointer.row>=o-1)return!1;this.freePointer.x===s?(g(),y()):this.enableWrapping?p():(g(),y())}return this.keyToLocation.set(r,v),this.needsBuffer=!0,v}},{key:"getOffsets",value:function(r){return this.keyToLocation.get(r)}},{key:"isEmpty",value:function(){return this.freePointer.x===0&&this.freePointer.row===0}},{key:"canFit",value:function(r){if(this.locked)return!1;var a=this.texSize,n=this.texRows,i=this.getScale(r),s=i.texW;return this.freePointer.x+s>a?this.freePointer.row1&&arguments[1]!==void 0?arguments[1]:{},i=n.forceRedraw,s=i===void 0?!1:i,o=n.filterEle,l=o===void 0?function(){return!0}:o,u=n.filterType,c=u===void 0?function(){return!0}:u,d=!1,f=!1,h=Lt(r),v;try{for(h.s();!(v=h.n()).done;){var y=v.value;if(l(y)){var p=Lt(this.renderTypes.values()),g;try{var m=function(){var w=g.value,E=w.type;if(c(E)){var C=a.collections.get(w.collection),x=w.getKey(y),T=Array.isArray(x)?x:[x];if(s)T.forEach(function(D){return C.markKeyForGC(D)}),f=!0;else{var k=w.getID?w.getID(y):y.id(),P=a._key(E,k),B=a.typeAndIdToKey.get(P);B!==void 0&&!H0(T,B)&&(d=!0,a.typeAndIdToKey.delete(P),B.forEach(function(D){return C.markKeyForGC(D)}))}}};for(p.s();!(g=p.n()).done;)m()}catch(b){p.e(b)}finally{p.f()}}}}catch(b){h.e(b)}finally{h.f()}return f&&(this.gc(),d=!1),d}},{key:"gc",value:function(){var r=Lt(this.collections.values()),a;try{for(r.s();!(a=r.n()).done;){var n=a.value;n.gc()}}catch(i){r.e(i)}finally{r.f()}}},{key:"getOrCreateAtlas",value:function(r,a,n,i){var s=this.renderTypes.get(a),o=this.collections.get(s.collection),l=!1,u=o.draw(i,n,function(f){s.drawClipped?(f.save(),f.beginPath(),f.rect(0,0,n.w,n.h),f.clip(),s.drawElement(f,r,n,!0,!0),f.restore()):s.drawElement(f,r,n,!0,!0),l=!0});if(l){var c=s.getID?s.getID(r):r.id(),d=this._key(a,c);this.typeAndIdToKey.has(d)?this.typeAndIdToKey.get(d).push(i):this.typeAndIdToKey.set(d,[i])}return u}},{key:"getAtlasInfo",value:function(r,a){var n=this,i=this.renderTypes.get(a),s=i.getKey(r),o=Array.isArray(s)?s:[s];return o.map(function(l){var u=i.getBoundingBox(r,l),c=n.getOrCreateAtlas(r,a,u,l),d=c.getOffsets(l),f=at(d,2),h=f[0],v=f[1];return{atlas:c,tex:h,tex1:h,tex2:v,bb:u}})}},{key:"getDebugInfo",value:function(){var r=[],a=Lt(this.collections),n;try{for(a.s();!(n=a.n()).done;){var i=at(n.value,2),s=i[0],o=i[1],l=o.getCounts(),u=l.keyCount,c=l.atlasCount;r.push({type:s,keyCount:u,atlasCount:c})}}catch(d){a.e(d)}finally{a.f()}return r}}])})(),rb=(function(){function t(e){Ar(this,t),this.globalOptions=e,this.atlasSize=e.webglTexSize,this.maxAtlasesPerBatch=e.webglTexPerBatch,this.batchAtlases=[]}return Lr(t,[{key:"getMaxAtlasesPerBatch",value:function(){return this.maxAtlasesPerBatch}},{key:"getAtlasSize",value:function(){return this.atlasSize}},{key:"getIndexArray",value:function(){return Array.from({length:this.maxAtlasesPerBatch},function(r,a){return a})}},{key:"startBatch",value:function(){this.batchAtlases=[]}},{key:"getAtlasCount",value:function(){return this.batchAtlases.length}},{key:"getAtlases",value:function(){return this.batchAtlases}},{key:"canAddToCurrentBatch",value:function(r){return this.batchAtlases.length===this.maxAtlasesPerBatch?this.batchAtlases.includes(r):!0}},{key:"getAtlasIndexForBatch",value:function(r){var a=this.batchAtlases.indexOf(r);if(a<0){if(this.batchAtlases.length===this.maxAtlasesPerBatch)throw new Error("cannot add more atlases to batch");this.batchAtlases.push(r),a=this.batchAtlases.length-1}return a}}])})(),ab=` + float circleSD(vec2 p, float r) { + return distance(vec2(0), p) - r; // signed distance + } +`,nb=` + float rectangleSD(vec2 p, vec2 b) { + vec2 d = abs(p)-b; + return distance(vec2(0),max(d,0.0)) + min(max(d.x,d.y),0.0); + } +`,ib=` + float roundRectangleSD(vec2 p, vec2 b, vec4 cr) { + cr.xy = (p.x > 0.0) ? cr.xy : cr.zw; + cr.x = (p.y > 0.0) ? cr.x : cr.y; + vec2 q = abs(p) - b + cr.x; + return min(max(q.x, q.y), 0.0) + distance(vec2(0), max(q, 0.0)) - cr.x; + } +`,sb=` + float ellipseSD(vec2 p, vec2 ab) { + p = abs( p ); // symmetry + + // find root with Newton solver + vec2 q = ab*(p-ab); + float w = (q.x1.0) ? d : -d; + } +`,Fa={SCREEN:{name:"screen",screen:!0},PICKING:{name:"picking",picking:!0}},ri={IGNORE:1,USE_BB:2},ro=0,Qc=1,Jc=2,ao=3,ia=4,Sn=5,Ma=6,_a=7,ob=(function(){function t(e,r,a){Ar(this,t),this.r=e,this.gl=r,this.maxInstances=a.webglBatchSize,this.atlasSize=a.webglTexSize,this.bgColor=a.bgColor,this.debug=a.webglDebug,this.batchDebugInfo=[],a.enableWrapping=!0,a.createTextureCanvas=$0,this.atlasManager=new tb(e,a),this.batchManager=new rb(a),this.simpleShapeOptions=new Map,this.program=this._createShaderProgram(Fa.SCREEN),this.pickingProgram=this._createShaderProgram(Fa.PICKING),this.vao=this._createVAO()}return Lr(t,[{key:"addAtlasCollection",value:function(r,a){this.atlasManager.addAtlasCollection(r,a)}},{key:"addTextureAtlasRenderType",value:function(r,a){this.atlasManager.addRenderType(r,a)}},{key:"addSimpleShapeRenderType",value:function(r,a){this.simpleShapeOptions.set(r,a)}},{key:"invalidate",value:function(r){var a=arguments.length>1&&arguments[1]!==void 0?arguments[1]:{},n=a.type,i=this.atlasManager;return n?i.invalidate(r,{filterType:function(o){return o===n},forceRedraw:!0}):i.invalidate(r)}},{key:"gc",value:function(){this.atlasManager.gc()}},{key:"_createShaderProgram",value:function(r){var a=this.gl,n=`#version 300 es + precision highp float; + + uniform mat3 uPanZoomMatrix; + uniform int uAtlasSize; + + // instanced + in vec2 aPosition; // a vertex from the unit square + + in mat3 aTransform; // used to transform verticies, eg into a bounding box + in int aVertType; // the type of thing we are rendering + + // the z-index that is output when using picking mode + in vec4 aIndex; + + // For textures + in int aAtlasId; // which shader unit/atlas to use + in vec4 aTex; // x/y/w/h of texture in atlas + + // for edges + in vec4 aPointAPointB; + in vec4 aPointCPointD; + in vec2 aLineWidth; // also used for node border width + + // simple shapes + in vec4 aCornerRadius; // for round-rectangle [top-right, bottom-right, top-left, bottom-left] + in vec4 aColor; // also used for edges + in vec4 aBorderColor; // aLineWidth is used for border width + + // output values passed to the fragment shader + out vec2 vTexCoord; + out vec4 vColor; + out vec2 vPosition; + // flat values are not interpolated + flat out int vAtlasId; + flat out int vVertType; + flat out vec2 vTopRight; + flat out vec2 vBotLeft; + flat out vec4 vCornerRadius; + flat out vec4 vBorderColor; + flat out vec2 vBorderWidth; + flat out vec4 vIndex; + + void main(void) { + int vid = gl_VertexID; + vec2 position = aPosition; // TODO make this a vec3, simplifies some code below + + if(aVertType == `.concat(ro,`) { + float texX = aTex.x; // texture coordinates + float texY = aTex.y; + float texW = aTex.z; + float texH = aTex.w; + + if(vid == 1 || vid == 2 || vid == 4) { + texX += texW; + } + if(vid == 2 || vid == 4 || vid == 5) { + texY += texH; + } + + float d = float(uAtlasSize); + vTexCoord = vec2(texX / d, texY / d); // tex coords must be between 0 and 1 + + gl_Position = vec4(uPanZoomMatrix * aTransform * vec3(position, 1.0), 1.0); + } + else if(aVertType == `).concat(ia," || aVertType == ").concat(_a,` + || aVertType == `).concat(Sn," || aVertType == ").concat(Ma,`) { // simple shapes + + // the bounding box is needed by the fragment shader + vBotLeft = (aTransform * vec3(0, 0, 1)).xy; // flat + vTopRight = (aTransform * vec3(1, 1, 1)).xy; // flat + vPosition = (aTransform * vec3(position, 1)).xy; // will be interpolated + + // calculations are done in the fragment shader, just pass these along + vColor = aColor; + vCornerRadius = aCornerRadius; + vBorderColor = aBorderColor; + vBorderWidth = aLineWidth; + + gl_Position = vec4(uPanZoomMatrix * aTransform * vec3(position, 1.0), 1.0); + } + else if(aVertType == `).concat(Qc,`) { + vec2 source = aPointAPointB.xy; + vec2 target = aPointAPointB.zw; + + // adjust the geometry so that the line is centered on the edge + position.y = position.y - 0.5; + + // stretch the unit square into a long skinny rectangle + vec2 xBasis = target - source; + vec2 yBasis = normalize(vec2(-xBasis.y, xBasis.x)); + vec2 point = source + xBasis * position.x + yBasis * aLineWidth[0] * position.y; + + gl_Position = vec4(uPanZoomMatrix * vec3(point, 1.0), 1.0); + vColor = aColor; + } + else if(aVertType == `).concat(Jc,`) { + vec2 pointA = aPointAPointB.xy; + vec2 pointB = aPointAPointB.zw; + vec2 pointC = aPointCPointD.xy; + vec2 pointD = aPointCPointD.zw; + + // adjust the geometry so that the line is centered on the edge + position.y = position.y - 0.5; + + vec2 p0, p1, p2, pos; + if(position.x == 0.0) { // The left side of the unit square + p0 = pointA; + p1 = pointB; + p2 = pointC; + pos = position; + } else { // The right side of the unit square, use same approach but flip the geometry upside down + p0 = pointD; + p1 = pointC; + p2 = pointB; + pos = vec2(0.0, -position.y); + } + + vec2 p01 = p1 - p0; + vec2 p12 = p2 - p1; + vec2 p21 = p1 - p2; + + // Find the normal vector. + vec2 tangent = normalize(normalize(p12) + normalize(p01)); + vec2 normal = vec2(-tangent.y, tangent.x); + + // Find the vector perpendicular to p0 -> p1. + vec2 p01Norm = normalize(vec2(-p01.y, p01.x)); + + // Determine the bend direction. + float sigma = sign(dot(p01 + p21, normal)); + float width = aLineWidth[0]; + + if(sign(pos.y) == -sigma) { + // This is an intersecting vertex. Adjust the position so that there's no overlap. + vec2 point = 0.5 * width * normal * -sigma / dot(normal, p01Norm); + gl_Position = vec4(uPanZoomMatrix * vec3(p1 + point, 1.0), 1.0); + } else { + // This is a non-intersecting vertex. Treat it like a mitre join. + vec2 point = 0.5 * width * normal * sigma * dot(normal, p01Norm); + gl_Position = vec4(uPanZoomMatrix * vec3(p1 + point, 1.0), 1.0); + } + + vColor = aColor; + } + else if(aVertType == `).concat(ao,` && vid < 3) { + // massage the first triangle into an edge arrow + if(vid == 0) + position = vec2(-0.15, -0.3); + if(vid == 1) + position = vec2( 0.0, 0.0); + if(vid == 2) + position = vec2( 0.15, -0.3); + + gl_Position = vec4(uPanZoomMatrix * aTransform * vec3(position, 1.0), 1.0); + vColor = aColor; + } + else { + gl_Position = vec4(2.0, 0.0, 0.0, 1.0); // discard vertex by putting it outside webgl clip space + } + + vAtlasId = aAtlasId; + vVertType = aVertType; + vIndex = aIndex; + } + `),i=this.batchManager.getIndexArray(),s=`#version 300 es + precision highp float; + + // declare texture unit for each texture atlas in the batch + `.concat(i.map(function(u){return"uniform sampler2D uTexture".concat(u,";")}).join(` + `),` + + uniform vec4 uBGColor; + uniform float uZoom; + + in vec2 vTexCoord; + in vec4 vColor; + in vec2 vPosition; // model coordinates + + flat in int vAtlasId; + flat in vec4 vIndex; + flat in int vVertType; + flat in vec2 vTopRight; + flat in vec2 vBotLeft; + flat in vec4 vCornerRadius; + flat in vec4 vBorderColor; + flat in vec2 vBorderWidth; + + out vec4 outColor; + + `).concat(ab,` + `).concat(nb,` + `).concat(ib,` + `).concat(sb,` + + vec4 blend(vec4 top, vec4 bot) { // blend colors with premultiplied alpha + return vec4( + top.rgb + (bot.rgb * (1.0 - top.a)), + top.a + (bot.a * (1.0 - top.a)) + ); + } + + vec4 distInterp(vec4 cA, vec4 cB, float d) { // interpolate color using Signed Distance + // scale to the zoom level so that borders don't look blurry when zoomed in + // note 1.5 is an aribitrary value chosen because it looks good + return mix(cA, cB, 1.0 - smoothstep(0.0, 1.5 / uZoom, abs(d))); + } + + void main(void) { + if(vVertType == `).concat(ro,`) { + // look up the texel from the texture unit + `).concat(i.map(function(u){return"if(vAtlasId == ".concat(u,") outColor = texture(uTexture").concat(u,", vTexCoord);")}).join(` + else `),` + } + else if(vVertType == `).concat(ao,`) { + // mimics how canvas renderer uses context.globalCompositeOperation = 'destination-out'; + outColor = blend(vColor, uBGColor); + outColor.a = 1.0; // make opaque, masks out line under arrow + } + else if(vVertType == `).concat(ia,` && vBorderWidth == vec2(0.0)) { // simple rectangle with no border + outColor = vColor; // unit square is already transformed to the rectangle, nothing else needs to be done + } + else if(vVertType == `).concat(ia," || vVertType == ").concat(_a,` + || vVertType == `).concat(Sn," || vVertType == ").concat(Ma,`) { // use SDF + + float outerBorder = vBorderWidth[0]; + float innerBorder = vBorderWidth[1]; + float borderPadding = outerBorder * 2.0; + float w = vTopRight.x - vBotLeft.x - borderPadding; + float h = vTopRight.y - vBotLeft.y - borderPadding; + vec2 b = vec2(w/2.0, h/2.0); // half width, half height + vec2 p = vPosition - vec2(vTopRight.x - b[0] - outerBorder, vTopRight.y - b[1] - outerBorder); // translate to center + + float d; // signed distance + if(vVertType == `).concat(ia,`) { + d = rectangleSD(p, b); + } else if(vVertType == `).concat(_a,` && w == h) { + d = circleSD(p, b.x); // faster than ellipse + } else if(vVertType == `).concat(_a,`) { + d = ellipseSD(p, b); + } else { + d = roundRectangleSD(p, b, vCornerRadius.wzyx); + } + + // use the distance to interpolate a color to smooth the edges of the shape, doesn't need multisampling + // we must smooth colors inwards, because we can't change pixels outside the shape's bounding box + if(d > 0.0) { + if(d > outerBorder) { + discard; + } else { + outColor = distInterp(vBorderColor, vec4(0), d - outerBorder); + } + } else { + if(d > innerBorder) { + vec4 outerColor = outerBorder == 0.0 ? vec4(0) : vBorderColor; + vec4 innerBorderColor = blend(vBorderColor, vColor); + outColor = distInterp(innerBorderColor, outerColor, d); + } + else { + vec4 outerColor; + if(innerBorder == 0.0 && outerBorder == 0.0) { + outerColor = vec4(0); + } else if(innerBorder == 0.0) { + outerColor = vBorderColor; + } else { + outerColor = blend(vBorderColor, vColor); + } + outColor = distInterp(vColor, outerColor, d - innerBorder); + } + } + } + else { + outColor = vColor; + } + + `).concat(r.picking?`if(outColor.a == 0.0) discard; + else outColor = vIndex;`:"",` + } + `),o=z0(a,n,s);o.aPosition=a.getAttribLocation(o,"aPosition"),o.aIndex=a.getAttribLocation(o,"aIndex"),o.aVertType=a.getAttribLocation(o,"aVertType"),o.aTransform=a.getAttribLocation(o,"aTransform"),o.aAtlasId=a.getAttribLocation(o,"aAtlasId"),o.aTex=a.getAttribLocation(o,"aTex"),o.aPointAPointB=a.getAttribLocation(o,"aPointAPointB"),o.aPointCPointD=a.getAttribLocation(o,"aPointCPointD"),o.aLineWidth=a.getAttribLocation(o,"aLineWidth"),o.aColor=a.getAttribLocation(o,"aColor"),o.aCornerRadius=a.getAttribLocation(o,"aCornerRadius"),o.aBorderColor=a.getAttribLocation(o,"aBorderColor"),o.uPanZoomMatrix=a.getUniformLocation(o,"uPanZoomMatrix"),o.uAtlasSize=a.getUniformLocation(o,"uAtlasSize"),o.uBGColor=a.getUniformLocation(o,"uBGColor"),o.uZoom=a.getUniformLocation(o,"uZoom"),o.uTextures=[];for(var l=0;l1&&arguments[1]!==void 0?arguments[1]:Fa.SCREEN;this.panZoomMatrix=r,this.renderTarget=a,this.batchDebugInfo=[],this.wrappedCount=0,this.simpleCount=0,this.startBatch()}},{key:"startBatch",value:function(){this.instanceCount=0,this.batchManager.startBatch()}},{key:"endFrame",value:function(){this.endBatch()}},{key:"_isVisible",value:function(r,a){return r.visible()?a&&a.isVisible?a.isVisible(r):!0:!1}},{key:"drawTexture",value:function(r,a,n){var i=this.atlasManager,s=this.batchManager,o=i.getRenderTypeOpts(n);if(this._isVisible(r,o)&&!(r.isEdge()&&!this._isValidEdge(r))){if(this.renderTarget.picking&&o.getTexPickingMode){var l=o.getTexPickingMode(r);if(l===ri.IGNORE)return;if(l==ri.USE_BB){this.drawPickingRectangle(r,a,n);return}}var u=i.getAtlasInfo(r,n),c=Lt(u),d;try{for(c.s();!(d=c.n()).done;){var f=d.value,h=f.atlas,v=f.tex1,y=f.tex2;s.canAddToCurrentBatch(h)||this.endBatch();for(var p=s.getAtlasIndexForBatch(h),g=0,m=[[v,!0],[y,!1]];g=this.maxInstances&&this.endBatch()}}}}catch(B){c.e(B)}finally{c.f()}}}},{key:"setTransformMatrix",value:function(r,a,n,i){var s=arguments.length>4&&arguments[4]!==void 0?arguments[4]:!0,o=0;if(n.shapeProps&&n.shapeProps.padding&&(o=r.pstyle(n.shapeProps.padding).pfValue),i){var l=i.bb,u=i.tex1,c=i.tex2,d=u.w/(u.w+c.w);s||(d=1-d);var f=this._getAdjustedBB(l,o,s,d);this._applyTransformMatrix(a,f,n,r)}else{var h=n.getBoundingBox(r),v=this._getAdjustedBB(h,o,!0,1);this._applyTransformMatrix(a,v,n,r)}}},{key:"_applyTransformMatrix",value:function(r,a,n,i){var s,o;Xc(r);var l=n.getRotation?n.getRotation(i):0;if(l!==0){var u=n.getRotationPoint(i),c=u.x,d=u.y;Vn(r,r,[c,d]),Zc(r,r,l);var f=n.getRotationOffset(i);s=f.x+(a.xOffset||0),o=f.y+(a.yOffset||0)}else s=a.x1,o=a.y1;Vn(r,r,[s,o]),Ro(r,r,[a.w,a.h])}},{key:"_getAdjustedBB",value:function(r,a,n,i){var s=r.x1,o=r.y1,l=r.w,u=r.h,c=r.yOffset;a&&(s-=a,o-=a,l+=2*a,u+=2*a);var d=0,f=l*i;return n&&i<1?l=f:!n&&i<1&&(d=l-f,s+=d,l=f),{x1:s,y1:o,w:l,h:u,xOffset:d,yOffset:c}}},{key:"drawPickingRectangle",value:function(r,a,n){var i=this.atlasManager.getRenderTypeOpts(n),s=this.instanceCount;this.vertTypeBuffer.getView(s)[0]=ia;var o=this.indexBuffer.getView(s);na(a,o);var l=this.colorBuffer.getView(s);zr([0,0,0],1,l);var u=this.transformBuffer.getMatrixView(s);this.setTransformMatrix(r,u,i),this.simpleCount++,this.instanceCount++,this.instanceCount>=this.maxInstances&&this.endBatch()}},{key:"drawNode",value:function(r,a,n){var i=this.simpleShapeOptions.get(n);if(this._isVisible(r,i)){var s=i.shapeProps,o=this._getVertTypeForShape(r,s.shape);if(o===void 0||i.isSimple&&!i.isSimple(r,this.renderTarget)){this.drawTexture(r,a,n);return}var l=this.instanceCount;if(this.vertTypeBuffer.getView(l)[0]=o,o===Sn||o===Ma){var u=i.getBoundingBox(r),c=this._getCornerRadius(r,s.radius,u),d=this.cornerRadiusBuffer.getView(l);d[0]=c,d[1]=c,d[2]=c,d[3]=c,o===Ma&&(d[0]=0,d[2]=0)}var f=this.indexBuffer.getView(l);na(a,f);var h=this.renderTarget.picking?1:n==="node-body"?r.effectiveOpacity():1,v=this.renderTarget.picking?1:r.pstyle(s.opacity).value*h,y=r.pstyle(s.color).value,p=this.colorBuffer.getView(l);zr(y,v,p);var g=this.lineWidthBuffer.getView(l);if(g[0]=0,g[1]=0,s.border){var m=r.pstyle("border-width").value;if(m>0){var b=r.pstyle("border-color").value,w=h*r.pstyle("border-opacity").value,E=this.borderColorBuffer.getView(l);zr(b,w,E);var C=r.pstyle("border-position").value;if(C==="inside")g[0]=0,g[1]=-m;else if(C==="outside")g[0]=m,g[1]=0;else{var x=m/2;g[0]=x,g[1]=-x}}}var T=this.transformBuffer.getMatrixView(l);this.setTransformMatrix(r,T,i),this.simpleCount++,this.instanceCount++,this.instanceCount>=this.maxInstances&&this.endBatch()}}},{key:"_getVertTypeForShape",value:function(r,a){var n=r.pstyle(a).value;switch(n){case"rectangle":return ia;case"ellipse":return _a;case"roundrectangle":case"round-rectangle":return Sn;case"bottom-round-rectangle":return Ma;default:return}}},{key:"_getCornerRadius",value:function(r,a,n){var i=n.w,s=n.h;if(r.pstyle(a).value==="auto")return Pr(i,s);var o=r.pstyle(a).pfValue,l=i/2,u=s/2;return Math.min(o,u,l)}},{key:"drawEdgeArrow",value:function(r,a,n){if(r.visible()){var i=r._private.rscratch,s,o,l;if(n==="source"?(s=i.arrowStartX,o=i.arrowStartY,l=i.srcArrowAngle):(s=i.arrowEndX,o=i.arrowEndY,l=i.tgtArrowAngle),!(isNaN(s)||s==null||isNaN(o)||o==null||isNaN(l)||l==null)){var u=r.pstyle(n+"-arrow-shape").value;if(u!=="none"){var c=r.pstyle(n+"-arrow-color").value,d=r.pstyle("opacity").value,f=r.pstyle("line-opacity").value,h=d*f,v=r.pstyle("width").pfValue,y=r.pstyle("arrow-scale").value,p=this.r.getArrowWidth(v,y),g=this.instanceCount,m=this.transformBuffer.getMatrixView(g);Xc(m),Vn(m,m,[s,o]),Ro(m,m,[p,p]),Zc(m,m,l),this.vertTypeBuffer.getView(g)[0]=ao;var b=this.indexBuffer.getView(g);na(a,b);var w=this.colorBuffer.getView(g);zr(c,h,w),this.instanceCount++,this.instanceCount>=this.maxInstances&&this.endBatch()}}}}},{key:"drawEdgeLine",value:function(r,a){if(r.visible()){var n=this._getEdgePoints(r);if(n){var i=r.pstyle("opacity").value,s=r.pstyle("line-opacity").value,o=r.pstyle("width").pfValue,l=r.pstyle("line-color").value,u=i*s;if(n.length/2+this.instanceCount>this.maxInstances&&this.endBatch(),n.length==4){var c=this.instanceCount;this.vertTypeBuffer.getView(c)[0]=Qc;var d=this.indexBuffer.getView(c);na(a,d);var f=this.colorBuffer.getView(c);zr(l,u,f);var h=this.lineWidthBuffer.getView(c);h[0]=o;var v=this.pointAPointBBuffer.getView(c);v[0]=n[0],v[1]=n[1],v[2]=n[2],v[3]=n[3],this.instanceCount++,this.instanceCount>=this.maxInstances&&this.endBatch()}else for(var y=0;y=this.maxInstances&&this.endBatch()}}}}},{key:"_isValidEdge",value:function(r){var a=r._private.rscratch;return!(a.badLine||a.allpts==null||isNaN(a.allpts[0]))}},{key:"_getEdgePoints",value:function(r){var a=r._private.rscratch;if(this._isValidEdge(r)){var n=a.allpts;if(n.length==4)return n;var i=this._getNumSegments(r);return this._getCurveSegmentPoints(n,i)}}},{key:"_getNumSegments",value:function(r){var a=15;return Math.min(Math.max(a,5),this.maxInstances)}},{key:"_getCurveSegmentPoints",value:function(r,a){if(r.length==4)return r;for(var n=Array((a+1)*2),i=0;i<=a;i++)if(i==0)n[0]=r[0],n[1]=r[1];else if(i==a)n[i*2]=r[r.length-2],n[i*2+1]=r[r.length-1];else{var s=i/a;this._setCurvePoint(r,s,n,i*2)}return n}},{key:"_setCurvePoint",value:function(r,a,n,i){if(r.length<=2)n[i]=r[0],n[i+1]=r[1];else{for(var s=Array(r.length-2),o=0;o0}},o=function(d){var f=d.pstyle("text-events").strValue==="yes";return f?ri.USE_BB:ri.IGNORE},l=function(d){var f=d.position(),h=f.x,v=f.y,y=d.outerWidth(),p=d.outerHeight();return{w:y,h:p,x1:h-y/2,y1:v-p/2}};r.drawing.addAtlasCollection("node",{texRows:t.webglTexRowsNodes}),r.drawing.addAtlasCollection("label",{texRows:t.webglTexRows}),r.drawing.addTextureAtlasRenderType("node-body",{collection:"node",getKey:e.getStyleKey,getBoundingBox:e.getElementBox,drawElement:e.drawElement}),r.drawing.addSimpleShapeRenderType("node-body",{getBoundingBox:l,isSimple:V0,shapeProps:{shape:"shape",color:"background-color",opacity:"background-opacity",radius:"corner-radius",border:!0}}),r.drawing.addSimpleShapeRenderType("node-overlay",{getBoundingBox:l,isVisible:s("overlay"),shapeProps:{shape:"overlay-shape",color:"overlay-color",opacity:"overlay-opacity",padding:"overlay-padding",radius:"overlay-corner-radius"}}),r.drawing.addSimpleShapeRenderType("node-underlay",{getBoundingBox:l,isVisible:s("underlay"),shapeProps:{shape:"underlay-shape",color:"underlay-color",opacity:"underlay-opacity",padding:"underlay-padding",radius:"underlay-corner-radius"}}),r.drawing.addTextureAtlasRenderType("label",{collection:"label",getTexPickingMode:o,getKey:no(e.getLabelKey,null),getBoundingBox:io(e.getLabelBox,null),drawClipped:!0,drawElement:e.drawLabel,getRotation:n(null),getRotationPoint:e.getLabelRotationPoint,getRotationOffset:e.getLabelRotationOffset,isVisible:i("label")}),r.drawing.addTextureAtlasRenderType("edge-source-label",{collection:"label",getTexPickingMode:o,getKey:no(e.getSourceLabelKey,"source"),getBoundingBox:io(e.getSourceLabelBox,"source"),drawClipped:!0,drawElement:e.drawSourceLabel,getRotation:n("source"),getRotationPoint:e.getSourceLabelRotationPoint,getRotationOffset:e.getSourceLabelRotationOffset,isVisible:i("source-label")}),r.drawing.addTextureAtlasRenderType("edge-target-label",{collection:"label",getTexPickingMode:o,getKey:no(e.getTargetLabelKey,"target"),getBoundingBox:io(e.getTargetLabelBox,"target"),drawClipped:!0,drawElement:e.drawTargetLabel,getRotation:n("target"),getRotationPoint:e.getTargetLabelRotationPoint,getRotationOffset:e.getTargetLabelRotationOffset,isVisible:i("target-label")});var u=an(function(){console.log("garbage collect flag set"),r.data.gc=!0},1e4);r.onUpdateEleCalcs(function(c,d){var f=!1;d&&d.length>0&&(f|=r.drawing.invalidate(d)),f&&u()}),ub(r)};function lb(t){var e=t.cy.container(),r=e&&e.style&&e.style.backgroundColor||"white";return bd(r)}function jf(t,e){var r=t._private.rscratch;return Rt(r,"labelWrapCachedLines",e)||[]}var no=function(e,r){return function(a){var n=e(a),i=jf(a,r);return i.length>1?i.map(function(s,o){return"".concat(n,"_").concat(o)}):n}},io=function(e,r){return function(a,n){var i=e(a);if(typeof n=="string"){var s=n.indexOf("_");if(s>0){var o=Number(n.substring(s+1)),l=jf(a,r),u=i.h/l.length,c=u*o,d=i.y1+c;return{x1:i.x1,w:i.w,y1:d,h:u,yOffset:c}}}return i}};function ub(t){{var e=t.render;t.render=function(i){i=i||{};var s=t.cy;t.webgl&&(s.zoom()>Ff?(cb(t),e.call(t,i)):(db(t),Xf(t,i,Fa.SCREEN)))}}{var r=t.matchCanvasSize;t.matchCanvasSize=function(i){r.call(t,i),t.pickingFrameBuffer.setFramebufferAttachmentSizes(t.canvasWidth,t.canvasHeight),t.pickingFrameBuffer.needsDraw=!0}}t.findNearestElements=function(i,s,o,l){return yb(t,i,s)};{var a=t.invalidateCachedZSortedEles;t.invalidateCachedZSortedEles=function(){a.call(t),t.pickingFrameBuffer.needsDraw=!0}}{var n=t.notify;t.notify=function(i,s){n.call(t,i,s),i==="viewport"||i==="bounds"?t.pickingFrameBuffer.needsDraw=!0:i==="background"&&t.drawing.invalidate(s,{type:"node-body"})}}}function cb(t){var e=t.data.contexts[t.WEBGL];e.clear(e.COLOR_BUFFER_BIT|e.DEPTH_BUFFER_BIT)}function db(t){var e=function(a){a.save(),a.setTransform(1,0,0,1,0,0),a.clearRect(0,0,t.canvasWidth,t.canvasHeight),a.restore()};e(t.data.contexts[t.NODE]),e(t.data.contexts[t.DRAG])}function fb(t){var e=t.canvasWidth,r=t.canvasHeight,a=pl(t),n=a.pan,i=a.zoom,s=to();Vn(s,s,[n.x,n.y]),Ro(s,s,[i,i]);var o=to();Z0(o,e,r);var l=to();return X0(l,o,s),l}function Yf(t,e){var r=t.canvasWidth,a=t.canvasHeight,n=pl(t),i=n.pan,s=n.zoom;e.setTransform(1,0,0,1,0,0),e.clearRect(0,0,r,a),e.translate(i.x,i.y),e.scale(s,s)}function vb(t,e){t.drawSelectionRectangle(e,function(r){return Yf(t,r)})}function hb(t){var e=t.data.contexts[t.NODE];e.save(),Yf(t,e),e.strokeStyle="rgba(0, 0, 0, 0.3)",e.beginPath(),e.moveTo(-1e3,0),e.lineTo(1e3,0),e.stroke(),e.beginPath(),e.moveTo(0,-1e3),e.lineTo(0,1e3),e.stroke(),e.restore()}function pb(t){var e=function(n,i,s){for(var o=n.atlasManager.getAtlasCollection(i),l=t.data.contexts[t.NODE],u=o.atlases,c=0;c=0&&w.add(x)}return w}function yb(t,e,r){var a=gb(t,e,r),n=t.getCachedZSortedEles(),i,s,o=Lt(a),l;try{for(o.s();!(l=o.n()).done;){var u=l.value,c=n[u];if(!i&&c.isNode()&&(i=c),!s&&c.isEdge()&&(s=c),i&&s)break}}catch(d){o.e(d)}finally{o.f()}return[i,s].filter(Boolean)}function so(t,e,r){var a=t.drawing;e+=1,r.isNode()?(a.drawNode(r,e,"node-underlay"),a.drawNode(r,e,"node-body"),a.drawTexture(r,e,"label"),a.drawNode(r,e,"node-overlay")):(a.drawEdgeLine(r,e),a.drawEdgeArrow(r,e,"source"),a.drawEdgeArrow(r,e,"target"),a.drawTexture(r,e,"label"),a.drawTexture(r,e,"edge-source-label"),a.drawTexture(r,e,"edge-target-label"))}function Xf(t,e,r){var a;t.webglDebug&&(a=performance.now());var n=t.drawing,i=0;if(r.screen&&t.data.canvasNeedsRedraw[t.SELECT_BOX]&&vb(t,e),t.data.canvasNeedsRedraw[t.NODE]||r.picking){var s=t.data.contexts[t.WEBGL];r.screen?(s.clearColor(0,0,0,0),s.enable(s.BLEND),s.blendFunc(s.ONE,s.ONE_MINUS_SRC_ALPHA)):s.disable(s.BLEND),s.clear(s.COLOR_BUFFER_BIT|s.DEPTH_BUFFER_BIT),s.viewport(0,0,s.canvas.width,s.canvas.height);var o=fb(t),l=t.getCachedZSortedEles();if(i=l.length,n.startFrame(o,r),r.screen){for(var u=0;u0&&s>0){h.clearRect(0,0,i,s),h.globalCompositeOperation="source-over";var v=this.getCachedZSortedEles();if(t.full)h.translate(-a.x1*u,-a.y1*u),h.scale(u,u),this.drawElements(h,v),h.scale(1/u,1/u),h.translate(a.x1*u,a.y1*u);else{var y=e.pan(),p={x:y.x*u,y:y.y*u};u*=e.zoom(),h.translate(p.x,p.y),h.scale(u,u),this.drawElements(h,v),h.scale(1/u,1/u),h.translate(-p.x,-p.y)}t.bg&&(h.globalCompositeOperation="destination-over",h.fillStyle=t.bg,h.rect(0,0,i,s),h.fill())}return f};function mb(t,e){for(var r=atob(t),a=new ArrayBuffer(r.length),n=new Uint8Array(a),i=0;i"u"?"undefined":ot(OffscreenCanvas))!=="undefined")r=new OffscreenCanvas(t,e);else{var a=this.cy.window(),n=a.document;r=n.createElement("canvas"),r.width=t,r.height=e}return r};[Vf,rr,hr,hl,Qr,_r,Pt,Wf,Ir,cn,Jf].forEach(function(t){xe(Be,t)});var xb=[{name:"null",impl:Pf},{name:"base",impl:zf},{name:"canvas",impl:bb}],Eb=[{type:"layout",extensions:Um},{type:"renderer",extensions:xb}],tv={},rv={};function av(t,e,r){var a=r,n=function(T){He("Can not register `"+e+"` for `"+t+"` since `"+T+"` already exists in the prototype and can not be overridden")};if(t==="core"){if(Xa.prototype[e])return n(e);Xa.prototype[e]=r}else if(t==="collection"){if(pt.prototype[e])return n(e);pt.prototype[e]=r}else if(t==="layout"){for(var i=function(T){this.options=T,r.call(this,T),Ne(this._private)||(this._private={}),this._private.cy=T.cy,this._private.listeners=[],this.createEmitter()},s=i.prototype=Object.create(r.prototype),o=[],l=0;l +
+ + + + + + + +
+
+ + +
+
+
+
+
+ + `}function Pb(){xt(F("graph")),F("graphSearch").addEventListener("input",t=>{bt.search=t.target.value.trim().toLowerCase(),kn()}),F("graphEntityType").addEventListener("change",t=>{bt.entity_type=t.target.value,kn()}),F("graphPredicate").addEventListener("change",t=>{bt.predicate=t.target.value,kn()}),F("graphMinConfidence").addEventListener("input",t=>{bt.min_confidence=Number(t.target.value),F("graphMinConfidenceOut").textContent=`${Math.round(bt.min_confidence*100)}%`,kn()}),F("graphLayout").addEventListener("change",t=>{bt.layout=t.target.value,iv()}),F("graphFitBtn").addEventListener("click",()=>Qt?.fit(void 0,40)),F("graphReloadBtn").addEventListener("click",()=>gl().catch(()=>{})),fr(()=>xt(F("graph")))}async function gl(){if(!ue.selectedProject)return;lr=await Fe(`/projects/${encodeURIComponent(ue.selectedProject)}/graph/neighborhood?limit=300`)??{nodes:[],edges:[]},Bb(lr.nodes),Db(lr),_o()}function Bb(t){const e=[...new Set(t.map(r=>r.type).filter(Boolean))].sort();ni=new Map,e.forEach((r,a)=>ni.set(r,ad[a%ad.length]))}function Db(t){const e=F("graphEntityType"),r=F("graphPredicate"),a=[...new Set(t.nodes.map(i=>i.type).filter(Boolean))].sort(),n=[...new Set(t.edges.map(i=>i.predicate).filter(Boolean))].sort();e.innerHTML=``+a.map(i=>``).join(""),r.innerHTML=``+n.map(i=>``).join(""),Rb(a)}function Rb(t){F("graphLegend").innerHTML=t.slice(0,12).map(e=>` + ${z(e)} + `).join("")}function Ab(){const t=bt.min_confidence,e=bt.search,r=bt.entity_type,a=bt.predicate,n=new Set,i=[];for(const l of lr.edges)l.confidencer&&l.type!==r||e&&!(l.name||"").toLowerCase().includes(e)&&!(l.canonical_name||"").toLowerCase().includes(e)?!1:bt.predicate||bt.min_confidence>0?n.has(l.id):!0),o=new Set(s.map(l=>l.id));return s.forEach(l=>{i.push({group:"nodes",data:{id:`n${l.id}`,entityId:l.id,label:l.name,type:l.type,canonical:l.canonical_name,metadata:l.metadata,color:ni.get(l.type)||"#7aa2ff"}})}),lr.edges.forEach(l=>{l.confidence{const n=a.target.data();ue.selection={kind:"graph_node",data:n},ur()}),Qt.on("tap","edge",a=>{const n=a.target.data();ue.selection={kind:"graph_edge",data:n},ur()}))}function kn(){if(!Qt)return _o();_o()}function iv(){Qt&&Qt.layout(sv(bt.layout)).run()}function sv(t){return t==="cose"?{name:"cose",animate:!1,idealEdgeLength:110,nodeRepulsion:16e3,padding:40}:t==="concentric"?{name:"concentric",concentric:e=>e.degree(!1),levelWidth:()=>1,animate:!1,padding:30}:t==="breadthfirst"?{name:"breadthfirst",directed:!0,animate:!1,padding:30,spacingFactor:1.2}:{name:t,animate:!1,padding:30}}function Lb(t){const e=lr.nodes.length,r=lr.edges.length;F("graphStats").textContent=`${e} ${V("graph.stats.nodes")} · ${r} ${V("graph.stats.edges")} · ${t} ${V("graph.stats.shown")}`}function Mb(){return[{selector:"node",style:{"background-color":"data(color)",label:"data(label)",color:"#e6ecf3","font-size":11,"text-outline-color":"#0f1216","text-outline-width":2,"text-valign":"bottom","text-margin-y":6,width:24,height:24,"border-width":1,"border-color":"#0f1216"}},{selector:"node:selected",style:{"border-color":"#ffffff","border-width":2,width:28,height:28}},{selector:"edge",style:{width:"mapData(confidence, 0, 1, 1, 4)","line-color":"#364154","target-arrow-color":"#364154","target-arrow-shape":"triangle","curve-style":"bezier",label:"data(predicate)","font-size":9,color:"#8c98ab","text-rotation":"autorotate","text-background-color":"#0f1216","text-background-opacity":.85,"text-background-padding":2,"arrow-scale":.8}},{selector:"edge:selected",style:{"line-color":"#7aa2ff","target-arrow-color":"#7aa2ff",color:"#7aa2ff",width:3}}]}let Si=[],_t=[],Ot=null;const wt=new Set,sr={status:"",predicate:"",search:""};let Pn=null;function _b(){return` +
+
+
+ + + + + + +
+
+
+
+
+
+ + + + + + + + + +
+
+
+ `}function Ib(){const t=F("claims");xt(t),fr(()=>{xt(t),Er()}),F("wbStatusFilter").addEventListener("change",e=>{sr.status=e.target.value,Er()}),F("wbPredicateFilter").addEventListener("change",e=>{sr.predicate=e.target.value,Er()}),F("wbSearch").addEventListener("input",e=>{sr.search=e.target.value.trim().toLowerCase(),Er()}),F("wbBulkAcceptBtn").addEventListener("click",()=>ii("validated_claim")),F("wbBulkRejectBtn").addEventListener("click",()=>ii("rejected")),Hb()}async function Ta(){if(!ue.selectedProject)return;const t=encodeURIComponent(ue.selectedProject),[e,r]=await Promise.all([Fe(`/projects/${t}/claims?limit=200&include_candidates=true`),Fe(`/projects/${t}/claims?limit=200&status=rejected`).catch(()=>[])]),a=new Map;for(const n of[...e,...r])a.set(n.id,n);Si=[...a.values()].sort((n,i)=>(i.last_seen_at??"").localeCompare(n.last_seen_at??"")),Ob(),Er()}function Ob(){const t=F("wbPredicateFilter");if(!t)return;const e=[...new Set(Si.map(a=>a.predicate).filter(Boolean))].sort(),r=sr.predicate;t.innerHTML=``+e.map(a=>``).join("")}function Nb(){const t=sr.search;_t=Si.filter(e=>!(sr.status&&e.status!==sr.status||sr.predicate&&e.predicate!==sr.predicate||t&&!`${e.subject??""} ${e.predicate??""} ${e.object??""} ${e.evidence_text??""}`.toLowerCase().includes(t))),_t.some(e=>e.id===Ot)||(Ot=_t[0]?.id??null);for(const e of[...wt])_t.some(r=>r.id===e)||wt.delete(e)}function Er(){Nb(),ov(),lv(),uv()}function ov(){const t=F("wbList");if(!_t.length){t.innerHTML=`
${z(V("workbench.empty"))}
`;return}t.innerHTML=_t.map(e=>zb(e)).join(""),t.querySelectorAll("[data-claim-row]").forEach(e=>{const r=Number(e.dataset.claimRow);e.addEventListener("click",a=>{a.shiftKey||a.ctrlKey||a.metaKey?cv(r):(Ot=r,wt.clear(),wt.add(r)),ov(),lv(),uv(),nd()})}),nd()}function zb(t){const e=t.id===Ot,r=wt.has(t.id),a=`workbench.status.${t.status??"active"}`;return` + + `}function lv(){const t=F("wbDetail"),e=_t.find(n=>n.id===Ot);if(!e){t.innerHTML=`
${z(V("workbench.select_hint"))}
`;return}const r=`workbench.status.${e.status??"active"}`,a=e.confidence_breakdown?`LLM ${Bn(e.confidence_breakdown.llm_confidence)} · evidence ${Bn(e.confidence_breakdown.evidence_confidence)} · ontology ${Bn(e.confidence_breakdown.ontology_confidence)} · stored ${Bn(e.confidence_breakdown.stored_confidence??e.confidence)}`:"";t.innerHTML=` +
+
+

${z(e.subject??"")} ${z(e.predicate??"")} ${z(e.object??"")}

+
+ ${z(V(r,e.status??""))} + ${Math.round((e.confidence??0)*100)}% +
+
+
+ + + +
+
+ +
+
${z(V("workbench.detail.subject"))}
${z(e.subject??"")} (${z(e.subject_type??"")})
+
${z(V("workbench.detail.predicate"))}
${z(e.predicate??"")}
+
${z(V("workbench.detail.object"))}
${z(e.object??JSON.stringify(e.object_value??""))}
+
${z(V("workbench.detail.source"))}
${z(e.source??"")}
+
${z(V("workbench.detail.page"))}
${e.page_url?`${z(e.page_url)}`:"-"}
+ ${a?`
${z(V("workbench.detail.breakdown"))}
${z(a)}
`:""} +
+ + ${e.evidence_text?` +
+

${z(V("workbench.detail.evidence"))}

+ 
${$b(e.evidence_text,e)}
+
+ `:""} + +
+ + + + +
+ `,xt(t),t.querySelectorAll("[data-action]").forEach(n=>{n.addEventListener("click",()=>Io(n.dataset.action))})}function $b(t,e){const r=[];e.subject&&r.push(e.subject),e.object&&r.push(e.object);let a=z(t);for(const n of r){if(!n||n.length<2)continue;const i=Fb(n);a=a.replace(new RegExp(i,"gi"),s=>`${s}`)}return a}function Fb(t){return t.replace(/[.*+?^${}()|[\]\\]/g,"\\$&")}function uv(){const t=wt.size;F("wbSelectedCount").textContent=t>1?V("workbench.bulk_count").replace("{count}",t):"";const e=t>1;F("wbBulkAcceptBtn").disabled=!e,F("wbBulkRejectBtn").disabled=!e}function Bn(t){return t==null?"-":`${Math.round(t*100)}%`}function cv(t){wt.has(t)?wt.delete(t):wt.add(t),wt.size===1&&(Ot=[...wt][0])}function nd(){const t=document.querySelector(`[data-claim-row="${Ot}"]`);t&&t.scrollIntoView({block:"nearest"})}async function Io(t){const e=_t.find(r=>r.id===Ot);e&&(t==="accept"?await oo(e.id,"validated_claim"):t==="reject"?await oo(e.id,"rejected"):t==="unreview"?await oo(e.id,"active"):t==="save-confidence"?await qb(e):t==="accept-similar"&&await Vb(e))}async function oo(t,e){const r=F("wbReason")?.value?.trim()||null;try{await Fe(`/claims/${t}/status`,{method:"PATCH",body:JSON.stringify({status:e,reason:r})}),Ae(V("workbench.toast.reviewed")),await Ta()}catch(a){Ae(a.message)}}async function qb(t){const e=Number(F("wbConfidence").value),r=F("wbReason").value.trim()||null;try{await Fe(`/claims/${t.id}/confidence`,{method:"PATCH",body:JSON.stringify({confidence:e,reason:r})}),Ae(V("toast.claim_updated")),await Ta()}catch(a){Ae(a.message)}}async function Vb(t){const e=t.object??JSON.stringify(t.object_value??""),r=Si.filter(a=>a.predicate===t.predicate&&(a.object??JSON.stringify(a.object_value??""))===e).map(a=>a.id);if(r.length)try{const a=await Fe("/claims/bulk-status",{method:"POST",body:JSON.stringify({claim_ids:r,status:"validated_claim",reason:F("wbReason")?.value?.trim()||null})});Ae(V("workbench.toast.bulk_done").replace("{count}",a.updated??r.length)),await Ta()}catch(a){Ae(a.message)}}async function ii(t){const e=[...wt];if(!(e.length<2))try{const r=await Fe("/claims/bulk-status",{method:"POST",body:JSON.stringify({claim_ids:e,status:t,reason:null})});Ae(V("workbench.toast.bulk_done").replace("{count}",r.updated??e.length)),wt.clear(),await Ta()}catch(r){Ae(r.message)}}function Hb(){Pn&&document.removeEventListener("keydown",Pn),Pn=t=>{if(!Gb()||t.target.matches("input, textarea, select"))return;const e=t.key;if(e==="j"||e==="ArrowDown"){t.preventDefault(),id(1);return}if(e==="k"||e==="ArrowUp"){t.preventDefault(),id(-1);return}if(e===" "){t.preventDefault(),Ot!=null&&(cv(Ot),Er());return}if(e==="Escape"){wt.clear(),Er();return}if(e==="/"){t.preventDefault(),F("wbSearch")?.focus();return}if(e==="a"&&!t.shiftKey){t.preventDefault(),Io("accept");return}if(e==="r"&&!t.shiftKey){t.preventDefault(),Io("reject");return}if(t.shiftKey&&(e==="A"||e==="a")){t.preventDefault(),ii("validated_claim");return}if(t.shiftKey&&(e==="R"||e==="r")){t.preventDefault(),ii("rejected");return}},document.addEventListener("keydown",Pn)}function Gb(){const t=F("claims");return t&&t.classList.contains("active")}function id(t){if(!_t.length)return;let r=_t.findIndex(a=>a.id===Ot)+t;r<0&&(r=0),r>=_t.length&&(r=_t.length-1),Ot=_t[r].id,wt.clear(),wt.add(Ot),Er()}let qr=null;function Ub(){return` +
+
+

+ +
+
+ +
+ +
+

+
+
+ +
+
+

+
+
+
+

+
+
+
+ +
+

+
+
+
+ `}function Kb(){const t=F("overview");xt(t),F("pipelineRefreshBtn").addEventListener("click",()=>yl().catch(()=>{})),fr(()=>{xt(t),dv()})}async function yl(){ue.selectedProject&&(qr=await Fe(`/projects/${encodeURIComponent(ue.selectedProject)}/pipeline`),dv())}function dv(){qr&&(jb(qr.stages||[]),Xb(qr.stages||[]),Zb(qr.entity_types||[]),Qb(qr.recent_pages||[]),Jb(qr.recent_claims||[]))}const Wb={crawled:"#7aa2ff",extracted:"#22d3ee",claims:"#facc15",validated:"#4ade80",graph:"#a78bfa"};function jb(t){const e=Math.max(1,...t.map(a=>a.count||0)),r=F("pipelineStages");r.innerHTML=t.map((a,n)=>{const i=n>0?t[n-1].count:null,s=i&&i>0?Math.round(a.count/i*100):null,o=a.count/e*100,l=Wb[a.key]||"#7aa2ff",u=Yb(a);return` +
+
+ ${z(V(`pipeline.stage.${a.key}`,a.key))} + ${s!=null?`${s}% ${z(V("pipeline.retained"))}`:""} +
+
${a.count}
+
+ ${u?`
${u}
`:""} +
+ `}).join("")}function Yb(t){return t.extra?t.key==="extracted"?`events ${t.extra.events??0} · errors ${t.extra.errors??0}`:t.key==="claims"?Object.entries(t.extra).map(([e,r])=>`${z(e)} ${r}`).join(" · "):t.key==="graph"?`entities ${t.extra.entities??0}`:"":""}function Xb(t){const e=ue.projectDetail,r=t.find(n=>n.key==="claims"),a=t.find(n=>n.key==="validated");F("pipelineSummary").innerHTML=` +
${z(V("overview.project"))}${z(e?.name??"-")}
+
${z(V("overview.domain"))}${z(e?.domain??"-")}
+
${z(V("overview.sources"))}${(e?.sources??[]).length}
+
${z(V("overview.entities"))}${t.find(n=>n.key==="graph")?.extra?.entities??0}
+
${z(V("pipeline.stage.claims"))}${r?.count??0}
+
${z(V("pipeline.stage.validated"))}${a?.count??0}
+ `}function Zb(t){const e=F("pipelineEntityTypes");if(!t.length){e.innerHTML=`

${z(V("pipeline.empty.claims"))}

`;return}const r=Math.max(1,...t.map(a=>a.count));e.innerHTML=t.map(a=>{const n=a.count/r*100;return` +
+ ${z(a.type)} +
+ ${a.count} +
+ `}).join("")}function Qb(t){const e=F("pipelineRecentPages");if(!t.length){e.innerHTML=`

${z(V("pipeline.empty.pages"))}

`;return}e.innerHTML=t.map(r=>` +
+
+ ${z(r.page_type??"")} + ${z(r.status_code??"-")} + ${z(fv(r.fetched_at))} +
+
${z(r.title??r.url??"")}
+ ${z(r.url??"")} +
+ `).join("")}function Jb(t){const e=F("pipelineRecentClaims");if(!t.length){e.innerHTML=`

${z(V("pipeline.empty.claims"))}

`;return}e.innerHTML=t.map(r=>` +
+
+ ${z(r.subject??"")} + ${z(r.predicate??"")} +
+
+ ${z(r.status??"")} + ${Math.round((r.confidence??0)*100)}% + ${z(fv(r.last_seen_at))} +
+
+ `).join("")}function fv(t){if(!t)return"";const e=new Date(t);if(isNaN(e.getTime()))return"";const a=(Date.now()-e.getTime())/1e3;return a<60?`${Math.floor(a)}s`:a<3600?`${Math.floor(a/60)}m`:a<86400?`${Math.floor(a/3600)}h`:`${Math.floor(a/86400)}d`}function ew(t){const e=F("sourceTable");if(!e)return;const r=t?.sources??[];if(!r.length){e.innerHTML=`
${z(V("table.no_data"))}
`;return}e.innerHTML=` + + + + + + + + + ${r.map(a=>` + + + + + + + + `).join("")} +
${z(V("table.name"))}${z(V("table.type"))}${z(V("table.trust"))}${z(V("table.robots"))}${z(V("table.rate"))}
${z(a.name)}${z(a.type)}${a.trust_level}${a.respect_robots_txt?"on":"off"}${a.rate_limit_per_minute}/min
+ `}let Dn=null;function tw(t){Dn=t,Dn.innerHTML=rw(),xt(Dn),document.querySelectorAll(".tab").forEach(e=>{e.addEventListener("click",()=>{document.querySelectorAll(".tab").forEach(r=>r.classList.remove("active")),document.querySelectorAll(".tab-panel").forEach(r=>r.classList.remove("active")),e.classList.add("active"),document.getElementById(e.dataset.tab).classList.add("active")})}),Pb(),Ib(),Kb(),F("loadOntologyRegistryBtn").addEventListener("click",Oo),F("loadEntitiesBtn").addEventListener("click",No),F("mergeEntitiesBtn").addEventListener("click",aw),F("loadExtractionLogsBtn").addEventListener("click",zo),F("runResearchBtn").addEventListener("click",iw),F("loadResearchBtn").addEventListener("click",$o),F("loadGraphQueryBtn").addEventListener("click",Fo),F("loadTagsBtn").addEventListener("click",qo),F("recommendBtn").addEventListener("click",lw),fr(()=>{xt(Dn),vv(),hv(),yl().catch(()=>{}),Oo().catch(()=>{}),No().catch(()=>{}),Ta().catch(()=>{}),zo().catch(()=>{}),$o().catch(()=>{}),Fo().catch(()=>{}),qo().catch(()=>{}),gl().catch(()=>{})})}function rw(){return` + + + ${kb()} + + ${Ub()} + +
+
+ +
+
+
+

+
+
+
+

+
+
+
+
+
+

+
+
+
+

+
+
+
+
+

+
+
+
+
+

+
+
+
+

+
+
+
+
+ +
+
+ + +
+
+ + + +
+
+
+ + ${_b()} + +
+
+ +
+
+
+ +
+
+

+
+ + + + +
+
+ + +
+
+
+
+
+

+
+ + + +
+
+
+
+ +
+
+ +
+
+
+ +
+
+ + + + + +
+ +
+
+ `}function vv(){ew(ue.projectDetail)}function hv(){const t=ue.ontology?.entity_types??[],e=ue.ontology?.predicates??[];F("ontologyEntities").innerHTML=t.map(Pl).join(""),F("ontologyPredicates").innerHTML=e.map(Pl).join(""),F("entityTypeFilter").innerHTML=`${t.map(r=>``).join("")}`}async function Oo(){if(!ue.selectedProject)return;const t=encodeURIComponent(ue.selectedProject),[e,r,a,n]=await Promise.all([Fe(`/projects/${t}/ontology/registry`),Fe(`/projects/${t}/ontology/proposals?limit=50`),Fe(`/projects/${t}/knowledge-gaps?limit=50`),Fe(`/projects/${t}/ontology/triples?limit=50`)]);F("ontologyRegistryEntityTable").innerHTML=Mt([V("table.type"),V("table.domain"),V("table.status"),V("table.confidence")],(e.entity_types??[]).map(i=>[i.name,i.domain,i.status,Math.round((i.confidence??0)*100)+"%"])),F("ontologyRegistryRelationTable").innerHTML=Mt([V("table.relation"),V("table.domain"),V("table.subject"),V("table.object"),V("table.status")],(e.relation_types??[]).map(i=>[i.name,i.domain,(i.allowed_subject_types??[]).join(", "),(i.allowed_object_types??[]).join(", ")||(i.semantic_constraints?.literal_value?"literal":""),i.status])),F("ontologyProposalTable").innerHTML=Mt([V("table.type"),V("table.name"),V("table.status"),V("table.reason")],(r??[]).map(i=>[i.proposal_type,i.name,i.status,z(i.reason??"")])),F("knowledgeGapTable").innerHTML=Mt([V("table.gap"),V("table.target"),V("table.priority"),V("table.description")],(a??[]).map(i=>[i.gap_type,`${i.target_type??""}: ${i.target_name??""}`,Math.round((i.priority??0)*100)+"%",z(i.description??"")])),F("ontologyTripleTable").innerHTML=Mt([V("table.subject"),V("table.relation"),V("table.object"),V("table.status"),V("table.support")],(n??[]).map(i=>[i.subject??"",i.predicate,i.object??JSON.stringify(i.object_value??""),i.status,i.support_count]))}let sd=[];async function No(){if(!ue.selectedProject)return;const t=F("entityTypeFilter").value,e=t?`?entity_type=${encodeURIComponent(t)}&limit=100`:"?limit=100",r=await Fe(`/projects/${encodeURIComponent(ue.selectedProject)}/entities${e}`);sd=r,F("metricEntities").textContent=r.length,F("entityTable").innerHTML=Mt([V("table.id"),V("table.type"),V("table.name"),V("table.metadata")],r.map(a=>[``,a.type,z(a.name),`${z(JSON.stringify(a.metadata??{}))}`])),F("entityTable").querySelectorAll("[data-entity-id]").forEach(a=>{a.addEventListener("click",()=>{const n=Number(a.dataset.entityId),i=sd.find(s=>s.id===n);i&&(ue.selection={kind:"entity",data:i},ur())})})}async function aw(){if(!ue.selectedProject)return;const t=Number(F("mergeSourceId").value),e=Number(F("mergeTargetId").value);if(!t||!e||t===e){Ae(V("toast.merge_check"));return}try{const r=await Fe("/entities/merge",{method:"POST",body:JSON.stringify({project_name:ue.selectedProject,source_entity_id:t,target_entity_id:e})});if(!r.ok){Ae(r.error??V("toast.merge_check"));return}Ae(V("toast.entity_merged")),F("mergeSourceId").value="",F("mergeTargetId").value="";const a=new CustomEvent("workspace:refresh");window.dispatchEvent(a)}catch(r){Ae(r.message)}}async function zo(){if(!ue.selectedProject)return;const t=await Fe(`/projects/${encodeURIComponent(ue.selectedProject)}/extraction-logs?limit=50`);F("extractionLogTable").innerHTML=t.map(nw).join("")}function nw(t){const e=t.validation??{},r=t.page_context??{},n=(e.rejected_claims??[]).slice(0,3).map(i=>`${i.predicate??""}: ${i.reason??""}`).join(" | ");return` +
+
+ ${z(t.extractor_name)} + ${z(t.provider)} + ${z(e.claim_status??"")} + accepted ${e.accepted_claim_count??0} + rejected ${e.rejected_claim_count??0} +
+
${z(r.page_type??"")} / ${z(r.crawl_status??"")} / ${z(r.extraction_status??"")}
+
${z(t.page_url??"")}
+
${z(n)}
+
+ `}async function iw(){if(!ue.selectedProject)return;F("researchResult").textContent=V("status.research_running");const t=F("researchSeedEntityId").value.trim();try{const e=await Fe("/research/run",{method:"POST",body:JSON.stringify({...Ho(),project_name:ue.selectedProject,seed_entity_id:t?Number(t):null,goal:F("researchGoal").value.trim()||"Semantic ontology exploration",max_depth:Number(F("siteMaxDepth").value||2),max_steps:Number(F("researchMaxSteps").value||8),max_branch:8,min_relevance:Number(F("researchMinRelevance").value||.35),same_domain_only:F("sameDomainOnly").checked,analyze_page_types:["ProductPage","BrandStoryPage","ReviewPage"]})});F("researchResult").textContent=`Research ${e.status}: explored ${e.explored_count}, analyzed ${e.analyzed_count}, queued ${e.queued_count}, skipped ${e.skipped_count}`,Ae(V("toast.research_completed"));const r=new CustomEvent("workspace:refresh");window.dispatchEvent(r)}catch(e){F("researchResult").textContent=`${V("toast.research_failed")}: ${e.message}`,Ae(V("toast.research_failed"))}}async function $o(){if(!ue.selectedProject)return;const t=await Fe(`/projects/${encodeURIComponent(ue.selectedProject)}/research/sessions?limit=20`);F("researchSessions").innerHTML=t.map(sw).join("")}function sw(t){const e=t.history??[],r=e.length?e[e.length-1]:null,n=(t.memory??{}).conflicts??[];return` +
+
+ ${z(t.name??"Research session")} + ${z(t.status??"")} + history ${e.length} + queue ${(t.queue??[]).length} + conflicts ${n.length} +
+
${z(t.goal??"")}
+
seed ${z(t.seed??"")}
+
${z(r?.outcome?.reason??r?.outcome?.status??"")}
+
+ `}async function Fo(){if(!ue.selectedProject)return;const t=F("graphQueryKind")?.value??"trend_summary",e=F("graphQueryText")?.value.trim()??"";let r=`kind=${encodeURIComponent(t)}`;t==="brand_products"&&e&&(r+=`&brand=${encodeURIComponent(e)}`),t==="products_by_tag"&&e&&(r+=`&tag=${encodeURIComponent(e)}`);const a=await Fe(`/projects/${encodeURIComponent(ue.selectedProject)}/graph/query?${r}`);F("graphQueryTable").innerHTML=ow(t,a)}function ow(t,e){return t==="brand_products"?Mt([V("table.product"),V("table.type"),V("table.brand"),V("table.confidence")],e.map(r=>[r.product,r.product_type,r.brand??"",Math.round(r.confidence*100)+"%"])):t==="products_by_tag"?Mt([V("table.product"),V("table.predicate"),V("table.tag"),V("table.confidence")],e.map(r=>[r.product,r.predicate,r.tag,Math.round(r.confidence*100)+"%"])):t==="relation_summary"?Mt([V("table.predicate"),V("table.status"),V("table.support"),V("table.confidence")],e.map(r=>[r.predicate,r.status,r.support_count,Math.round((r.max_confidence??0)*100)+"%"])):t==="entity_type_summary"?Mt([V("table.type"),V("table.count")],e.map(r=>[r.entity_type,r.count])):Mt([V("table.predicate"),V("table.name"),V("table.type"),V("table.support"),V("table.confidence")],e.map(r=>[r.predicate,r.name,r.entity_type,r.support_count,Math.round((r.max_confidence??0)*100)+"%"]))}async function qo(){if(!ue.selectedProject)return;const t=await Fe(`/projects/${encodeURIComponent(ue.selectedProject)}/recommendation-tags`);F("tagTable").innerHTML=Mt([V("table.predicate"),V("table.type"),V("table.name"),V("table.support"),V("table.confidence")],t.map(e=>[e.predicate,e.type,e.name,e.support_count,Math.round((e.max_confidence??0)*100)+"%"]))}async function lw(){if(ue.selectedProject)try{const t=await Fe("/recommend",{method:"POST",body:JSON.stringify({project_name:ue.selectedProject,target_entity_type:ue.projectDetail?.config?.recommendation?.target_entity_type??"Perfume",preferences:{preferred_notes:Di(F("preferredNotes").value),avoided_notes:Di(F("avoidedNotes").value),preferred_moods:Di(F("preferredMoods").value),season_context:F("seasonContext").value.trim()||null,occasion_context:F("occasionContext").value.trim()||null},limit:10})});F("recommendTable").innerHTML=Mt([V("table.name"),V("table.type"),V("table.score"),V("table.reasons")],t.map(e=>[e.name,e.entity_type,e.score,(e.reasons??[]).join(", ")]))}catch(t){Ae(t.message)}}let lo=null,pv="",qa=null;function uw(){const t=F("globalSearchInput"),e=F("globalSearchPanel");xt(t.parentElement),t.addEventListener("input",()=>{const r=t.value.trim();lo&&window.clearTimeout(lo),lo=window.setTimeout(()=>cw(r),200)}),t.addEventListener("focus",()=>{pv?Vo(qa):gv(),e.hidden=!1}),t.addEventListener("keydown",r=>{r.key==="Escape"&&(t.value="",e.hidden=!0,t.blur()),r.key==="ArrowDown"&&(r.preventDefault(),e.querySelector("[data-search-item]")?.focus())}),document.addEventListener("click",r=>{!e.contains(r.target)&&r.target!==t&&(e.hidden=!0)}),document.addEventListener("keydown",r=>{r.key==="/"&&!r.target.matches("input, textarea, select")&&(r.preventDefault(),t.focus(),t.select())}),fr(()=>{xt(t.parentElement),qa&&Vo(qa)})}async function cw(t){if(!ue.selectedProject)return;pv=t;const e=F("globalSearchPanel");if(!t){gv();return}try{qa=await Fe(`/projects/${encodeURIComponent(ue.selectedProject)}/search?q=${encodeURIComponent(t)}&limit=8`),Vo(qa)}catch(r){e.innerHTML=`
${z(r.message)}
`,e.hidden=!1}}function gv(){const t=F("globalSearchPanel");t.innerHTML=`
${z(V("search.hint"))}
`,t.hidden=!1}function Vo(t){const e=F("globalSearchPanel");if(!t)return;const r=[{key:"entities",items:t.entities,render:dw},{key:"claims",items:t.claims,render:fw},{key:"pages",items:t.pages,render:vw},{key:"predicates",items:t.predicates,render:hw}].filter(a=>(a.items??[]).length>0);r.length?(e.innerHTML=r.map(a=>` +
+
${z(V(`search.group.${a.key}`))}
+ ${a.items.map((n,i)=>a.render(n,i)).join("")} +
+ `).join(""),e.querySelectorAll("[data-search-item]").forEach(a=>{a.addEventListener("click",()=>pw(a.dataset.searchItem,JSON.parse(a.dataset.payload))),a.addEventListener("keydown",n=>{n.key==="Enter"&&(n.preventDefault(),a.click()),n.key==="ArrowDown"&&(n.preventDefault(),(a.nextElementSibling||e.querySelector("[data-search-item]"))?.focus?.()),n.key==="ArrowUp"&&(n.preventDefault(),(a.previousElementSibling||e.querySelector("[data-search-item]:last-of-type"))?.focus?.()),n.key==="Escape"&&F("globalSearchInput").focus()})})):e.innerHTML=`
${z(V("search.empty"))}
`,e.hidden=!1}function dw(t){return` + + `}function fw(t){const e=t.object_value?JSON.stringify(t.object_value):"";return` + + `}function vw(t){return` + + `}function hw(t){return` + + `}function ki(t){return String(t).replaceAll("&","&").replaceAll("'","'").replaceAll('"',""")}function pw(t,e){const r=F("globalSearchPanel");if(r.hidden=!0,t==="entity")uo("entities"),ue.selection={kind:"entity",data:{id:e.id,name:e.name,type:e.type,metadata:{}}},ur();else if(t==="claim")uo("claims"),ue.selection={kind:"claim",data:{...e,object:void 0}},ur();else if(t==="page")e.url&&window.open(e.url,"_blank","noreferrer");else if(t==="predicate"){uo("graph");const a=F("graphPredicate");a&&(a.value=e,a.dispatchEvent(new Event("change")))}}function uo(t){document.querySelectorAll(".tab").forEach(e=>e.classList.toggle("active",e.dataset.tab===t)),document.querySelectorAll(".tab-panel").forEach(e=>e.classList.toggle("active",e.id===t))}const gw=document.getElementById("app"),ml=Rv(gw);Av(ml.sidebarHost,{onProjectChanged:bl,onCrawlUpdate:()=>ur(),refreshAll:wl});tw(ml.workspaceHost);Hv(ml.inspectorHost);uw();document.getElementById("refreshBtn").addEventListener("click",bl);window.addEventListener("project:select",t=>yv(t.detail));window.addEventListener("workspace:refresh",wl);fr(()=>ur());bl().catch(t=>Ae(t.message));async function bl(){ue.projects=await Fe("/projects"),!ue.selectedProject&&ue.projects.length&&(ue.selectedProject=ue.projects[0].name),Go(),ue.selectedProject&&await yv(ue.selectedProject)}async function yv(t){ue.selectedProject=t,ue.projectDetail=await Fe(`/projects/${encodeURIComponent(t)}`),ue.ontology=await Fe(`/ontology/${encodeURIComponent(ue.projectDetail.domain)}`),Go(),Mv(ue.projectDetail),vv(),hv(),ur(),await wl()}async function wl(){if(!ue.selectedProject)return;const t=[["pipeline",yl],["ontology registry",Oo],["entities",No],["claims workbench",Ta],["extraction logs",zo],["research sessions",$o],["graph query",Fo],["tags",qo],["graph",gl]];await Promise.all(t.map(async([e,r])=>{try{await r()}catch(a){console.warn(`${e} load failed`,a)}})),ur()} +//# sourceMappingURL=index-BsoXaTIL.js.map diff --git a/crawler_platform/app/web/static/assets/index-BsoXaTIL.js.map b/crawler_platform/app/web/static/assets/index-BsoXaTIL.js.map new file mode 100644 index 0000000..4113585 --- /dev/null +++ b/crawler_platform/app/web/static/assets/index-BsoXaTIL.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index-BsoXaTIL.js","sources":["../../frontend/src/i18n.js","../../frontend/src/shell.js","../../frontend/src/api.js","../../frontend/src/state.js","../../frontend/src/utils.js","../../frontend/src/sidebar.js","../../frontend/src/inspector.js","../../frontend/node_modules/cytoscape/dist/cytoscape.esm.mjs","../../frontend/src/graph.js","../../frontend/src/workbench.js","../../frontend/src/pipeline.js","../../frontend/src/workspace.js","../../frontend/src/search.js","../../frontend/src/main.js"],"sourcesContent":["const STORAGE_KEY = \"ocp.lang\";\nconst SUPPORTED = [\"ko\", \"en\"];\nconst DEFAULT_LANG = \"ko\";\n\nconst dict = {\n ko: {\n \"app.title\": \"RPG Onta\",\n \"app.subtitle\": \"프로젝트 크롤, 온톨로지 매핑, 클레임 리뷰, 추천 태그\",\n \"header.refresh\": \"새로고침\",\n \"header.lang.ko\": \"한\",\n \"header.lang.en\": \"EN\",\n\n \"sidebar.projects\": \"프로젝트\",\n \"sidebar.config_path\": \"설정 파일 경로\",\n \"sidebar.create_project\": \"프로젝트 생성\",\n \"sidebar.reset_project\": \"현재 프로젝트 데이터 리셋\",\n \"sidebar.crawl\": \"크롤\",\n \"sidebar.source\": \"소스\",\n \"sidebar.url_or_seed\": \"URL / 시드 URL\",\n \"sidebar.analyzer\": \"분석기\",\n \"sidebar.model\": \"모델\",\n \"sidebar.base_url\": \"Base URL\",\n \"sidebar.test_analyzer\": \"분석기 테스트\",\n \"sidebar.discover\": \"링크 탐색\",\n \"sidebar.crawl_url\": \"URL 크롤\",\n \"sidebar.max_depth\": \"최대 깊이\",\n \"sidebar.max_pages\": \"최대 페이지\",\n \"sidebar.same_domain\": \"같은 도메인만\",\n \"sidebar.respect_robots\": \"robots.txt 준수\",\n \"sidebar.crawl_site\": \"시드부터 사이트 크롤\",\n \"sidebar.stop_crawl\": \"현재 크롤 중지\",\n\n \"tabs.overview\": \"개요\",\n \"tabs.graph\": \"그래프\",\n \"tabs.ontology\": \"온톨로지\",\n \"tabs.entities\": \"엔티티\",\n \"tabs.claims\": \"클레임\",\n \"tabs.debug\": \"디버그\",\n \"tabs.research\": \"리서치\",\n \"tabs.tags\": \"태그\",\n \"tabs.recommend\": \"추천\",\n\n \"graph.search_placeholder\": \"엔티티명 검색...\",\n \"graph.entity_type_all\": \"전체 타입\",\n \"graph.predicate_all\": \"전체 관계\",\n \"graph.min_confidence\": \"최소 신뢰도\",\n \"graph.layout\": \"레이아웃\",\n \"graph.layout.cose\": \"Force\",\n \"graph.layout.circle\": \"원형\",\n \"graph.layout.concentric\": \"동심원\",\n \"graph.layout.grid\": \"그리드\",\n \"graph.layout.breadthfirst\": \"트리\",\n \"graph.reload\": \"그래프 새로고침\",\n \"graph.fit\": \"맞춤\",\n \"graph.empty\": \"아직 데이터가 없습니다. 크롤이 클레임을 만들면 그래프에 노드가 나타납니다.\",\n \"graph.stats.nodes\": \"노드\",\n \"graph.stats.edges\": \"엣지\",\n \"graph.stats.shown\": \"표시\",\n\n \"overview.project\": \"프로젝트\",\n \"overview.domain\": \"도메인\",\n \"overview.sources\": \"소스\",\n \"overview.entities\": \"엔티티\",\n \"overview.sources_heading\": \"소스\",\n\n \"pipeline.heading\": \"파이프라인\",\n \"pipeline.refresh\": \"새로고침\",\n \"pipeline.stage.crawled\": \"크롤됨\",\n \"pipeline.stage.extracted\": \"추출됨\",\n \"pipeline.stage.claims\": \"클레임\",\n \"pipeline.stage.validated\": \"승인됨\",\n \"pipeline.stage.graph\": \"그래프 트리플\",\n \"pipeline.retained\": \"유지율\",\n \"pipeline.entity_types_heading\": \"엔티티 타입 분포\",\n \"pipeline.recent_pages\": \"최근 페이지\",\n \"pipeline.recent_claims\": \"최근 클레임\",\n \"pipeline.empty.pages\": \"아직 크롤된 페이지가 없습니다.\",\n \"pipeline.empty.claims\": \"아직 생성된 클레임이 없습니다.\",\n\n \"search.placeholder\": \"엔티티 · 클레임 · 페이지 검색...\",\n \"search.aria_label\": \"전역 검색\",\n \"search.group.entities\": \"엔티티\",\n \"search.group.claims\": \"클레임\",\n \"search.group.pages\": \"페이지\",\n \"search.group.predicates\": \"술어\",\n \"search.empty\": \"결과 없음\",\n \"search.hint\": \"최소 1글자 이상 입력하세요\",\n \"table.name\": \"이름\",\n \"table.type\": \"타입\",\n \"table.trust\": \"신뢰도\",\n \"table.robots\": \"robots\",\n \"table.rate\": \"속도제한\",\n \"table.domain\": \"도메인\",\n \"table.status\": \"상태\",\n \"table.confidence\": \"신뢰도\",\n \"table.relation\": \"관계\",\n \"table.subject\": \"주체\",\n \"table.object\": \"객체\",\n \"table.support\": \"근거 수\",\n \"table.predicate\": \"술어\",\n \"table.tag\": \"태그\",\n \"table.brand\": \"브랜드\",\n \"table.product\": \"상품\",\n \"table.count\": \"수\",\n \"table.no_data\": \"데이터 없음.\",\n \"table.id\": \"ID\",\n \"table.metadata\": \"메타데이터\",\n \"table.reason\": \"사유\",\n \"table.gap\": \"갭\",\n \"table.target\": \"대상\",\n \"table.priority\": \"우선순위\",\n \"table.description\": \"설명\",\n \"table.score\": \"점수\",\n \"table.reasons\": \"근거\",\n\n \"ontology.refresh_registry\": \"레지스트리 새로고침\",\n \"ontology.configured_entity_types\": \"설정된 엔티티 타입\",\n \"ontology.configured_predicates\": \"설정된 술어\",\n \"ontology.registry_entity_types\": \"레지스트리 엔티티 타입\",\n \"ontology.registry_relation_types\": \"레지스트리 관계 타입\",\n \"ontology.triples\": \"온톨로지 트리플\",\n \"ontology.proposals\": \"스키마 제안\",\n \"ontology.knowledge_gaps\": \"지식 갭\",\n\n \"entities.all_types\": \"전체 타입\",\n \"entities.load\": \"불러오기\",\n \"entities.merge_source_placeholder\": \"병합할 엔티티 ID\",\n \"entities.merge_target_placeholder\": \"유지할 엔티티 ID\",\n \"entities.merge\": \"병합\",\n\n \"claims.show_candidates\": \"후보 포함\",\n \"claims.refresh\": \"클레임 새로고침\",\n \"claims.save\": \"저장\",\n \"claims.confidence_label\": \"신뢰도\",\n \"claims.reason_label\": \"사유\",\n\n \"workbench.status_filter\": \"상태\",\n \"workbench.status.all\": \"전체\",\n \"workbench.status.active\": \"활성\",\n \"workbench.status.validated_claim\": \"승인됨\",\n \"workbench.status.rejected\": \"거절됨\",\n \"workbench.search_placeholder\": \"주체/술어/객체 검색...\",\n \"workbench.predicate_filter\": \"관계\",\n \"workbench.predicate_all\": \"전체 관계\",\n \"workbench.empty\": \"표시할 클레임이 없습니다.\",\n \"workbench.select_hint\": \"왼쪽 목록에서 클레임을 선택하세요.\",\n \"workbench.bulk_count\": \"{count}개 선택됨\",\n \"workbench.action.accept\": \"승인\",\n \"workbench.action.reject\": \"거절\",\n \"workbench.action.unreview\": \"리뷰 취소\",\n \"workbench.action.save_confidence\": \"신뢰도 저장\",\n \"workbench.action.accept_similar\": \"동일 술어+객체 일괄 승인\",\n \"workbench.shortcuts\": \"단축키\",\n \"workbench.shortcut.nav\": \"J/K · 이동\",\n \"workbench.shortcut.accept\": \"A · 승인\",\n \"workbench.shortcut.reject\": \"R · 거절\",\n \"workbench.shortcut.bulk_accept\": \"Shift+A · 선택 일괄 승인\",\n \"workbench.shortcut.bulk_reject\": \"Shift+R · 선택 일괄 거절\",\n \"workbench.shortcut.toggle\": \"Space · 선택 토글\",\n \"workbench.shortcut.search\": \"/ · 검색\",\n \"workbench.shortcut.escape\": \"Esc · 선택 해제\",\n \"workbench.detail.subject\": \"주체\",\n \"workbench.detail.predicate\": \"술어\",\n \"workbench.detail.object\": \"객체\",\n \"workbench.detail.status\": \"상태\",\n \"workbench.detail.confidence\": \"신뢰도\",\n \"workbench.detail.evidence\": \"근거 텍스트\",\n \"workbench.detail.source\": \"출처\",\n \"workbench.detail.page\": \"페이지\",\n \"workbench.detail.breakdown\": \"신뢰도 분해\",\n \"workbench.detail.reason_placeholder\": \"사유 (선택)\",\n \"workbench.toast.reviewed\": \"리뷰 적용됨\",\n \"workbench.toast.bulk_done\": \"{count}개 적용됨\",\n\n \"debug.refresh_logs\": \"추출 로그 새로고침\",\n\n \"research.heading\": \"그래프 리서치\",\n \"research.goal\": \"목표\",\n \"research.seed_entity\": \"시드 엔티티 ID\",\n \"research.max_steps\": \"최대 스텝\",\n \"research.min_relevance\": \"최소 관련도\",\n \"research.run\": \"그래프 리서치 실행\",\n \"research.load_sessions\": \"세션 불러오기\",\n \"research.graph_query\": \"그래프 쿼리\",\n \"research.filter_placeholder\": \"브랜드 또는 태그 필터\",\n \"research.run_query\": \"쿼리 실행\",\n \"research.query.trend_summary\": \"트렌드 요약\",\n \"research.query.brand_products\": \"브랜드 상품\",\n \"research.query.products_by_tag\": \"태그별 상품\",\n \"research.query.relation_summary\": \"관계 요약\",\n \"research.query.entity_type_summary\": \"엔티티 타입 요약\",\n\n \"tags.load\": \"태그 불러오기\",\n\n \"recommend.preferred_notes\": \"선호 노트\",\n \"recommend.avoided_notes\": \"기피 노트\",\n \"recommend.preferred_moods\": \"선호 무드\",\n \"recommend.season\": \"계절\",\n \"recommend.occasion\": \"상황\",\n \"recommend.test\": \"추천 테스트\",\n\n \"inspector.title\": \"인스펙터\",\n \"inspector.hint\": \"엔티티/클레임/페이지를 선택하면 상세가 표시됩니다.\",\n \"inspector.project_info\": \"프로젝트 정보\",\n \"inspector.no_selection\": \"선택 없음\",\n \"inspector.latest_activity\": \"최근 활동\",\n \"inspector.latest_activity.empty\": \"최근 크롤/추출 활동이 없습니다.\",\n\n \"extractor.rule_based\": \"규칙 기반\",\n \"extractor.openai\": \"OpenAI API\",\n \"extractor.ollama\": \"Ollama\",\n \"extractor.lm_studio\": \"LM Studio\",\n\n \"toast.project_created\": \"프로젝트 생성됨\",\n \"toast.reset_confirm\": \"현재 프로젝트의 크롤 데이터(페이지/엔티티/클레임)를 리셋하시겠습니까?\",\n \"toast.reset_done\": \"리셋 완료\",\n \"toast.url_copied\": \"URL이 입력란에 복사됨\",\n \"toast.url_crawl_completed\": \"URL 크롤 완료\",\n \"toast.crawl_failed\": \"크롤 실패\",\n \"toast.site_crawl_started\": \"사이트 크롤 시작\",\n \"toast.site_crawl_failed\": \"사이트 크롤 실패\",\n \"toast.site_crawl_completed\": \"사이트 크롤 완료\",\n \"toast.stop_requested\": \"중지 요청됨\",\n \"toast.stop_failed\": \"중지 실패\",\n \"toast.analyzer_connected\": \"분석기 연결됨\",\n \"toast.analyzer_failed\": \"분석기 실패\",\n \"toast.discovery_failed\": \"탐색 실패\",\n \"toast.claim_updated\": \"클레임 업데이트됨\",\n \"toast.entity_merged\": \"엔티티 병합됨\",\n \"toast.merge_check\": \"엔티티 ID를 확인하세요\",\n \"toast.research_completed\": \"리서치 완료\",\n \"toast.research_failed\": \"리서치 실패\",\n \"toast.config_path_required\": \"설정 파일 경로가 필요합니다\",\n\n \"status.starting_site_crawl\": \"시드부터 사이트 크롤 시작 중...\",\n \"status.discovering\": \"링크 탐색 중...\",\n \"status.testing_analyzer\": \"분석기 테스트 중...\",\n \"status.crawling_url\": \"URL 1건 크롤 중...\",\n \"status.stopping_crawl\": \"사이트 크롤 중지 중\",\n \"status.research_running\": \"그래프 리서치 실행 중...\",\n \"status.no_crawl\": \"준비됨\",\n },\n en: {\n \"app.title\": \"Ontology Crawler\",\n \"app.subtitle\": \"Project crawler, ontology mapping, claim review, recommendation tags\",\n \"header.refresh\": \"Refresh\",\n \"header.lang.ko\": \"한\",\n \"header.lang.en\": \"EN\",\n\n \"sidebar.projects\": \"Projects\",\n \"sidebar.config_path\": \"Config path\",\n \"sidebar.create_project\": \"Create project\",\n \"sidebar.reset_project\": \"Reset current project data\",\n \"sidebar.crawl\": \"Crawl\",\n \"sidebar.source\": \"Source\",\n \"sidebar.url_or_seed\": \"URL / Seed URL\",\n \"sidebar.analyzer\": \"Analyzer\",\n \"sidebar.model\": \"Model\",\n \"sidebar.base_url\": \"Base URL\",\n \"sidebar.test_analyzer\": \"Test analyzer\",\n \"sidebar.discover\": \"Discover links\",\n \"sidebar.crawl_url\": \"Crawl URL\",\n \"sidebar.max_depth\": \"Max depth\",\n \"sidebar.max_pages\": \"Max pages\",\n \"sidebar.same_domain\": \"Same domain\",\n \"sidebar.respect_robots\": \"Respect robots.txt\",\n \"sidebar.crawl_site\": \"Crawl site from seed\",\n \"sidebar.stop_crawl\": \"Stop current crawl\",\n\n \"tabs.overview\": \"Overview\",\n \"tabs.graph\": \"Graph\",\n \"tabs.ontology\": \"Ontology\",\n \"tabs.entities\": \"Entities\",\n \"tabs.claims\": \"Claims\",\n \"tabs.debug\": \"Debug\",\n \"tabs.research\": \"Research\",\n \"tabs.tags\": \"Tags\",\n \"tabs.recommend\": \"Recommend\",\n\n \"graph.search_placeholder\": \"Search entity name...\",\n \"graph.entity_type_all\": \"All types\",\n \"graph.predicate_all\": \"All relations\",\n \"graph.min_confidence\": \"Min confidence\",\n \"graph.layout\": \"Layout\",\n \"graph.layout.cose\": \"Force\",\n \"graph.layout.circle\": \"Circle\",\n \"graph.layout.concentric\": \"Concentric\",\n \"graph.layout.grid\": \"Grid\",\n \"graph.layout.breadthfirst\": \"Tree\",\n \"graph.reload\": \"Reload graph\",\n \"graph.fit\": \"Fit\",\n \"graph.empty\": \"No data yet. As crawling produces claims, nodes will appear on the graph.\",\n \"graph.stats.nodes\": \"nodes\",\n \"graph.stats.edges\": \"edges\",\n \"graph.stats.shown\": \"shown\",\n\n \"overview.project\": \"Project\",\n \"overview.domain\": \"Domain\",\n \"overview.sources\": \"Sources\",\n \"overview.entities\": \"Entities\",\n \"overview.sources_heading\": \"Sources\",\n\n \"pipeline.heading\": \"Pipeline\",\n \"pipeline.refresh\": \"Refresh\",\n \"pipeline.stage.crawled\": \"Crawled\",\n \"pipeline.stage.extracted\": \"Extracted\",\n \"pipeline.stage.claims\": \"Claims\",\n \"pipeline.stage.validated\": \"Validated\",\n \"pipeline.stage.graph\": \"Graph triples\",\n \"pipeline.retained\": \"retained\",\n \"pipeline.entity_types_heading\": \"Entity type breakdown\",\n \"pipeline.recent_pages\": \"Recent pages\",\n \"pipeline.recent_claims\": \"Recent claims\",\n \"pipeline.empty.pages\": \"No pages crawled yet.\",\n \"pipeline.empty.claims\": \"No claims generated yet.\",\n\n \"search.placeholder\": \"Search entities, claims, pages...\",\n \"search.aria_label\": \"Global search\",\n \"search.group.entities\": \"Entities\",\n \"search.group.claims\": \"Claims\",\n \"search.group.pages\": \"Pages\",\n \"search.group.predicates\": \"Predicates\",\n \"search.empty\": \"No results\",\n \"search.hint\": \"Type at least 1 character\",\n \"table.name\": \"Name\",\n \"table.type\": \"Type\",\n \"table.trust\": \"Trust\",\n \"table.robots\": \"Robots\",\n \"table.rate\": \"Rate\",\n \"table.domain\": \"Domain\",\n \"table.status\": \"Status\",\n \"table.confidence\": \"Confidence\",\n \"table.relation\": \"Relation\",\n \"table.subject\": \"Subject\",\n \"table.object\": \"Object\",\n \"table.support\": \"Support\",\n \"table.predicate\": \"Predicate\",\n \"table.tag\": \"Tag\",\n \"table.brand\": \"Brand\",\n \"table.product\": \"Product\",\n \"table.count\": \"Count\",\n \"table.no_data\": \"No data.\",\n \"table.id\": \"ID\",\n \"table.metadata\": \"Metadata\",\n \"table.reason\": \"Reason\",\n \"table.gap\": \"Gap\",\n \"table.target\": \"Target\",\n \"table.priority\": \"Priority\",\n \"table.description\": \"Description\",\n \"table.score\": \"Score\",\n \"table.reasons\": \"Reasons\",\n\n \"ontology.refresh_registry\": \"Refresh registry\",\n \"ontology.configured_entity_types\": \"Configured Entity Types\",\n \"ontology.configured_predicates\": \"Configured Predicates\",\n \"ontology.registry_entity_types\": \"Registry Entity Types\",\n \"ontology.registry_relation_types\": \"Registry Relation Types\",\n \"ontology.triples\": \"Ontology Triples\",\n \"ontology.proposals\": \"Schema Proposals\",\n \"ontology.knowledge_gaps\": \"Knowledge Gaps\",\n\n \"entities.all_types\": \"All types\",\n \"entities.load\": \"Load\",\n \"entities.merge_source_placeholder\": \"Entity ID to merge\",\n \"entities.merge_target_placeholder\": \"Entity ID to keep\",\n \"entities.merge\": \"Merge\",\n\n \"claims.show_candidates\": \"Show candidates\",\n \"claims.refresh\": \"Refresh claims\",\n \"claims.save\": \"Save\",\n \"claims.confidence_label\": \"confidence\",\n \"claims.reason_label\": \"reason\",\n\n \"workbench.status_filter\": \"Status\",\n \"workbench.status.all\": \"All\",\n \"workbench.status.active\": \"Active\",\n \"workbench.status.validated_claim\": \"Validated\",\n \"workbench.status.rejected\": \"Rejected\",\n \"workbench.search_placeholder\": \"Search subject/predicate/object...\",\n \"workbench.predicate_filter\": \"Predicate\",\n \"workbench.predicate_all\": \"All predicates\",\n \"workbench.empty\": \"No claims to display.\",\n \"workbench.select_hint\": \"Select a claim from the list on the left.\",\n \"workbench.bulk_count\": \"{count} selected\",\n \"workbench.action.accept\": \"Accept\",\n \"workbench.action.reject\": \"Reject\",\n \"workbench.action.unreview\": \"Unreview\",\n \"workbench.action.save_confidence\": \"Save confidence\",\n \"workbench.action.accept_similar\": \"Accept all with same predicate+object\",\n \"workbench.shortcuts\": \"Shortcuts\",\n \"workbench.shortcut.nav\": \"J/K · navigate\",\n \"workbench.shortcut.accept\": \"A · accept\",\n \"workbench.shortcut.reject\": \"R · reject\",\n \"workbench.shortcut.bulk_accept\": \"Shift+A · bulk accept\",\n \"workbench.shortcut.bulk_reject\": \"Shift+R · bulk reject\",\n \"workbench.shortcut.toggle\": \"Space · toggle select\",\n \"workbench.shortcut.search\": \"/ · search\",\n \"workbench.shortcut.escape\": \"Esc · clear selection\",\n \"workbench.detail.subject\": \"Subject\",\n \"workbench.detail.predicate\": \"Predicate\",\n \"workbench.detail.object\": \"Object\",\n \"workbench.detail.status\": \"Status\",\n \"workbench.detail.confidence\": \"Confidence\",\n \"workbench.detail.evidence\": \"Evidence text\",\n \"workbench.detail.source\": \"Source\",\n \"workbench.detail.page\": \"Page\",\n \"workbench.detail.breakdown\": \"Confidence breakdown\",\n \"workbench.detail.reason_placeholder\": \"reason (optional)\",\n \"workbench.toast.reviewed\": \"Review applied\",\n \"workbench.toast.bulk_done\": \"{count} updated\",\n\n \"debug.refresh_logs\": \"Refresh extraction logs\",\n\n \"research.heading\": \"Semantic Exploration\",\n \"research.goal\": \"Goal\",\n \"research.seed_entity\": \"Seed entity ID\",\n \"research.max_steps\": \"Max steps\",\n \"research.min_relevance\": \"Min relevance\",\n \"research.run\": \"Run graph research\",\n \"research.load_sessions\": \"Load sessions\",\n \"research.graph_query\": \"Graph Query\",\n \"research.filter_placeholder\": \"brand or tag filter\",\n \"research.run_query\": \"Run query\",\n \"research.query.trend_summary\": \"Trend summary\",\n \"research.query.brand_products\": \"Brand products\",\n \"research.query.products_by_tag\": \"Products by tag\",\n \"research.query.relation_summary\": \"Relation summary\",\n \"research.query.entity_type_summary\": \"Entity type summary\",\n\n \"tags.load\": \"Load tags\",\n\n \"recommend.preferred_notes\": \"Preferred notes\",\n \"recommend.avoided_notes\": \"Avoided notes\",\n \"recommend.preferred_moods\": \"Preferred moods\",\n \"recommend.season\": \"Season\",\n \"recommend.occasion\": \"Occasion\",\n \"recommend.test\": \"Test recommendation\",\n\n \"inspector.title\": \"Inspector\",\n \"inspector.hint\": \"Select an entity/claim/page to see details.\",\n \"inspector.project_info\": \"Project info\",\n \"inspector.no_selection\": \"Nothing selected\",\n \"inspector.latest_activity\": \"Latest activity\",\n \"inspector.latest_activity.empty\": \"No recent crawl or extraction activity.\",\n\n \"extractor.rule_based\": \"Rule-based\",\n \"extractor.openai\": \"OpenAI API\",\n \"extractor.ollama\": \"Ollama\",\n \"extractor.lm_studio\": \"LM Studio\",\n\n \"toast.project_created\": \"Project created\",\n \"toast.reset_confirm\": \"Reset current project crawl data (pages/entities/claims)?\",\n \"toast.reset_done\": \"Reset completed\",\n \"toast.url_copied\": \"URL copied to input\",\n \"toast.url_crawl_completed\": \"URL crawl completed\",\n \"toast.crawl_failed\": \"Crawl failed\",\n \"toast.site_crawl_started\": \"Site crawl started\",\n \"toast.site_crawl_failed\": \"Site crawl failed\",\n \"toast.site_crawl_completed\": \"Site crawl completed\",\n \"toast.stop_requested\": \"Stop requested\",\n \"toast.stop_failed\": \"Stop failed\",\n \"toast.analyzer_connected\": \"Analyzer connected\",\n \"toast.analyzer_failed\": \"Analyzer failed\",\n \"toast.discovery_failed\": \"Discovery failed\",\n \"toast.claim_updated\": \"Claim updated\",\n \"toast.entity_merged\": \"Entity merged\",\n \"toast.merge_check\": \"Check entity IDs\",\n \"toast.research_completed\": \"Research completed\",\n \"toast.research_failed\": \"Research failed\",\n \"toast.config_path_required\": \"Config path is required\",\n\n \"status.starting_site_crawl\": \"Starting site crawl from seed...\",\n \"status.discovering\": \"Discovering links...\",\n \"status.testing_analyzer\": \"Testing analyzer...\",\n \"status.crawling_url\": \"Crawling one URL...\",\n \"status.stopping_crawl\": \"Stopping site crawl\",\n \"status.research_running\": \"Running graph research...\",\n \"status.no_crawl\": \"Ready\",\n },\n};\n\nlet current = pickInitialLang();\nconst listeners = new Set();\n\nfunction pickInitialLang() {\n try {\n const stored = localStorage.getItem(STORAGE_KEY);\n if (stored && SUPPORTED.includes(stored)) return stored;\n } catch {}\n const navLang = (navigator.language || \"\").toLowerCase();\n if (navLang.startsWith(\"en\")) return \"en\";\n return DEFAULT_LANG;\n}\n\nexport function getLang() {\n return current;\n}\n\nexport function setLang(next) {\n if (!SUPPORTED.includes(next) || next === current) return;\n current = next;\n try { localStorage.setItem(STORAGE_KEY, next); } catch {}\n document.documentElement.setAttribute(\"lang\", next);\n applyI18n(document);\n listeners.forEach((fn) => {\n try { fn(next); } catch (error) { console.warn(\"i18n listener failed\", error); }\n });\n}\n\nexport function onLangChange(fn) {\n listeners.add(fn);\n return () => listeners.delete(fn);\n}\n\nexport function t(key, fallback) {\n const table = dict[current] || dict[DEFAULT_LANG];\n if (key in table) return table[key];\n if (fallback != null) return fallback;\n return key;\n}\n\nexport function applyI18n(root = document) {\n root.querySelectorAll(\"[data-i18n]\").forEach((node) => {\n const key = node.getAttribute(\"data-i18n\");\n if (!key) return;\n node.textContent = t(key);\n });\n root.querySelectorAll(\"[data-i18n-placeholder]\").forEach((node) => {\n const key = node.getAttribute(\"data-i18n-placeholder\");\n if (!key) return;\n node.setAttribute(\"placeholder\", t(key));\n });\n root.querySelectorAll(\"[data-i18n-title]\").forEach((node) => {\n const key = node.getAttribute(\"data-i18n-title\");\n if (!key) return;\n node.setAttribute(\"title\", t(key));\n });\n root.querySelectorAll(\"[data-i18n-aria-label]\").forEach((node) => {\n const key = node.getAttribute(\"data-i18n-aria-label\");\n if (!key) return;\n node.setAttribute(\"aria-label\", t(key));\n });\n}\n","import { getLang, setLang, t, applyI18n, onLangChange } from \"./i18n.js\";\n\nexport function mountShell(root) {\n root.innerHTML = `\n
\n
\n

\n

\n
\n
\n
\n \n \n \n \n \n
\n
\n \n \n
\n \n
\n
\n\n
\n \n
\n \n
\n\n
\n `;\n\n document.documentElement.setAttribute(\"lang\", getLang());\n applyI18n(root);\n\n const langButtons = root.querySelectorAll(\".lang-btn\");\n const syncLangButtons = () => {\n const current = getLang();\n langButtons.forEach((btn) => {\n btn.classList.toggle(\"active\", btn.dataset.lang === current);\n });\n };\n langButtons.forEach((btn) => {\n btn.addEventListener(\"click\", () => setLang(btn.dataset.lang));\n });\n syncLangButtons();\n onLangChange(syncLangButtons);\n\n return {\n sidebarHost: document.getElementById(\"sidebarHost\"),\n workspaceHost: document.getElementById(\"workspaceHost\"),\n inspectorHost: document.getElementById(\"inspectorHost\"),\n };\n}\n","export async function api(path, options = {}) {\n const response = await fetch(path, {\n headers: { \"Content-Type\": \"application/json\" },\n ...options,\n });\n if (!response.ok) {\n let detail = `${response.status} ${response.statusText}`;\n try {\n const body = await response.json();\n detail = body.detail || body.error || detail;\n } catch {\n // Keep status text.\n }\n throw new Error(detail);\n }\n return response.json();\n}\n","export const state = {\n projects: [],\n selectedProject: null,\n projectDetail: null,\n ontology: null,\n\n siteCrawlPoll: null,\n siteCrawlPageCount: 0,\n siteCrawlPollErrorCount: 0,\n activeSiteCrawlJobId: null,\n\n latestActivity: null,\n\n selection: null,\n};\n","import { t } from \"./i18n.js\";\n\nexport const $ = (id) => document.getElementById(id);\n\nexport function csv(value) {\n return String(value || \"\").split(\",\").map((item) => item.trim()).filter(Boolean);\n}\n\nexport function escapeHtml(value) {\n return String(value ?? \"\")\n .replaceAll(\"&\", \"&\")\n .replaceAll(\"<\", \"<\")\n .replaceAll(\">\", \">\")\n .replaceAll('\"', \""\")\n .replaceAll(\"'\", \"'\");\n}\n\nexport function chip(value) {\n return `${escapeHtml(value)}`;\n}\n\nexport function table(headers, rows) {\n if (!rows.length) {\n return `
${escapeHtml(t(\"table.no_data\"))}
`;\n }\n return `\n \n ${headers.map((header) => ``).join(\"\")}\n ${rows.map((row) => `${row.map((cell) => ``).join(\"\")}`).join(\"\")}\n
${escapeHtml(header)}
${String(cell ?? \"\")}
\n `;\n}\n\nlet toastTimer = null;\nexport function toast(message) {\n const node = document.getElementById(\"toast\");\n if (!node) return;\n node.textContent = message;\n node.classList.add(\"show\");\n if (toastTimer) window.clearTimeout(toastTimer);\n toastTimer = window.setTimeout(() => node.classList.remove(\"show\"), 2400);\n}\n","import { api } from \"./api.js\";\nimport { state } from \"./state.js\";\nimport { $, escapeHtml, toast } from \"./utils.js\";\nimport { t, applyI18n, onLangChange } from \"./i18n.js\";\n\nexport function mountSidebar(host, { onProjectChanged, onCrawlUpdate, refreshAll }) {\n host.innerHTML = sidebarHtml();\n applyI18n(host);\n\n $(\"createProjectBtn\").addEventListener(\"click\", () => createProject(onProjectChanged));\n $(\"initProjectBtn\").addEventListener(\"click\", () => initializeCurrentProject(onProjectChanged));\n $(\"discoverBtn\").addEventListener(\"click\", discover);\n $(\"crawlBtn\").addEventListener(\"click\", () => crawlOnce(refreshAll));\n $(\"siteCrawlBtn\").addEventListener(\"click\", () => crawlSite(onCrawlUpdate, refreshAll));\n $(\"stopSiteCrawlBtn\").addEventListener(\"click\", stopSiteCrawl);\n $(\"extractorProvider\").addEventListener(\"change\", updateExtractorOptions);\n $(\"testExtractorBtn\").addEventListener(\"click\", () => testExtractor({ announce: true }));\n\n updateExtractorOptions();\n connectDefaultAnalyzer();\n\n onLangChange(() => {\n applyI18n(host);\n renderProjects();\n renderSiteCrawlProgress(latestJob);\n });\n}\n\nfunction sidebarHtml() {\n return `\n
\n

\n
\n \n \n
\n \n
\n
\n\n
\n

\n \n \n \n
\n \n \n \n
\n
\n \n \n
\n
\n \n \n \n \n
\n \n \n
\n
\n
\n `;\n}\n\nexport function requestBase() {\n return {\n config_path: $(\"configPath\").value.trim(),\n source_name: $(\"sourceSelect\").value,\n url: $(\"crawlUrl\").value.trim(),\n extractor_provider: $(\"extractorProvider\").value,\n extractor_model: $(\"extractorModel\").value.trim() || null,\n extractor_base_url: $(\"extractorBaseUrl\").value.trim() || null,\n check_robots_txt: $(\"respectRobotsTxt\")?.checked ?? false,\n };\n}\n\nexport function renderProjects() {\n const list = $(\"projectList\");\n if (!list) return;\n list.innerHTML = \"\";\n state.projects.forEach((project) => {\n const button = document.createElement(\"button\");\n button.className = `project-item ${state.selectedProject === project.name ? \"active\" : \"\"}`;\n button.innerHTML = `${escapeHtml(project.name)}${escapeHtml(project.domain)}`;\n button.addEventListener(\"click\", () => {\n const event = new CustomEvent(\"project:select\", { detail: project.name });\n window.dispatchEvent(event);\n });\n list.appendChild(button);\n });\n}\n\nexport function renderSourceOptions(detail) {\n const sourceSelect = $(\"sourceSelect\");\n if (!sourceSelect) return;\n sourceSelect.innerHTML = \"\";\n (detail?.sources ?? []).forEach((source) => {\n const option = document.createElement(\"option\");\n option.value = source.name;\n option.textContent = `${source.name} (${source.type})`;\n sourceSelect.appendChild(option);\n });\n}\n\nasync function createProject(onProjectChanged) {\n const configPath = $(\"configPath\").value.trim();\n if (!configPath) return;\n try {\n const result = await api(\"/projects\", {\n method: \"POST\",\n body: JSON.stringify({ config_path: configPath }),\n });\n toast(`${t(\"toast.project_created\")}: ${result.name}`);\n state.selectedProject = result.name;\n await onProjectChanged?.();\n } catch (error) {\n toast(error.message);\n }\n}\n\nasync function initializeCurrentProject(onProjectChanged) {\n const configPath = $(\"configPath\").value.trim();\n if (!configPath) {\n toast(t(\"toast.config_path_required\"));\n return;\n }\n if (!window.confirm(t(\"toast.reset_confirm\"))) return;\n try {\n const result = await api(\"/projects/reset\", {\n method: \"POST\",\n body: JSON.stringify({\n config_path: configPath,\n project_name: state.selectedProject,\n }),\n });\n state.selectedProject = result.name;\n await onProjectChanged?.();\n if (result.reset) {\n const c = result.deleted?.claims ?? 0;\n const e = result.deleted?.entities ?? 0;\n const p = result.deleted?.pages ?? 0;\n $(\"crawlResult\").textContent = `${t(\"toast.reset_done\")}: pages ${p}, entities ${e}, claims ${c}`;\n toast(`${t(\"toast.reset_done\")}: ${result.name}`);\n } else {\n $(\"crawlResult\").textContent = `${t(\"toast.project_created\")}: ${result.name}`;\n toast(`${t(\"toast.project_created\")}: ${result.name}`);\n }\n } catch (error) {\n toast(error.message);\n }\n}\n\nasync function crawlOnce(refreshAll) {\n if (!state.selectedProject) return;\n $(\"crawlResult\").textContent = t(\"status.crawling_url\");\n try {\n const result = await api(\"/crawl\", {\n method: \"POST\",\n body: JSON.stringify(requestBase()),\n });\n $(\"crawlResult\").textContent =\n `URL done: page ${result.page_id}, ${result.crawl_status}/${result.extraction_status}, ${result.page_type}, raw ${result.raw_text_length}, clean ${result.clean_text_length}, claims ${result.claim_count}, entities ${result.entity_count}`;\n state.latestActivity = { kind: \"url_crawl\", result };\n toast(t(\"toast.url_crawl_completed\"));\n await refreshAll?.();\n } catch (error) {\n $(\"crawlResult\").textContent = `${t(\"toast.crawl_failed\")}: ${error.message}`;\n toast(t(\"toast.crawl_failed\"));\n }\n}\n\nlet latestJob = null;\n\nasync function crawlSite(onCrawlUpdate, refreshAll) {\n if (!state.selectedProject) return;\n stopSiteCrawlPolling();\n state.siteCrawlPageCount = 0;\n state.siteCrawlPollErrorCount = 0;\n $(\"crawlResult\").textContent = t(\"status.starting_site_crawl\");\n $(\"discoveredLinks\").innerHTML = \"\";\n try {\n const job = await api(\"/crawl-site\", {\n method: \"POST\",\n body: JSON.stringify({\n ...requestBase(),\n max_depth: Number($(\"siteMaxDepth\").value || 0),\n max_pages: Number($(\"siteMaxPages\").value || 1),\n same_domain_only: $(\"sameDomainOnly\").checked,\n analyze_page_types: [\"ProductPage\", \"BrandStoryPage\", \"ReviewPage\"],\n }),\n });\n $(\"crawlResult\").textContent = `Site crawl queued: job ${job.job_id}`;\n state.activeSiteCrawlJobId = job.job_id;\n $(\"stopSiteCrawlBtn\").disabled = false;\n renderSiteCrawlProgress(job);\n pollSiteCrawl(job.job_id, onCrawlUpdate, refreshAll);\n toast(t(\"toast.site_crawl_started\"));\n } catch (error) {\n $(\"crawlResult\").textContent = `${t(\"toast.site_crawl_failed\")}: ${error.message}`;\n toast(t(\"toast.site_crawl_failed\"));\n }\n}\n\nasync function stopSiteCrawl() {\n const jobId = state.activeSiteCrawlJobId;\n if (!jobId) return;\n $(\"stopSiteCrawlBtn\").disabled = true;\n $(\"crawlResult\").textContent = `${t(\"status.stopping_crawl\")}: job ${jobId}`;\n try {\n const job = await api(`/crawl-site/jobs/${jobId}/cancel`, { method: \"POST\" });\n renderSiteCrawlProgress(job);\n toast(t(\"toast.stop_requested\"));\n } catch (error) {\n $(\"stopSiteCrawlBtn\").disabled = false;\n $(\"crawlResult\").textContent = `${t(\"toast.stop_failed\")}: ${error.message}`;\n toast(t(\"toast.stop_failed\"));\n }\n}\n\nfunction stopSiteCrawlPolling() {\n if (state.siteCrawlPoll) {\n window.clearTimeout(state.siteCrawlPoll);\n state.siteCrawlPoll = null;\n }\n}\n\nasync function pollSiteCrawl(jobId, onCrawlUpdate, refreshAll) {\n let job;\n try {\n job = await api(`/crawl-site/jobs/${jobId}`);\n } catch (error) {\n const transient = state.siteCrawlPollErrorCount = (state.siteCrawlPollErrorCount ?? 0) + 1;\n $(\"crawlResult\").textContent = `Site crawl status check failed (retry ${transient}): ${error.message}`;\n if (transient >= 5) {\n stopSiteCrawlPolling();\n toast(t(\"toast.site_crawl_failed\"));\n return;\n }\n state.siteCrawlPoll = window.setTimeout(() => pollSiteCrawl(jobId, onCrawlUpdate, refreshAll), 3000);\n return;\n }\n state.siteCrawlPollErrorCount = 0;\n\n const progress = job.progress ?? {};\n const pageCount = progress.pages?.length ?? 0;\n renderSiteCrawlProgress(job);\n onCrawlUpdate?.(job);\n if (pageCount !== state.siteCrawlPageCount) {\n state.siteCrawlPageCount = pageCount;\n try { await refreshAll?.(); } catch (error) { console.warn(\"refreshAll during poll failed\", error); }\n }\n if ([\"completed\", \"failed\", \"canceled\"].includes(job.status)) {\n stopSiteCrawlPolling();\n state.activeSiteCrawlJobId = null;\n $(\"stopSiteCrawlBtn\").disabled = true;\n toast(job.status === \"completed\" ? t(\"toast.site_crawl_completed\") : `${t(\"toast.site_crawl_failed\")}: ${job.status}`);\n return;\n }\n state.siteCrawlPoll = window.setTimeout(() => pollSiteCrawl(jobId, onCrawlUpdate, refreshAll), 1500);\n}\n\nfunction renderSiteCrawlProgress(job) {\n latestJob = job;\n if (!job) return;\n const progress = job.progress ?? {};\n const pages = progress.pages ?? [];\n $(\"crawlResult\").textContent =\n `Site ${job.status}: visited ${progress.visited_count ?? 0}, analyzed ${progress.analyzed_count ?? 0}, skipped ${progress.skipped_count ?? 0}, queued ${progress.queued_count ?? 0}`;\n $(\"discoveredLinks\").innerHTML = pages.map(renderSitePage).join(\"\");\n document.querySelectorAll(\"[data-discovered-url]\").forEach((button) => {\n button.addEventListener(\"click\", () => {\n $(\"crawlUrl\").value = button.dataset.discoveredUrl;\n toast(t(\"toast.url_copied\"));\n });\n });\n state.latestActivity = { kind: \"site_crawl\", job };\n}\n\nfunction renderSitePage(page) {\n const diagnostics = `raw ${page.raw_text_length ?? 0}, clean ${page.clean_text_length ?? 0}, removed ${page.removed_noise_zones_count ?? 0}`;\n const warnings = (page.warnings ?? []).length ? `, warnings: ${(page.warnings ?? []).join(\"; \")}` : \"\";\n return `\n \n `;\n}\n\nasync function discover() {\n $(\"crawlResult\").textContent = t(\"status.discovering\");\n $(\"discoveredLinks\").innerHTML = \"\";\n try {\n const result = await api(\"/discover\", {\n method: \"POST\",\n body: JSON.stringify({\n config_path: $(\"configPath\").value.trim(),\n source_name: $(\"sourceSelect\").value,\n url: $(\"crawlUrl\").value.trim(),\n limit: 30,\n check_robots_txt: $(\"respectRobotsTxt\")?.checked ?? false,\n }),\n });\n if (!result.ok) {\n $(\"crawlResult\").textContent = result.error ?? t(\"toast.discovery_failed\");\n return;\n }\n $(\"crawlResult\").textContent = `Discovered ${result.links.length} links`;\n $(\"discoveredLinks\").innerHTML = result.links.map(renderDiscoveredLink).join(\"\");\n document.querySelectorAll(\"[data-discovered-url]\").forEach((button) => {\n button.addEventListener(\"click\", () => {\n $(\"crawlUrl\").value = button.dataset.discoveredUrl;\n toast(t(\"toast.url_copied\"));\n });\n });\n } catch (error) {\n $(\"crawlResult\").textContent = `${t(\"toast.discovery_failed\")}: ${error.message}`;\n toast(t(\"toast.discovery_failed\"));\n }\n}\n\nfunction renderDiscoveredLink(link) {\n return `\n \n `;\n}\n\nfunction updateExtractorOptions() {\n const provider = $(\"extractorProvider\").value;\n $(\"extractorOptions\").classList.toggle(\"active\", provider !== \"rule_based\");\n if (provider === \"ollama\" && !$(\"extractorBaseUrl\").value.trim()) {\n $(\"extractorBaseUrl\").placeholder = \"http://localhost:11434/api/chat\";\n } else if (provider === \"lm_studio\" && !$(\"extractorBaseUrl\").value.trim()) {\n $(\"extractorBaseUrl\").value = \"http://localhost:1234/v1\";\n $(\"extractorBaseUrl\").placeholder = \"http://localhost:1234/v1\";\n } else {\n $(\"extractorBaseUrl\").placeholder = \"optional provider endpoint\";\n }\n}\n\nasync function testExtractor({ announce = true } = {}) {\n const provider = $(\"extractorProvider\").value;\n const baseUrl = $(\"extractorBaseUrl\").value.trim();\n if (announce) $(\"crawlResult\").textContent = t(\"status.testing_analyzer\");\n try {\n const result = await api(\"/extractors/models\", {\n method: \"POST\",\n body: JSON.stringify({ provider, base_url: baseUrl || null }),\n });\n if (!result.ok) {\n if (announce) {\n $(\"crawlResult\").textContent = `${t(\"toast.analyzer_failed\")}: ${result.error}`;\n toast(t(\"toast.analyzer_failed\"));\n }\n return;\n }\n const models = result.models ?? [];\n if (models.length && !$(\"extractorModel\").value.trim()) {\n $(\"extractorModel\").value = models[0].id;\n }\n if (announce) {\n $(\"crawlResult\").textContent = models.length\n ? `${t(\"toast.analyzer_connected\")}. Models: ${models.map((m) => m.id).join(\", \")}`\n : `${t(\"toast.analyzer_connected\")}. No models returned.`;\n toast(t(\"toast.analyzer_connected\"));\n }\n } catch (error) {\n if (announce) toast(error.message);\n }\n}\n\nasync function connectDefaultAnalyzer() {\n if ($(\"extractorProvider\").value !== \"lm_studio\") return;\n try {\n await testExtractor({ announce: false });\n } catch {\n // LM Studio may not be running yet.\n }\n}\n","import { state } from \"./state.js\";\nimport { escapeHtml } from \"./utils.js\";\nimport { t, applyI18n, onLangChange } from \"./i18n.js\";\n\nlet host = null;\n\nexport function mountInspector(target) {\n host = target;\n render();\n onLangChange(render);\n window.addEventListener(\"inspector:update\", render);\n}\n\nexport function refreshInspector() {\n render();\n}\n\nfunction render() {\n if (!host) return;\n host.innerHTML = template();\n applyI18n(host);\n}\n\nfunction template() {\n const selection = state.selection;\n return `\n
\n

\n

\n
\n ${selection ? renderSelection(selection) : renderDefault()}\n `;\n}\n\nfunction renderDefault() {\n const detail = state.projectDetail;\n const activity = state.latestActivity;\n return `\n
\n

\n ${detail ? `\n
\n
${escapeHtml(t(\"overview.project\"))}
${escapeHtml(detail.name)}
\n
${escapeHtml(t(\"overview.domain\"))}
${escapeHtml(detail.domain)}
\n
${escapeHtml(t(\"overview.sources\"))}
${(detail.sources ?? []).length}
\n
\n ` : `

`}\n
\n
\n

\n ${activity ? renderActivity(activity) : `

`}\n
\n `;\n}\n\nfunction renderActivity(activity) {\n if (activity.kind === \"site_crawl\") {\n const job = activity.job ?? {};\n const progress = job.progress ?? {};\n return `\n
\n
job
${escapeHtml(job.job_id ?? \"-\")}
\n
status
${escapeHtml(job.status ?? \"-\")}
\n
visited
${progress.visited_count ?? 0}
\n
analyzed
${progress.analyzed_count ?? 0}
\n
skipped
${progress.skipped_count ?? 0}
\n
queued
${progress.queued_count ?? 0}
\n
\n `;\n }\n if (activity.kind === \"url_crawl\") {\n const r = activity.result ?? {};\n return `\n
\n
page
${escapeHtml(r.page_id ?? \"-\")}
\n
page_type
${escapeHtml(r.page_type ?? \"-\")}
\n
crawl
${escapeHtml(r.crawl_status ?? \"-\")}
\n
extraction
${escapeHtml(r.extraction_status ?? \"-\")}
\n
claims
${r.claim_count ?? 0}
\n
entities
${r.entity_count ?? 0}
\n
\n `;\n }\n return \"\";\n}\n\nfunction renderSelection(selection) {\n if (selection.kind === \"entity\") {\n const e = selection.data;\n return `\n
\n

${escapeHtml(e.name)}

\n
\n
${escapeHtml(t(\"table.id\"))}
${escapeHtml(e.id)}
\n
${escapeHtml(t(\"table.type\"))}
${escapeHtml(e.type)}
\n
\n 
${escapeHtml(JSON.stringify(e.metadata ?? {}, null, 2))}
\n
\n `;\n }\n if (selection.kind === \"claim\") {\n const c = selection.data;\n return `\n
\n

${escapeHtml(c.subject)} · ${escapeHtml(c.predicate)}

\n
\n
${escapeHtml(t(\"table.object\"))}
${escapeHtml(c.object ?? JSON.stringify(c.object_value ?? \"\"))}
\n
${escapeHtml(t(\"table.status\"))}
${escapeHtml(c.status ?? \"\")}
\n
${escapeHtml(t(\"table.confidence\"))}
${Math.round((c.confidence ?? 0) * 100)}%
\n
\n ${c.evidence_text ? `

${escapeHtml(c.evidence_text)}

` : \"\"}\n
\n `;\n }\n if (selection.kind === \"graph_node\") {\n const n = selection.data;\n return `\n
\n

${escapeHtml(n.label)}

\n
\n
${escapeHtml(t(\"table.id\"))}
${escapeHtml(n.entityId)}
\n
${escapeHtml(t(\"table.type\"))}
${escapeHtml(n.type ?? \"\")}
\n ${n.canonical ? `
canonical
${escapeHtml(n.canonical)}
` : \"\"}\n
\n ${n.metadata && Object.keys(n.metadata).length\n ? `
${escapeHtml(JSON.stringify(n.metadata, null, 2))}
`\n : \"\"}\n
\n `;\n }\n if (selection.kind === \"graph_edge\") {\n const e = selection.data;\n return `\n
\n

${escapeHtml(e.predicate)}

\n
\n
${escapeHtml(t(\"table.confidence\"))}
${Math.round((e.confidence ?? 0) * 100)}%
\n ${e.targetName ? `
${escapeHtml(t(\"table.object\"))}
${escapeHtml(e.targetName)}
` : \"\"}\n
claim
${escapeHtml(e.claimId)}
\n
\n ${e.metadata && Object.keys(e.metadata).length\n ? `
${escapeHtml(JSON.stringify(e.metadata, null, 2))}
`\n : \"\"}\n
\n `;\n }\n return \"\";\n}\n","/**\n * Copyright (c) 2016-2026, The Cytoscape Consortium.\n *\n * Permission is hereby granted, free of charge, to any person obtaining a copy of\n * this software and associated documentation files (the “Software”), to deal in\n * the Software without restriction, including without limitation the rights to\n * use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies\n * of the Software, and to permit persons to whom the Software is furnished to do\n * so, subject to the following conditions:\n *\n * The above copyright notice and this permission notice shall be included in all\n * copies or substantial portions of the Software.\n *\n * THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\n * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\n * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n * SOFTWARE.\n */\n\nfunction _arrayLikeToArray(r, a) {\n (null == a || a > r.length) && (a = r.length);\n for (var e = 0, n = Array(a); e < a; e++) n[e] = r[e];\n return n;\n}\nfunction _arrayWithHoles(r) {\n if (Array.isArray(r)) return r;\n}\nfunction _arrayWithoutHoles(r) {\n if (Array.isArray(r)) return _arrayLikeToArray(r);\n}\nfunction _classCallCheck(a, n) {\n if (!(a instanceof n)) throw new TypeError(\"Cannot call a class as a function\");\n}\nfunction _defineProperties(e, r) {\n for (var t = 0; t < r.length; t++) {\n var o = r[t];\n o.enumerable = o.enumerable || false, o.configurable = true, \"value\" in o && (o.writable = true), Object.defineProperty(e, _toPropertyKey(o.key), o);\n }\n}\nfunction _createClass(e, r, t) {\n return r && _defineProperties(e.prototype, r), Object.defineProperty(e, \"prototype\", {\n writable: false\n }), e;\n}\nfunction _createForOfIteratorHelper(r, e) {\n var t = \"undefined\" != typeof Symbol && r[Symbol.iterator] || r[\"@@iterator\"];\n if (!t) {\n if (Array.isArray(r) || (t = _unsupportedIterableToArray(r)) || e) {\n t && (r = t);\n var n = 0,\n F = function () {};\n return {\n s: F,\n n: function () {\n return n >= r.length ? {\n done: true\n } : {\n done: false,\n value: r[n++]\n };\n },\n e: function (r) {\n throw r;\n },\n f: F\n };\n }\n throw new TypeError(\"Invalid attempt to iterate non-iterable instance.\\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.\");\n }\n var o,\n a = true,\n u = false;\n return {\n s: function () {\n t = t.call(r);\n },\n n: function () {\n var r = t.next();\n return a = r.done, r;\n },\n e: function (r) {\n u = true, o = r;\n },\n f: function () {\n try {\n a || null == t.return || t.return();\n } finally {\n if (u) throw o;\n }\n }\n };\n}\nfunction _defineProperty$1(e, r, t) {\n return (r = _toPropertyKey(r)) in e ? Object.defineProperty(e, r, {\n value: t,\n enumerable: true,\n configurable: true,\n writable: true\n }) : e[r] = t, e;\n}\nfunction _iterableToArray(r) {\n if (\"undefined\" != typeof Symbol && null != r[Symbol.iterator] || null != r[\"@@iterator\"]) return Array.from(r);\n}\nfunction _iterableToArrayLimit(r, l) {\n var t = null == r ? null : \"undefined\" != typeof Symbol && r[Symbol.iterator] || r[\"@@iterator\"];\n if (null != t) {\n var e,\n n,\n i,\n u,\n a = [],\n f = true,\n o = false;\n try {\n if (i = (t = t.call(r)).next, 0 === l) {\n if (Object(t) !== t) return;\n f = !1;\n } else for (; !(f = (e = i.call(t)).done) && (a.push(e.value), a.length !== l); f = !0);\n } catch (r) {\n o = true, n = r;\n } finally {\n try {\n if (!f && null != t.return && (u = t.return(), Object(u) !== u)) return;\n } finally {\n if (o) throw n;\n }\n }\n return a;\n }\n}\nfunction _nonIterableRest() {\n throw new TypeError(\"Invalid attempt to destructure non-iterable instance.\\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.\");\n}\nfunction _nonIterableSpread() {\n throw new TypeError(\"Invalid attempt to spread non-iterable instance.\\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.\");\n}\nfunction _slicedToArray(r, e) {\n return _arrayWithHoles(r) || _iterableToArrayLimit(r, e) || _unsupportedIterableToArray(r, e) || _nonIterableRest();\n}\nfunction _toConsumableArray(r) {\n return _arrayWithoutHoles(r) || _iterableToArray(r) || _unsupportedIterableToArray(r) || _nonIterableSpread();\n}\nfunction _toPrimitive(t, r) {\n if (\"object\" != typeof t || !t) return t;\n var e = t[Symbol.toPrimitive];\n if (undefined !== e) {\n var i = e.call(t, r);\n if (\"object\" != typeof i) return i;\n throw new TypeError(\"@@toPrimitive must return a primitive value.\");\n }\n return (String )(t);\n}\nfunction _toPropertyKey(t) {\n var i = _toPrimitive(t, \"string\");\n return \"symbol\" == typeof i ? i : i + \"\";\n}\nfunction _typeof(o) {\n \"@babel/helpers - typeof\";\n\n return _typeof = \"function\" == typeof Symbol && \"symbol\" == typeof Symbol.iterator ? function (o) {\n return typeof o;\n } : function (o) {\n return o && \"function\" == typeof Symbol && o.constructor === Symbol && o !== Symbol.prototype ? \"symbol\" : typeof o;\n }, _typeof(o);\n}\nfunction _unsupportedIterableToArray(r, a) {\n if (r) {\n if (\"string\" == typeof r) return _arrayLikeToArray(r, a);\n var t = {}.toString.call(r).slice(8, -1);\n return \"Object\" === t && r.constructor && (t = r.constructor.name), \"Map\" === t || \"Set\" === t ? Array.from(r) : \"Arguments\" === t || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(t) ? _arrayLikeToArray(r, a) : undefined;\n }\n}\n\nvar _window = typeof window === 'undefined' ? null : window; // eslint-disable-line no-undef\n\nvar navigator = _window ? _window.navigator : null;\n_window ? _window.document : null;\nvar typeofstr = _typeof('');\nvar typeofobj = _typeof({});\nvar typeoffn = _typeof(function () {});\nvar typeofhtmlele = typeof HTMLElement === \"undefined\" ? \"undefined\" : _typeof(HTMLElement);\nvar instanceStr = function instanceStr(obj) {\n return obj && obj.instanceString && fn$6(obj.instanceString) ? obj.instanceString() : null;\n};\n\nvar string = function string(obj) {\n return obj != null && _typeof(obj) == typeofstr;\n};\nvar fn$6 = function fn(obj) {\n return obj != null && _typeof(obj) === typeoffn;\n};\nvar array = function array(obj) {\n return !elementOrCollection(obj) && (Array.isArray ? Array.isArray(obj) : obj != null && obj instanceof Array);\n};\nvar plainObject = function plainObject(obj) {\n return obj != null && _typeof(obj) === typeofobj && !array(obj) && obj.constructor === Object;\n};\nvar object = function object(obj) {\n return obj != null && _typeof(obj) === typeofobj;\n};\nvar number$1 = function number(obj) {\n return obj != null && _typeof(obj) === _typeof(1) && !isNaN(obj);\n};\nvar integer = function integer(obj) {\n return number$1(obj) && Math.floor(obj) === obj;\n};\nvar htmlElement = function htmlElement(obj) {\n if ('undefined' === typeofhtmlele) {\n return undefined;\n } else {\n return null != obj && obj instanceof HTMLElement;\n }\n};\nvar elementOrCollection = function elementOrCollection(obj) {\n return element(obj) || collection(obj);\n};\nvar element = function element(obj) {\n return instanceStr(obj) === 'collection' && obj._private.single;\n};\nvar collection = function collection(obj) {\n return instanceStr(obj) === 'collection' && !obj._private.single;\n};\nvar core = function core(obj) {\n return instanceStr(obj) === 'core';\n};\nvar stylesheet = function stylesheet(obj) {\n return instanceStr(obj) === 'stylesheet';\n};\nvar event = function event(obj) {\n return instanceStr(obj) === 'event';\n};\nvar emptyString = function emptyString(obj) {\n if (obj === undefined || obj === null) {\n // null is empty\n return true;\n } else if (obj === '' || obj.match(/^\\s+$/)) {\n return true; // empty string is empty\n }\n return false; // otherwise, we don't know what we've got\n};\nvar domElement = function domElement(obj) {\n if (typeof HTMLElement === 'undefined') {\n return false; // we're not in a browser so it doesn't matter\n } else {\n return obj instanceof HTMLElement;\n }\n};\nvar boundingBox = function boundingBox(obj) {\n return plainObject(obj) && number$1(obj.x1) && number$1(obj.x2) && number$1(obj.y1) && number$1(obj.y2);\n};\nvar promise = function promise(obj) {\n return object(obj) && fn$6(obj.then);\n};\nvar ms = function ms() {\n return navigator && navigator.userAgent.match(/msie|trident|edge/i);\n}; // probably a better way to detect this...\n\nvar memoize = function memoize(fn, keyFn) {\n if (!keyFn) {\n keyFn = function keyFn() {\n if (arguments.length === 1) {\n return arguments[0];\n } else if (arguments.length === 0) {\n return 'undefined';\n }\n var args = [];\n for (var i = 0; i < arguments.length; i++) {\n args.push(arguments[i]);\n }\n return args.join('$');\n };\n }\n var _memoizedFn = function memoizedFn() {\n var self = this;\n var args = arguments;\n var ret;\n var k = keyFn.apply(self, args);\n var cache = _memoizedFn.cache;\n if (!(ret = cache[k])) {\n ret = cache[k] = fn.apply(self, args);\n }\n return ret;\n };\n _memoizedFn.cache = {};\n return _memoizedFn;\n};\n\nvar camel2dash = memoize(function (str) {\n return str.replace(/([A-Z])/g, function (v) {\n return '-' + v.toLowerCase();\n });\n});\nvar dash2camel = memoize(function (str) {\n return str.replace(/(-\\w)/g, function (v) {\n return v[1].toUpperCase();\n });\n});\nvar prependCamel = memoize(function (prefix, str) {\n return prefix + str[0].toUpperCase() + str.substring(1);\n}, function (prefix, str) {\n return prefix + '$' + str;\n});\nvar capitalize = function capitalize(str) {\n if (emptyString(str)) {\n return str;\n }\n return str.charAt(0).toUpperCase() + str.substring(1);\n};\nvar endsWith = function endsWith(string, suffix) {\n return string.slice(-1 * suffix.length) === suffix;\n};\n\nvar number = '(?:[-+]?(?:(?:\\\\d+|\\\\d*\\\\.\\\\d+)(?:[Ee][+-]?\\\\d+)?))';\nvar rgba = 'rgb[a]?\\\\((' + number + '[%]?)\\\\s*,\\\\s*(' + number + '[%]?)\\\\s*,\\\\s*(' + number + '[%]?)(?:\\\\s*,\\\\s*(' + number + '))?\\\\)';\nvar rgbaNoBackRefs = 'rgb[a]?\\\\((?:' + number + '[%]?)\\\\s*,\\\\s*(?:' + number + '[%]?)\\\\s*,\\\\s*(?:' + number + '[%]?)(?:\\\\s*,\\\\s*(?:' + number + '))?\\\\)';\nvar hsla = 'hsl[a]?\\\\((' + number + ')\\\\s*,\\\\s*(' + number + '[%])\\\\s*,\\\\s*(' + number + '[%])(?:\\\\s*,\\\\s*(' + number + '))?\\\\)';\nvar hslaNoBackRefs = 'hsl[a]?\\\\((?:' + number + ')\\\\s*,\\\\s*(?:' + number + '[%])\\\\s*,\\\\s*(?:' + number + '[%])(?:\\\\s*,\\\\s*(?:' + number + '))?\\\\)';\nvar hex3 = '\\\\#[0-9a-fA-F]{3}';\nvar hex6 = '\\\\#[0-9a-fA-F]{6}';\n\nvar ascending = function ascending(a, b) {\n if (a < b) {\n return -1;\n } else if (a > b) {\n return 1;\n } else {\n return 0;\n }\n};\nvar descending = function descending(a, b) {\n return -1 * ascending(a, b);\n};\n\nvar extend = Object.assign != null ? Object.assign.bind(Object) : function (tgt) {\n var args = arguments;\n for (var i = 1; i < args.length; i++) {\n var obj = args[i];\n if (obj == null) {\n continue;\n }\n var keys = Object.keys(obj);\n for (var j = 0; j < keys.length; j++) {\n var k = keys[j];\n tgt[k] = obj[k];\n }\n }\n return tgt;\n};\n\n// get [r, g, b] from #abc or #aabbcc\nvar hex2tuple = function hex2tuple(hex) {\n if (!(hex.length === 4 || hex.length === 7) || hex[0] !== '#') {\n return;\n }\n var shortHex = hex.length === 4;\n var r, g, b;\n var base = 16;\n if (shortHex) {\n r = parseInt(hex[1] + hex[1], base);\n g = parseInt(hex[2] + hex[2], base);\n b = parseInt(hex[3] + hex[3], base);\n } else {\n r = parseInt(hex[1] + hex[2], base);\n g = parseInt(hex[3] + hex[4], base);\n b = parseInt(hex[5] + hex[6], base);\n }\n return [r, g, b];\n};\n\n// get [r, g, b, a] from hsl(0, 0, 0) or hsla(0, 0, 0, 0)\nvar hsl2tuple = function hsl2tuple(hsl) {\n var ret;\n var h, s, l, a, r, g, b;\n function hue2rgb(p, q, t) {\n if (t < 0) t += 1;\n if (t > 1) t -= 1;\n if (t < 1 / 6) return p + (q - p) * 6 * t;\n if (t < 1 / 2) return q;\n if (t < 2 / 3) return p + (q - p) * (2 / 3 - t) * 6;\n return p;\n }\n var m = new RegExp('^' + hsla + '$').exec(hsl);\n if (m) {\n // get hue\n h = parseInt(m[1]);\n if (h < 0) {\n h = (360 - -1 * h % 360) % 360;\n } else if (h > 360) {\n h = h % 360;\n }\n h /= 360; // normalise on [0, 1]\n\n s = parseFloat(m[2]);\n if (s < 0 || s > 100) {\n return;\n } // saturation is [0, 100]\n s = s / 100; // normalise on [0, 1]\n\n l = parseFloat(m[3]);\n if (l < 0 || l > 100) {\n return;\n } // lightness is [0, 100]\n l = l / 100; // normalise on [0, 1]\n\n a = m[4];\n if (a !== undefined) {\n a = parseFloat(a);\n if (a < 0 || a > 1) {\n return;\n } // alpha is [0, 1]\n }\n\n // now, convert to rgb\n // code from http://mjijackson.com/2008/02/rgb-to-hsl-and-rgb-to-hsv-color-model-conversion-algorithms-in-javascript\n if (s === 0) {\n r = g = b = Math.round(l * 255); // achromatic\n } else {\n var q = l < 0.5 ? l * (1 + s) : l + s - l * s;\n var p = 2 * l - q;\n r = Math.round(255 * hue2rgb(p, q, h + 1 / 3));\n g = Math.round(255 * hue2rgb(p, q, h));\n b = Math.round(255 * hue2rgb(p, q, h - 1 / 3));\n }\n ret = [r, g, b, a];\n }\n return ret;\n};\n\n// get [r, g, b, a] from rgb(0, 0, 0) or rgba(0, 0, 0, 0)\nvar rgb2tuple = function rgb2tuple(rgb) {\n var ret;\n var m = new RegExp('^' + rgba + '$').exec(rgb);\n if (m) {\n ret = [];\n var isPct = [];\n for (var i = 1; i <= 3; i++) {\n var channel = m[i];\n if (channel[channel.length - 1] === '%') {\n isPct[i] = true;\n }\n channel = parseFloat(channel);\n if (isPct[i]) {\n channel = channel / 100 * 255; // normalise to [0, 255]\n }\n if (channel < 0 || channel > 255) {\n return;\n } // invalid channel value\n\n ret.push(Math.floor(channel));\n }\n var atLeastOneIsPct = isPct[1] || isPct[2] || isPct[3];\n var allArePct = isPct[1] && isPct[2] && isPct[3];\n if (atLeastOneIsPct && !allArePct) {\n return;\n } // must all be percent values if one is\n\n var alpha = m[4];\n if (alpha !== undefined) {\n alpha = parseFloat(alpha);\n if (alpha < 0 || alpha > 1) {\n return;\n } // invalid alpha value\n\n ret.push(alpha);\n }\n }\n return ret;\n};\nvar colorname2tuple = function colorname2tuple(color) {\n return colors[color.toLowerCase()];\n};\nvar color2tuple = function color2tuple(color) {\n return (array(color) ? color : null) || colorname2tuple(color) || hex2tuple(color) || rgb2tuple(color) || hsl2tuple(color);\n};\nvar colors = {\n // special colour names\n transparent: [0, 0, 0, 0],\n // NB alpha === 0\n\n // regular colours\n aliceblue: [240, 248, 255],\n antiquewhite: [250, 235, 215],\n aqua: [0, 255, 255],\n aquamarine: [127, 255, 212],\n azure: [240, 255, 255],\n beige: [245, 245, 220],\n bisque: [255, 228, 196],\n black: [0, 0, 0],\n blanchedalmond: [255, 235, 205],\n blue: [0, 0, 255],\n blueviolet: [138, 43, 226],\n brown: [165, 42, 42],\n burlywood: [222, 184, 135],\n cadetblue: [95, 158, 160],\n chartreuse: [127, 255, 0],\n chocolate: [210, 105, 30],\n coral: [255, 127, 80],\n cornflowerblue: [100, 149, 237],\n cornsilk: [255, 248, 220],\n crimson: [220, 20, 60],\n cyan: [0, 255, 255],\n darkblue: [0, 0, 139],\n darkcyan: [0, 139, 139],\n darkgoldenrod: [184, 134, 11],\n darkgray: [169, 169, 169],\n darkgreen: [0, 100, 0],\n darkgrey: [169, 169, 169],\n darkkhaki: [189, 183, 107],\n darkmagenta: [139, 0, 139],\n darkolivegreen: [85, 107, 47],\n darkorange: [255, 140, 0],\n darkorchid: [153, 50, 204],\n darkred: [139, 0, 0],\n darksalmon: [233, 150, 122],\n darkseagreen: [143, 188, 143],\n darkslateblue: [72, 61, 139],\n darkslategray: [47, 79, 79],\n darkslategrey: [47, 79, 79],\n darkturquoise: [0, 206, 209],\n darkviolet: [148, 0, 211],\n deeppink: [255, 20, 147],\n deepskyblue: [0, 191, 255],\n dimgray: [105, 105, 105],\n dimgrey: [105, 105, 105],\n dodgerblue: [30, 144, 255],\n firebrick: [178, 34, 34],\n floralwhite: [255, 250, 240],\n forestgreen: [34, 139, 34],\n fuchsia: [255, 0, 255],\n gainsboro: [220, 220, 220],\n ghostwhite: [248, 248, 255],\n gold: [255, 215, 0],\n goldenrod: [218, 165, 32],\n gray: [128, 128, 128],\n grey: [128, 128, 128],\n green: [0, 128, 0],\n greenyellow: [173, 255, 47],\n honeydew: [240, 255, 240],\n hotpink: [255, 105, 180],\n indianred: [205, 92, 92],\n indigo: [75, 0, 130],\n ivory: [255, 255, 240],\n khaki: [240, 230, 140],\n lavender: [230, 230, 250],\n lavenderblush: [255, 240, 245],\n lawngreen: [124, 252, 0],\n lemonchiffon: [255, 250, 205],\n lightblue: [173, 216, 230],\n lightcoral: [240, 128, 128],\n lightcyan: [224, 255, 255],\n lightgoldenrodyellow: [250, 250, 210],\n lightgray: [211, 211, 211],\n lightgreen: [144, 238, 144],\n lightgrey: [211, 211, 211],\n lightpink: [255, 182, 193],\n lightsalmon: [255, 160, 122],\n lightseagreen: [32, 178, 170],\n lightskyblue: [135, 206, 250],\n lightslategray: [119, 136, 153],\n lightslategrey: [119, 136, 153],\n lightsteelblue: [176, 196, 222],\n lightyellow: [255, 255, 224],\n lime: [0, 255, 0],\n limegreen: [50, 205, 50],\n linen: [250, 240, 230],\n magenta: [255, 0, 255],\n maroon: [128, 0, 0],\n mediumaquamarine: [102, 205, 170],\n mediumblue: [0, 0, 205],\n mediumorchid: [186, 85, 211],\n mediumpurple: [147, 112, 219],\n mediumseagreen: [60, 179, 113],\n mediumslateblue: [123, 104, 238],\n mediumspringgreen: [0, 250, 154],\n mediumturquoise: [72, 209, 204],\n mediumvioletred: [199, 21, 133],\n midnightblue: [25, 25, 112],\n mintcream: [245, 255, 250],\n mistyrose: [255, 228, 225],\n moccasin: [255, 228, 181],\n navajowhite: [255, 222, 173],\n navy: [0, 0, 128],\n oldlace: [253, 245, 230],\n olive: [128, 128, 0],\n olivedrab: [107, 142, 35],\n orange: [255, 165, 0],\n orangered: [255, 69, 0],\n orchid: [218, 112, 214],\n palegoldenrod: [238, 232, 170],\n palegreen: [152, 251, 152],\n paleturquoise: [175, 238, 238],\n palevioletred: [219, 112, 147],\n papayawhip: [255, 239, 213],\n peachpuff: [255, 218, 185],\n peru: [205, 133, 63],\n pink: [255, 192, 203],\n plum: [221, 160, 221],\n powderblue: [176, 224, 230],\n purple: [128, 0, 128],\n red: [255, 0, 0],\n rosybrown: [188, 143, 143],\n royalblue: [65, 105, 225],\n saddlebrown: [139, 69, 19],\n salmon: [250, 128, 114],\n sandybrown: [244, 164, 96],\n seagreen: [46, 139, 87],\n seashell: [255, 245, 238],\n sienna: [160, 82, 45],\n silver: [192, 192, 192],\n skyblue: [135, 206, 235],\n slateblue: [106, 90, 205],\n slategray: [112, 128, 144],\n slategrey: [112, 128, 144],\n snow: [255, 250, 250],\n springgreen: [0, 255, 127],\n steelblue: [70, 130, 180],\n tan: [210, 180, 140],\n teal: [0, 128, 128],\n thistle: [216, 191, 216],\n tomato: [255, 99, 71],\n turquoise: [64, 224, 208],\n violet: [238, 130, 238],\n wheat: [245, 222, 179],\n white: [255, 255, 255],\n whitesmoke: [245, 245, 245],\n yellow: [255, 255, 0],\n yellowgreen: [154, 205, 50]\n};\n\n// sets the value in a map (map may not be built)\nvar setMap = function setMap(options) {\n var obj = options.map;\n var keys = options.keys;\n var l = keys.length;\n for (var i = 0; i < l; i++) {\n var key = keys[i];\n if (plainObject(key)) {\n throw Error('Tried to set map with object key');\n }\n if (i < keys.length - 1) {\n // extend the map if necessary\n if (obj[key] == null) {\n obj[key] = {};\n }\n obj = obj[key];\n } else {\n // set the value\n obj[key] = options.value;\n }\n }\n};\n\n// gets the value in a map even if it's not built in places\nvar getMap = function getMap(options) {\n var obj = options.map;\n var keys = options.keys;\n var l = keys.length;\n for (var i = 0; i < l; i++) {\n var key = keys[i];\n if (plainObject(key)) {\n throw Error('Tried to get map with object key');\n }\n obj = obj[key];\n if (obj == null) {\n return obj;\n }\n }\n return obj;\n};\n\nvar commonjsGlobal = typeof globalThis !== 'undefined' ? globalThis : typeof window !== 'undefined' ? window : typeof global !== 'undefined' ? global : typeof self !== 'undefined' ? self : {};\n\nfunction getDefaultExportFromCjs (x) {\n\treturn x && x.__esModule && Object.prototype.hasOwnProperty.call(x, 'default') ? x['default'] : x;\n}\n\n/**\n * Checks if `value` is the\n * [language type](http://www.ecma-international.org/ecma-262/7.0/#sec-ecmascript-language-types)\n * of `Object`. (e.g. arrays, functions, objects, regexes, `new Number(0)`, and `new String('')`)\n *\n * @static\n * @memberOf _\n * @since 0.1.0\n * @category Lang\n * @param {*} value The value to check.\n * @returns {boolean} Returns `true` if `value` is an object, else `false`.\n * @example\n *\n * _.isObject({});\n * // => true\n *\n * _.isObject([1, 2, 3]);\n * // => true\n *\n * _.isObject(_.noop);\n * // => true\n *\n * _.isObject(null);\n * // => false\n */\n\nvar isObject_1;\nvar hasRequiredIsObject;\n\nfunction requireIsObject () {\n\tif (hasRequiredIsObject) return isObject_1;\n\thasRequiredIsObject = 1;\n\tfunction isObject(value) {\n\t var type = typeof value;\n\t return value != null && (type == 'object' || type == 'function');\n\t}\n\n\tisObject_1 = isObject;\n\treturn isObject_1;\n}\n\n/** Detect free variable `global` from Node.js. */\n\nvar _freeGlobal;\nvar hasRequired_freeGlobal;\n\nfunction require_freeGlobal () {\n\tif (hasRequired_freeGlobal) return _freeGlobal;\n\thasRequired_freeGlobal = 1;\n\tvar freeGlobal = typeof commonjsGlobal == 'object' && commonjsGlobal && commonjsGlobal.Object === Object && commonjsGlobal;\n\n\t_freeGlobal = freeGlobal;\n\treturn _freeGlobal;\n}\n\nvar _root;\nvar hasRequired_root;\n\nfunction require_root () {\n\tif (hasRequired_root) return _root;\n\thasRequired_root = 1;\n\tvar freeGlobal = require_freeGlobal();\n\n\t/** Detect free variable `self`. */\n\tvar freeSelf = typeof self == 'object' && self && self.Object === Object && self;\n\n\t/** Used as a reference to the global object. */\n\tvar root = freeGlobal || freeSelf || Function('return this')();\n\n\t_root = root;\n\treturn _root;\n}\n\nvar now_1;\nvar hasRequiredNow;\n\nfunction requireNow () {\n\tif (hasRequiredNow) return now_1;\n\thasRequiredNow = 1;\n\tvar root = require_root();\n\n\t/**\n\t * Gets the timestamp of the number of milliseconds that have elapsed since\n\t * the Unix epoch (1 January 1970 00:00:00 UTC).\n\t *\n\t * @static\n\t * @memberOf _\n\t * @since 2.4.0\n\t * @category Date\n\t * @returns {number} Returns the timestamp.\n\t * @example\n\t *\n\t * _.defer(function(stamp) {\n\t * console.log(_.now() - stamp);\n\t * }, _.now());\n\t * // => Logs the number of milliseconds it took for the deferred invocation.\n\t */\n\tvar now = function() {\n\t return root.Date.now();\n\t};\n\n\tnow_1 = now;\n\treturn now_1;\n}\n\n/** Used to match a single whitespace character. */\n\nvar _trimmedEndIndex;\nvar hasRequired_trimmedEndIndex;\n\nfunction require_trimmedEndIndex () {\n\tif (hasRequired_trimmedEndIndex) return _trimmedEndIndex;\n\thasRequired_trimmedEndIndex = 1;\n\tvar reWhitespace = /\\s/;\n\n\t/**\n\t * Used by `_.trim` and `_.trimEnd` to get the index of the last non-whitespace\n\t * character of `string`.\n\t *\n\t * @private\n\t * @param {string} string The string to inspect.\n\t * @returns {number} Returns the index of the last non-whitespace character.\n\t */\n\tfunction trimmedEndIndex(string) {\n\t var index = string.length;\n\n\t while (index-- && reWhitespace.test(string.charAt(index))) {}\n\t return index;\n\t}\n\n\t_trimmedEndIndex = trimmedEndIndex;\n\treturn _trimmedEndIndex;\n}\n\nvar _baseTrim;\nvar hasRequired_baseTrim;\n\nfunction require_baseTrim () {\n\tif (hasRequired_baseTrim) return _baseTrim;\n\thasRequired_baseTrim = 1;\n\tvar trimmedEndIndex = require_trimmedEndIndex();\n\n\t/** Used to match leading whitespace. */\n\tvar reTrimStart = /^\\s+/;\n\n\t/**\n\t * The base implementation of `_.trim`.\n\t *\n\t * @private\n\t * @param {string} string The string to trim.\n\t * @returns {string} Returns the trimmed string.\n\t */\n\tfunction baseTrim(string) {\n\t return string\n\t ? string.slice(0, trimmedEndIndex(string) + 1).replace(reTrimStart, '')\n\t : string;\n\t}\n\n\t_baseTrim = baseTrim;\n\treturn _baseTrim;\n}\n\nvar _Symbol;\nvar hasRequired_Symbol;\n\nfunction require_Symbol () {\n\tif (hasRequired_Symbol) return _Symbol;\n\thasRequired_Symbol = 1;\n\tvar root = require_root();\n\n\t/** Built-in value references. */\n\tvar Symbol = root.Symbol;\n\n\t_Symbol = Symbol;\n\treturn _Symbol;\n}\n\nvar _getRawTag;\nvar hasRequired_getRawTag;\n\nfunction require_getRawTag () {\n\tif (hasRequired_getRawTag) return _getRawTag;\n\thasRequired_getRawTag = 1;\n\tvar Symbol = require_Symbol();\n\n\t/** Used for built-in method references. */\n\tvar objectProto = Object.prototype;\n\n\t/** Used to check objects for own properties. */\n\tvar hasOwnProperty = objectProto.hasOwnProperty;\n\n\t/**\n\t * Used to resolve the\n\t * [`toStringTag`](http://ecma-international.org/ecma-262/7.0/#sec-object.prototype.tostring)\n\t * of values.\n\t */\n\tvar nativeObjectToString = objectProto.toString;\n\n\t/** Built-in value references. */\n\tvar symToStringTag = Symbol ? Symbol.toStringTag : undefined;\n\n\t/**\n\t * A specialized version of `baseGetTag` which ignores `Symbol.toStringTag` values.\n\t *\n\t * @private\n\t * @param {*} value The value to query.\n\t * @returns {string} Returns the raw `toStringTag`.\n\t */\n\tfunction getRawTag(value) {\n\t var isOwn = hasOwnProperty.call(value, symToStringTag),\n\t tag = value[symToStringTag];\n\n\t try {\n\t value[symToStringTag] = undefined;\n\t var unmasked = true;\n\t } catch (e) {}\n\n\t var result = nativeObjectToString.call(value);\n\t if (unmasked) {\n\t if (isOwn) {\n\t value[symToStringTag] = tag;\n\t } else {\n\t delete value[symToStringTag];\n\t }\n\t }\n\t return result;\n\t}\n\n\t_getRawTag = getRawTag;\n\treturn _getRawTag;\n}\n\n/** Used for built-in method references. */\n\nvar _objectToString;\nvar hasRequired_objectToString;\n\nfunction require_objectToString () {\n\tif (hasRequired_objectToString) return _objectToString;\n\thasRequired_objectToString = 1;\n\tvar objectProto = Object.prototype;\n\n\t/**\n\t * Used to resolve the\n\t * [`toStringTag`](http://ecma-international.org/ecma-262/7.0/#sec-object.prototype.tostring)\n\t * of values.\n\t */\n\tvar nativeObjectToString = objectProto.toString;\n\n\t/**\n\t * Converts `value` to a string using `Object.prototype.toString`.\n\t *\n\t * @private\n\t * @param {*} value The value to convert.\n\t * @returns {string} Returns the converted string.\n\t */\n\tfunction objectToString(value) {\n\t return nativeObjectToString.call(value);\n\t}\n\n\t_objectToString = objectToString;\n\treturn _objectToString;\n}\n\nvar _baseGetTag;\nvar hasRequired_baseGetTag;\n\nfunction require_baseGetTag () {\n\tif (hasRequired_baseGetTag) return _baseGetTag;\n\thasRequired_baseGetTag = 1;\n\tvar Symbol = require_Symbol(),\n\t getRawTag = require_getRawTag(),\n\t objectToString = require_objectToString();\n\n\t/** `Object#toString` result references. */\n\tvar nullTag = '[object Null]',\n\t undefinedTag = '[object Undefined]';\n\n\t/** Built-in value references. */\n\tvar symToStringTag = Symbol ? Symbol.toStringTag : undefined;\n\n\t/**\n\t * The base implementation of `getTag` without fallbacks for buggy environments.\n\t *\n\t * @private\n\t * @param {*} value The value to query.\n\t * @returns {string} Returns the `toStringTag`.\n\t */\n\tfunction baseGetTag(value) {\n\t if (value == null) {\n\t return value === undefined ? undefinedTag : nullTag;\n\t }\n\t return (symToStringTag && symToStringTag in Object(value))\n\t ? getRawTag(value)\n\t : objectToString(value);\n\t}\n\n\t_baseGetTag = baseGetTag;\n\treturn _baseGetTag;\n}\n\n/**\n * Checks if `value` is object-like. A value is object-like if it's not `null`\n * and has a `typeof` result of \"object\".\n *\n * @static\n * @memberOf _\n * @since 4.0.0\n * @category Lang\n * @param {*} value The value to check.\n * @returns {boolean} Returns `true` if `value` is object-like, else `false`.\n * @example\n *\n * _.isObjectLike({});\n * // => true\n *\n * _.isObjectLike([1, 2, 3]);\n * // => true\n *\n * _.isObjectLike(_.noop);\n * // => false\n *\n * _.isObjectLike(null);\n * // => false\n */\n\nvar isObjectLike_1;\nvar hasRequiredIsObjectLike;\n\nfunction requireIsObjectLike () {\n\tif (hasRequiredIsObjectLike) return isObjectLike_1;\n\thasRequiredIsObjectLike = 1;\n\tfunction isObjectLike(value) {\n\t return value != null && typeof value == 'object';\n\t}\n\n\tisObjectLike_1 = isObjectLike;\n\treturn isObjectLike_1;\n}\n\nvar isSymbol_1;\nvar hasRequiredIsSymbol;\n\nfunction requireIsSymbol () {\n\tif (hasRequiredIsSymbol) return isSymbol_1;\n\thasRequiredIsSymbol = 1;\n\tvar baseGetTag = require_baseGetTag(),\n\t isObjectLike = requireIsObjectLike();\n\n\t/** `Object#toString` result references. */\n\tvar symbolTag = '[object Symbol]';\n\n\t/**\n\t * Checks if `value` is classified as a `Symbol` primitive or object.\n\t *\n\t * @static\n\t * @memberOf _\n\t * @since 4.0.0\n\t * @category Lang\n\t * @param {*} value The value to check.\n\t * @returns {boolean} Returns `true` if `value` is a symbol, else `false`.\n\t * @example\n\t *\n\t * _.isSymbol(Symbol.iterator);\n\t * // => true\n\t *\n\t * _.isSymbol('abc');\n\t * // => false\n\t */\n\tfunction isSymbol(value) {\n\t return typeof value == 'symbol' ||\n\t (isObjectLike(value) && baseGetTag(value) == symbolTag);\n\t}\n\n\tisSymbol_1 = isSymbol;\n\treturn isSymbol_1;\n}\n\nvar toNumber_1;\nvar hasRequiredToNumber;\n\nfunction requireToNumber () {\n\tif (hasRequiredToNumber) return toNumber_1;\n\thasRequiredToNumber = 1;\n\tvar baseTrim = require_baseTrim(),\n\t isObject = requireIsObject(),\n\t isSymbol = requireIsSymbol();\n\n\t/** Used as references for various `Number` constants. */\n\tvar NAN = 0 / 0;\n\n\t/** Used to detect bad signed hexadecimal string values. */\n\tvar reIsBadHex = /^[-+]0x[0-9a-f]+$/i;\n\n\t/** Used to detect binary string values. */\n\tvar reIsBinary = /^0b[01]+$/i;\n\n\t/** Used to detect octal string values. */\n\tvar reIsOctal = /^0o[0-7]+$/i;\n\n\t/** Built-in method references without a dependency on `root`. */\n\tvar freeParseInt = parseInt;\n\n\t/**\n\t * Converts `value` to a number.\n\t *\n\t * @static\n\t * @memberOf _\n\t * @since 4.0.0\n\t * @category Lang\n\t * @param {*} value The value to process.\n\t * @returns {number} Returns the number.\n\t * @example\n\t *\n\t * _.toNumber(3.2);\n\t * // => 3.2\n\t *\n\t * _.toNumber(Number.MIN_VALUE);\n\t * // => 5e-324\n\t *\n\t * _.toNumber(Infinity);\n\t * // => Infinity\n\t *\n\t * _.toNumber('3.2');\n\t * // => 3.2\n\t */\n\tfunction toNumber(value) {\n\t if (typeof value == 'number') {\n\t return value;\n\t }\n\t if (isSymbol(value)) {\n\t return NAN;\n\t }\n\t if (isObject(value)) {\n\t var other = typeof value.valueOf == 'function' ? value.valueOf() : value;\n\t value = isObject(other) ? (other + '') : other;\n\t }\n\t if (typeof value != 'string') {\n\t return value === 0 ? value : +value;\n\t }\n\t value = baseTrim(value);\n\t var isBinary = reIsBinary.test(value);\n\t return (isBinary || reIsOctal.test(value))\n\t ? freeParseInt(value.slice(2), isBinary ? 2 : 8)\n\t : (reIsBadHex.test(value) ? NAN : +value);\n\t}\n\n\ttoNumber_1 = toNumber;\n\treturn toNumber_1;\n}\n\nvar debounce_1;\nvar hasRequiredDebounce;\n\nfunction requireDebounce () {\n\tif (hasRequiredDebounce) return debounce_1;\n\thasRequiredDebounce = 1;\n\tvar isObject = requireIsObject(),\n\t now = requireNow(),\n\t toNumber = requireToNumber();\n\n\t/** Error message constants. */\n\tvar FUNC_ERROR_TEXT = 'Expected a function';\n\n\t/* Built-in method references for those with the same name as other `lodash` methods. */\n\tvar nativeMax = Math.max,\n\t nativeMin = Math.min;\n\n\t/**\n\t * Creates a debounced function that delays invoking `func` until after `wait`\n\t * milliseconds have elapsed since the last time the debounced function was\n\t * invoked. The debounced function comes with a `cancel` method to cancel\n\t * delayed `func` invocations and a `flush` method to immediately invoke them.\n\t * Provide `options` to indicate whether `func` should be invoked on the\n\t * leading and/or trailing edge of the `wait` timeout. The `func` is invoked\n\t * with the last arguments provided to the debounced function. Subsequent\n\t * calls to the debounced function return the result of the last `func`\n\t * invocation.\n\t *\n\t * **Note:** If `leading` and `trailing` options are `true`, `func` is\n\t * invoked on the trailing edge of the timeout only if the debounced function\n\t * is invoked more than once during the `wait` timeout.\n\t *\n\t * If `wait` is `0` and `leading` is `false`, `func` invocation is deferred\n\t * until to the next tick, similar to `setTimeout` with a timeout of `0`.\n\t *\n\t * See [David Corbacho's article](https://css-tricks.com/debouncing-throttling-explained-examples/)\n\t * for details over the differences between `_.debounce` and `_.throttle`.\n\t *\n\t * @static\n\t * @memberOf _\n\t * @since 0.1.0\n\t * @category Function\n\t * @param {Function} func The function to debounce.\n\t * @param {number} [wait=0] The number of milliseconds to delay.\n\t * @param {Object} [options={}] The options object.\n\t * @param {boolean} [options.leading=false]\n\t * Specify invoking on the leading edge of the timeout.\n\t * @param {number} [options.maxWait]\n\t * The maximum time `func` is allowed to be delayed before it's invoked.\n\t * @param {boolean} [options.trailing=true]\n\t * Specify invoking on the trailing edge of the timeout.\n\t * @returns {Function} Returns the new debounced function.\n\t * @example\n\t *\n\t * // Avoid costly calculations while the window size is in flux.\n\t * jQuery(window).on('resize', _.debounce(calculateLayout, 150));\n\t *\n\t * // Invoke `sendMail` when clicked, debouncing subsequent calls.\n\t * jQuery(element).on('click', _.debounce(sendMail, 300, {\n\t * 'leading': true,\n\t * 'trailing': false\n\t * }));\n\t *\n\t * // Ensure `batchLog` is invoked once after 1 second of debounced calls.\n\t * var debounced = _.debounce(batchLog, 250, { 'maxWait': 1000 });\n\t * var source = new EventSource('/stream');\n\t * jQuery(source).on('message', debounced);\n\t *\n\t * // Cancel the trailing debounced invocation.\n\t * jQuery(window).on('popstate', debounced.cancel);\n\t */\n\tfunction debounce(func, wait, options) {\n\t var lastArgs,\n\t lastThis,\n\t maxWait,\n\t result,\n\t timerId,\n\t lastCallTime,\n\t lastInvokeTime = 0,\n\t leading = false,\n\t maxing = false,\n\t trailing = true;\n\n\t if (typeof func != 'function') {\n\t throw new TypeError(FUNC_ERROR_TEXT);\n\t }\n\t wait = toNumber(wait) || 0;\n\t if (isObject(options)) {\n\t leading = !!options.leading;\n\t maxing = 'maxWait' in options;\n\t maxWait = maxing ? nativeMax(toNumber(options.maxWait) || 0, wait) : maxWait;\n\t trailing = 'trailing' in options ? !!options.trailing : trailing;\n\t }\n\n\t function invokeFunc(time) {\n\t var args = lastArgs,\n\t thisArg = lastThis;\n\n\t lastArgs = lastThis = undefined;\n\t lastInvokeTime = time;\n\t result = func.apply(thisArg, args);\n\t return result;\n\t }\n\n\t function leadingEdge(time) {\n\t // Reset any `maxWait` timer.\n\t lastInvokeTime = time;\n\t // Start the timer for the trailing edge.\n\t timerId = setTimeout(timerExpired, wait);\n\t // Invoke the leading edge.\n\t return leading ? invokeFunc(time) : result;\n\t }\n\n\t function remainingWait(time) {\n\t var timeSinceLastCall = time - lastCallTime,\n\t timeSinceLastInvoke = time - lastInvokeTime,\n\t timeWaiting = wait - timeSinceLastCall;\n\n\t return maxing\n\t ? nativeMin(timeWaiting, maxWait - timeSinceLastInvoke)\n\t : timeWaiting;\n\t }\n\n\t function shouldInvoke(time) {\n\t var timeSinceLastCall = time - lastCallTime,\n\t timeSinceLastInvoke = time - lastInvokeTime;\n\n\t // Either this is the first call, activity has stopped and we're at the\n\t // trailing edge, the system time has gone backwards and we're treating\n\t // it as the trailing edge, or we've hit the `maxWait` limit.\n\t return (lastCallTime === undefined || (timeSinceLastCall >= wait) ||\n\t (timeSinceLastCall < 0) || (maxing && timeSinceLastInvoke >= maxWait));\n\t }\n\n\t function timerExpired() {\n\t var time = now();\n\t if (shouldInvoke(time)) {\n\t return trailingEdge(time);\n\t }\n\t // Restart the timer.\n\t timerId = setTimeout(timerExpired, remainingWait(time));\n\t }\n\n\t function trailingEdge(time) {\n\t timerId = undefined;\n\n\t // Only invoke if we have `lastArgs` which means `func` has been\n\t // debounced at least once.\n\t if (trailing && lastArgs) {\n\t return invokeFunc(time);\n\t }\n\t lastArgs = lastThis = undefined;\n\t return result;\n\t }\n\n\t function cancel() {\n\t if (timerId !== undefined) {\n\t clearTimeout(timerId);\n\t }\n\t lastInvokeTime = 0;\n\t lastArgs = lastCallTime = lastThis = timerId = undefined;\n\t }\n\n\t function flush() {\n\t return timerId === undefined ? result : trailingEdge(now());\n\t }\n\n\t function debounced() {\n\t var time = now(),\n\t isInvoking = shouldInvoke(time);\n\n\t lastArgs = arguments;\n\t lastThis = this;\n\t lastCallTime = time;\n\n\t if (isInvoking) {\n\t if (timerId === undefined) {\n\t return leadingEdge(lastCallTime);\n\t }\n\t if (maxing) {\n\t // Handle invocations in a tight loop.\n\t clearTimeout(timerId);\n\t timerId = setTimeout(timerExpired, wait);\n\t return invokeFunc(lastCallTime);\n\t }\n\t }\n\t if (timerId === undefined) {\n\t timerId = setTimeout(timerExpired, wait);\n\t }\n\t return result;\n\t }\n\t debounced.cancel = cancel;\n\t debounced.flush = flush;\n\t return debounced;\n\t}\n\n\tdebounce_1 = debounce;\n\treturn debounce_1;\n}\n\nvar debounceExports = requireDebounce();\nvar debounce = /*@__PURE__*/getDefaultExportFromCjs(debounceExports);\n\nvar performance$1 = _window ? _window.performance : null;\nvar pnow = performance$1 && performance$1.now ? function () {\n return performance$1.now();\n} : function () {\n return Date.now();\n};\nvar raf = function () {\n if (_window) {\n if (_window.requestAnimationFrame) {\n return function (fn) {\n _window.requestAnimationFrame(fn);\n };\n } else if (_window.mozRequestAnimationFrame) {\n return function (fn) {\n _window.mozRequestAnimationFrame(fn);\n };\n } else if (_window.webkitRequestAnimationFrame) {\n return function (fn) {\n _window.webkitRequestAnimationFrame(fn);\n };\n } else if (_window.msRequestAnimationFrame) {\n return function (fn) {\n _window.msRequestAnimationFrame(fn);\n };\n }\n }\n return function (fn) {\n if (fn) {\n setTimeout(function () {\n fn(pnow());\n }, 1000 / 60);\n }\n };\n}();\nvar requestAnimationFrame = function requestAnimationFrame(fn) {\n return raf(fn);\n};\nvar performanceNow = pnow;\n\nvar DEFAULT_HASH_SEED = 9261;\nvar K = 65599; // 37 also works pretty well\nvar DEFAULT_HASH_SEED_ALT = 5381;\nvar hashIterableInts = function hashIterableInts(iterator) {\n var seed = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : DEFAULT_HASH_SEED;\n // sdbm/string-hash\n var hash = seed;\n var entry;\n for (;;) {\n entry = iterator.next();\n if (entry.done) {\n break;\n }\n hash = hash * K + entry.value | 0;\n }\n return hash;\n};\nvar hashInt = function hashInt(num) {\n var seed = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : DEFAULT_HASH_SEED;\n // sdbm/string-hash\n return seed * K + num | 0;\n};\nvar hashIntAlt = function hashIntAlt(num) {\n var seed = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : DEFAULT_HASH_SEED_ALT;\n // djb2/string-hash\n return (seed << 5) + seed + num | 0;\n};\nvar combineHashes = function combineHashes(hash1, hash2) {\n return hash1 * 0x200000 + hash2;\n};\nvar combineHashesArray = function combineHashesArray(hashes) {\n return hashes[0] * 0x200000 + hashes[1];\n};\nvar hashArrays = function hashArrays(hashes1, hashes2) {\n return [hashInt(hashes1[0], hashes2[0]), hashIntAlt(hashes1[1], hashes2[1])];\n};\nvar hashIntsArray = function hashIntsArray(ints, seed) {\n var entry = {\n value: 0,\n done: false\n };\n var i = 0;\n var length = ints.length;\n var iterator = {\n next: function next() {\n if (i < length) {\n entry.value = ints[i++];\n } else {\n entry.done = true;\n }\n return entry;\n }\n };\n return hashIterableInts(iterator, seed);\n};\nvar hashString = function hashString(str, seed) {\n var entry = {\n value: 0,\n done: false\n };\n var i = 0;\n var length = str.length;\n var iterator = {\n next: function next() {\n if (i < length) {\n entry.value = str.charCodeAt(i++);\n } else {\n entry.done = true;\n }\n return entry;\n }\n };\n return hashIterableInts(iterator, seed);\n};\nvar hashStrings = function hashStrings() {\n return hashStringsArray(arguments);\n};\nvar hashStringsArray = function hashStringsArray(strs) {\n var hash;\n for (var i = 0; i < strs.length; i++) {\n var str = strs[i];\n if (i === 0) {\n hash = hashString(str);\n } else {\n hash = hashString(str, hash);\n }\n }\n return hash;\n};\n\nfunction rotatePoint(x, y, centerX, centerY, angleDegrees) {\n var angleRadians = angleDegrees * Math.PI / 180;\n var rotatedX = Math.cos(angleRadians) * (x - centerX) - Math.sin(angleRadians) * (y - centerY) + centerX;\n var rotatedY = Math.sin(angleRadians) * (x - centerX) + Math.cos(angleRadians) * (y - centerY) + centerY;\n return {\n x: rotatedX,\n y: rotatedY\n };\n}\nvar movePointByBoxAspect = function movePointByBoxAspect(x, y, boxX, boxY, skewX, skewY) {\n return {\n x: (x - boxX) * skewX + boxX,\n y: (y - boxY) * skewY + boxY\n };\n};\nfunction rotatePosAndSkewByBox(pos, box, angleDegrees) {\n if (angleDegrees === 0) return pos;\n var centerX = (box.x1 + box.x2) / 2;\n var centerY = (box.y1 + box.y2) / 2;\n var skewX = box.w / box.h;\n var skewY = 1 / skewX;\n var rotated = rotatePoint(pos.x, pos.y, centerX, centerY, angleDegrees);\n var skewed = movePointByBoxAspect(rotated.x, rotated.y, centerX, centerY, skewX, skewY);\n return {\n x: skewed.x,\n y: skewed.y\n };\n}\n\nvar warningsEnabled = true;\nvar warnSupported = console.warn != null;\nvar traceSupported = console.trace != null;\nvar MAX_INT$1 = Number.MAX_SAFE_INTEGER || 9007199254740991;\nvar trueify = function trueify() {\n return true;\n};\nvar falsify = function falsify() {\n return false;\n};\nvar zeroify = function zeroify() {\n return 0;\n};\nvar noop$1 = function noop() {};\nvar error = function error(msg) {\n throw new Error(msg);\n};\nvar warnings = function warnings(enabled) {\n if (enabled !== undefined) {\n warningsEnabled = !!enabled;\n } else {\n return warningsEnabled;\n }\n};\nvar warn = function warn(msg) {\n if (!warnings()) {\n return;\n }\n if (warnSupported) {\n console.warn(msg);\n } else {\n console.log(msg);\n if (traceSupported) {\n console.trace();\n }\n }\n};\nvar clone = function clone(obj) {\n return extend({}, obj);\n};\n\n// gets a shallow copy of the argument\nvar copy = function copy(obj) {\n if (obj == null) {\n return obj;\n }\n if (array(obj)) {\n return obj.slice();\n } else if (plainObject(obj)) {\n return clone(obj);\n } else {\n return obj;\n }\n};\nvar copyArray = function copyArray(arr) {\n return arr.slice();\n};\nvar uuid = function uuid(a, b /* placeholders */) {\n for (\n // loop :)\n b = a = '';\n // b - result , a - numeric letiable\n a++ < 36;\n //\n b += a * 51 & 52 // if \"a\" is not 9 or 14 or 19 or 24\n ?\n // return a random number or 4\n (a ^ 15 // if \"a\" is not 15\n ?\n // generate a random number from 0 to 15\n 8 ^ Math.random() * (a ^ 20 ? 16 : 4) // unless \"a\" is 20, in which case a random number from 8 to 11\n : 4 // otherwise 4\n ).toString(16) : '-' // in other cases (if \"a\" is 9,14,19,24) insert \"-\"\n );\n return b;\n};\nvar _staticEmptyObject = {};\nvar staticEmptyObject = function staticEmptyObject() {\n return _staticEmptyObject;\n};\nvar defaults$g = function defaults(_defaults) {\n var keys = Object.keys(_defaults);\n return function (opts) {\n var filledOpts = {};\n for (var i = 0; i < keys.length; i++) {\n var key = keys[i];\n var optVal = opts == null ? undefined : opts[key];\n filledOpts[key] = optVal === undefined ? _defaults[key] : optVal;\n }\n return filledOpts;\n };\n};\nvar removeFromArray = function removeFromArray(arr, ele, oneCopy) {\n for (var i = arr.length - 1; i >= 0; i--) {\n if (arr[i] === ele) {\n arr.splice(i, 1);\n }\n }\n};\nvar clearArray = function clearArray(arr) {\n arr.splice(0, arr.length);\n};\nvar push = function push(arr, otherArr) {\n for (var i = 0; i < otherArr.length; i++) {\n var el = otherArr[i];\n arr.push(el);\n }\n};\nvar getPrefixedProperty = function getPrefixedProperty(obj, propName, prefix) {\n if (prefix) {\n propName = prependCamel(prefix, propName); // e.g. (labelWidth, source) => sourceLabelWidth\n }\n return obj[propName];\n};\nvar setPrefixedProperty = function setPrefixedProperty(obj, propName, prefix, value) {\n if (prefix) {\n propName = prependCamel(prefix, propName); // e.g. (labelWidth, source) => sourceLabelWidth\n }\n obj[propName] = value;\n};\n\n/* global Map */\nvar ObjectMap = /*#__PURE__*/function () {\n function ObjectMap() {\n _classCallCheck(this, ObjectMap);\n this._obj = {};\n }\n return _createClass(ObjectMap, [{\n key: \"set\",\n value: function set(key, val) {\n this._obj[key] = val;\n return this;\n }\n }, {\n key: \"delete\",\n value: function _delete(key) {\n this._obj[key] = undefined;\n return this;\n }\n }, {\n key: \"clear\",\n value: function clear() {\n this._obj = {};\n }\n }, {\n key: \"has\",\n value: function has(key) {\n return this._obj[key] !== undefined;\n }\n }, {\n key: \"get\",\n value: function get(key) {\n return this._obj[key];\n }\n }]);\n}();\nvar Map$1 = typeof Map !== 'undefined' ? Map : ObjectMap;\n\n/* global Set */\n\nvar undef = \"undefined\" ;\nvar ObjectSet = /*#__PURE__*/function () {\n function ObjectSet(arrayOrObjectSet) {\n _classCallCheck(this, ObjectSet);\n this._obj = Object.create(null);\n this.size = 0;\n if (arrayOrObjectSet != null) {\n var arr;\n if (arrayOrObjectSet.instanceString != null && arrayOrObjectSet.instanceString() === this.instanceString()) {\n arr = arrayOrObjectSet.toArray();\n } else {\n arr = arrayOrObjectSet;\n }\n for (var i = 0; i < arr.length; i++) {\n this.add(arr[i]);\n }\n }\n }\n return _createClass(ObjectSet, [{\n key: \"instanceString\",\n value: function instanceString() {\n return 'set';\n }\n }, {\n key: \"add\",\n value: function add(val) {\n var o = this._obj;\n if (o[val] !== 1) {\n o[val] = 1;\n this.size++;\n }\n }\n }, {\n key: \"delete\",\n value: function _delete(val) {\n var o = this._obj;\n if (o[val] === 1) {\n o[val] = 0;\n this.size--;\n }\n }\n }, {\n key: \"clear\",\n value: function clear() {\n this._obj = Object.create(null);\n }\n }, {\n key: \"has\",\n value: function has(val) {\n return this._obj[val] === 1;\n }\n }, {\n key: \"toArray\",\n value: function toArray() {\n var _this = this;\n return Object.keys(this._obj).filter(function (key) {\n return _this.has(key);\n });\n }\n }, {\n key: \"forEach\",\n value: function forEach(callback, thisArg) {\n return this.toArray().forEach(callback, thisArg);\n }\n }]);\n}();\nvar Set$1 = (typeof Set === \"undefined\" ? \"undefined\" : _typeof(Set)) !== undef ? Set : ObjectSet;\n\n// represents a node or an edge\nvar Element = function Element(cy, params) {\n var restore = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : true;\n if (cy === undefined || params === undefined || !core(cy)) {\n error('An element must have a core reference and parameters set');\n return;\n }\n var group = params.group;\n\n // try to automatically infer the group if unspecified\n if (group == null) {\n if (params.data && params.data.source != null && params.data.target != null) {\n group = 'edges';\n } else {\n group = 'nodes';\n }\n }\n\n // validate group\n if (group !== 'nodes' && group !== 'edges') {\n error('An element must be of type `nodes` or `edges`; you specified `' + group + '`');\n return;\n }\n\n // make the element array-like, just like a collection\n this.length = 1;\n this[0] = this;\n\n // NOTE: when something is added here, add also to ele.json()\n var _p = this._private = {\n cy: cy,\n single: true,\n // indicates this is an element\n data: params.data || {},\n // data object\n position: params.position || {\n x: 0,\n y: 0\n },\n // (x, y) position pair\n autoWidth: undefined,\n // width and height of nodes calculated by the renderer when set to special 'auto' value\n autoHeight: undefined,\n autoPadding: undefined,\n compoundBoundsClean: false,\n // whether the compound dimensions need to be recalculated the next time dimensions are read\n listeners: [],\n // array of bound listeners\n group: group,\n // string; 'nodes' or 'edges'\n style: {},\n // properties as set by the style\n rstyle: {},\n // properties for style sent from the renderer to the core\n styleCxts: [],\n // applied style contexts from the styler\n styleKeys: {},\n // per-group keys of style property values\n removed: true,\n // whether it's inside the vis; true if removed (set true here since we call restore)\n selected: params.selected ? true : false,\n // whether it's selected\n selectable: params.selectable === undefined ? true : params.selectable ? true : false,\n // whether it's selectable\n locked: params.locked ? true : false,\n // whether the element is locked (cannot be moved)\n grabbed: false,\n // whether the element is grabbed by the mouse; renderer sets this privately\n grabbable: params.grabbable === undefined ? true : params.grabbable ? true : false,\n // whether the element can be grabbed\n pannable: params.pannable === undefined ? group === 'edges' ? true : false : params.pannable ? true : false,\n // whether the element has passthrough panning enabled\n active: false,\n // whether the element is active from user interaction\n classes: new Set$1(),\n // map ( className => true )\n animation: {\n // object for currently-running animations\n current: [],\n queue: []\n },\n rscratch: {},\n // object in which the renderer can store information\n scratch: params.scratch || {},\n // scratch objects\n edges: [],\n // array of connected edges\n children: [],\n // array of children\n parent: params.parent && params.parent.isNode() ? params.parent : null,\n // parent ref\n traversalCache: {},\n // cache of output of traversal functions\n backgrounding: false,\n // whether background images are loading\n bbCache: null,\n // cache of the current bounding box\n bbCacheShift: {\n x: 0,\n y: 0\n },\n // shift applied to cached bb to be applied on next get\n bodyBounds: null,\n // bounds cache of element body, w/o overlay\n overlayBounds: null,\n // bounds cache of element body, including overlay\n labelBounds: {\n // bounds cache of labels\n all: null,\n source: null,\n target: null,\n main: null\n },\n arrowBounds: {\n // bounds cache of edge arrows\n source: null,\n target: null,\n 'mid-source': null,\n 'mid-target': null\n }\n };\n if (_p.position.x == null) {\n _p.position.x = 0;\n }\n if (_p.position.y == null) {\n _p.position.y = 0;\n }\n\n // renderedPosition overrides if specified\n if (params.renderedPosition) {\n var rpos = params.renderedPosition;\n var pan = cy.pan();\n var zoom = cy.zoom();\n _p.position = {\n x: (rpos.x - pan.x) / zoom,\n y: (rpos.y - pan.y) / zoom\n };\n }\n var classes = [];\n if (array(params.classes)) {\n classes = params.classes;\n } else if (string(params.classes)) {\n classes = params.classes.split(/\\s+/);\n }\n for (var i = 0, l = classes.length; i < l; i++) {\n var cls = classes[i];\n if (!cls || cls === '') {\n continue;\n }\n _p.classes.add(cls);\n }\n this.createEmitter();\n if (restore === undefined || restore) {\n this.restore();\n }\n var bypass = params.style || params.css;\n if (bypass) {\n warn('Setting a `style` bypass at element creation should be done only when absolutely necessary. Try to use the stylesheet instead.');\n this.style(bypass);\n }\n};\n\nvar defineSearch = function defineSearch(params) {\n params = {\n bfs: params.bfs || !params.dfs,\n dfs: params.dfs || !params.bfs\n };\n\n // from pseudocode on wikipedia\n return function searchFn(roots, fn, directed) {\n var options;\n if (plainObject(roots) && !elementOrCollection(roots)) {\n options = roots;\n roots = options.roots || options.root;\n fn = options.visit;\n directed = options.directed;\n }\n directed = arguments.length === 2 && !fn$6(fn) ? fn : directed;\n fn = fn$6(fn) ? fn : function () {};\n var cy = this._private.cy;\n var v = roots = string(roots) ? this.filter(roots) : roots;\n var Q = [];\n var connectedNodes = [];\n var connectedBy = {};\n var id2depth = {};\n var V = {};\n var j = 0;\n var found;\n var _this$byGroup = this.byGroup(),\n nodes = _this$byGroup.nodes,\n edges = _this$byGroup.edges;\n\n // enqueue v\n for (var i = 0; i < v.length; i++) {\n var vi = v[i];\n var viId = vi.id();\n if (vi.isNode()) {\n Q.unshift(vi);\n if (params.bfs) {\n V[viId] = true;\n connectedNodes.push(vi);\n }\n id2depth[viId] = 0;\n }\n }\n var _loop = function _loop() {\n var v = params.bfs ? Q.shift() : Q.pop();\n var vId = v.id();\n if (params.dfs) {\n if (V[vId]) {\n return 0; // continue\n }\n V[vId] = true;\n connectedNodes.push(v);\n }\n var depth = id2depth[vId];\n var prevEdge = connectedBy[vId];\n var src = prevEdge != null ? prevEdge.source() : null;\n var tgt = prevEdge != null ? prevEdge.target() : null;\n var prevNode = prevEdge == null ? undefined : v.same(src) ? tgt[0] : src[0];\n var ret;\n ret = fn(v, prevEdge, prevNode, j++, depth);\n if (ret === true) {\n found = v;\n return 1; // break\n }\n if (ret === false) {\n return 1; // break\n }\n var vwEdges = v.connectedEdges().filter(function (e) {\n return (!directed || e.source().same(v)) && edges.has(e);\n });\n for (var _i2 = 0; _i2 < vwEdges.length; _i2++) {\n var e = vwEdges[_i2];\n var w = e.connectedNodes().filter(function (n) {\n return !n.same(v) && nodes.has(n);\n });\n var wId = w.id();\n if (w.length !== 0 && !V[wId]) {\n w = w[0];\n Q.push(w);\n if (params.bfs) {\n V[wId] = true;\n connectedNodes.push(w);\n }\n connectedBy[wId] = e;\n id2depth[wId] = id2depth[vId] + 1;\n }\n }\n },\n _ret;\n while (Q.length !== 0) {\n _ret = _loop();\n if (_ret === 0) continue;\n if (_ret === 1) break;\n }\n var connectedEles = cy.collection();\n for (var _i = 0; _i < connectedNodes.length; _i++) {\n var node = connectedNodes[_i];\n var edge = connectedBy[node.id()];\n if (edge != null) {\n connectedEles.push(edge);\n }\n connectedEles.push(node);\n }\n return {\n path: cy.collection(connectedEles),\n found: cy.collection(found)\n };\n };\n};\n\n// search, spanning trees, etc\nvar elesfn$v = {\n breadthFirstSearch: defineSearch({\n bfs: true\n }),\n depthFirstSearch: defineSearch({\n dfs: true\n })\n};\n\n// nice, short mathematical alias\nelesfn$v.bfs = elesfn$v.breadthFirstSearch;\nelesfn$v.dfs = elesfn$v.depthFirstSearch;\n\nvar heap$2 = {exports: {}};\n\nvar heap$1 = heap$2.exports;\n\nvar hasRequiredHeap$1;\n\nfunction requireHeap$1 () {\n\tif (hasRequiredHeap$1) return heap$2.exports;\n\thasRequiredHeap$1 = 1;\n\t(function (module, exports) {\n\t\t// Generated by CoffeeScript 1.8.0\n\t\t(function() {\n\t\t var Heap, defaultCmp, floor, heapify, heappop, heappush, heappushpop, heapreplace, insort, min, nlargest, nsmallest, updateItem, _siftdown, _siftup;\n\n\t\t floor = Math.floor, min = Math.min;\n\n\n\t\t /*\n\t\t Default comparison function to be used\n\t\t */\n\n\t\t defaultCmp = function(x, y) {\n\t\t if (x < y) {\n\t\t return -1;\n\t\t }\n\t\t if (x > y) {\n\t\t return 1;\n\t\t }\n\t\t return 0;\n\t\t };\n\n\n\t\t /*\n\t\t Insert item x in list a, and keep it sorted assuming a is sorted.\n\t\t \n\t\t If x is already in a, insert it to the right of the rightmost x.\n\t\t \n\t\t Optional args lo (default 0) and hi (default a.length) bound the slice\n\t\t of a to be searched.\n\t\t */\n\n\t\t insort = function(a, x, lo, hi, cmp) {\n\t\t var mid;\n\t\t if (lo == null) {\n\t\t lo = 0;\n\t\t }\n\t\t if (cmp == null) {\n\t\t cmp = defaultCmp;\n\t\t }\n\t\t if (lo < 0) {\n\t\t throw new Error('lo must be non-negative');\n\t\t }\n\t\t if (hi == null) {\n\t\t hi = a.length;\n\t\t }\n\t\t while (lo < hi) {\n\t\t mid = floor((lo + hi) / 2);\n\t\t if (cmp(x, a[mid]) < 0) {\n\t\t hi = mid;\n\t\t } else {\n\t\t lo = mid + 1;\n\t\t }\n\t\t }\n\t\t return ([].splice.apply(a, [lo, lo - lo].concat(x)), x);\n\t\t };\n\n\n\t\t /*\n\t\t Push item onto heap, maintaining the heap invariant.\n\t\t */\n\n\t\t heappush = function(array, item, cmp) {\n\t\t if (cmp == null) {\n\t\t cmp = defaultCmp;\n\t\t }\n\t\t array.push(item);\n\t\t return _siftdown(array, 0, array.length - 1, cmp);\n\t\t };\n\n\n\t\t /*\n\t\t Pop the smallest item off the heap, maintaining the heap invariant.\n\t\t */\n\n\t\t heappop = function(array, cmp) {\n\t\t var lastelt, returnitem;\n\t\t if (cmp == null) {\n\t\t cmp = defaultCmp;\n\t\t }\n\t\t lastelt = array.pop();\n\t\t if (array.length) {\n\t\t returnitem = array[0];\n\t\t array[0] = lastelt;\n\t\t _siftup(array, 0, cmp);\n\t\t } else {\n\t\t returnitem = lastelt;\n\t\t }\n\t\t return returnitem;\n\t\t };\n\n\n\t\t /*\n\t\t Pop and return the current smallest value, and add the new item.\n\t\t \n\t\t This is more efficient than heappop() followed by heappush(), and can be\n\t\t more appropriate when using a fixed size heap. Note that the value\n\t\t returned may be larger than item! That constrains reasonable use of\n\t\t this routine unless written as part of a conditional replacement:\n\t\t if item > array[0]\n\t\t item = heapreplace(array, item)\n\t\t */\n\n\t\t heapreplace = function(array, item, cmp) {\n\t\t var returnitem;\n\t\t if (cmp == null) {\n\t\t cmp = defaultCmp;\n\t\t }\n\t\t returnitem = array[0];\n\t\t array[0] = item;\n\t\t _siftup(array, 0, cmp);\n\t\t return returnitem;\n\t\t };\n\n\n\t\t /*\n\t\t Fast version of a heappush followed by a heappop.\n\t\t */\n\n\t\t heappushpop = function(array, item, cmp) {\n\t\t var _ref;\n\t\t if (cmp == null) {\n\t\t cmp = defaultCmp;\n\t\t }\n\t\t if (array.length && cmp(array[0], item) < 0) {\n\t\t _ref = [array[0], item], item = _ref[0], array[0] = _ref[1];\n\t\t _siftup(array, 0, cmp);\n\t\t }\n\t\t return item;\n\t\t };\n\n\n\t\t /*\n\t\t Transform list into a heap, in-place, in O(array.length) time.\n\t\t */\n\n\t\t heapify = function(array, cmp) {\n\t\t var i, _i, _len, _ref1, _results, _results1;\n\t\t if (cmp == null) {\n\t\t cmp = defaultCmp;\n\t\t }\n\t\t _ref1 = (function() {\n\t\t _results1 = [];\n\t\t for (var _j = 0, _ref = floor(array.length / 2); 0 <= _ref ? _j < _ref : _j > _ref; 0 <= _ref ? _j++ : _j--){ _results1.push(_j); }\n\t\t return _results1;\n\t\t }).apply(this).reverse();\n\t\t _results = [];\n\t\t for (_i = 0, _len = _ref1.length; _i < _len; _i++) {\n\t\t i = _ref1[_i];\n\t\t _results.push(_siftup(array, i, cmp));\n\t\t }\n\t\t return _results;\n\t\t };\n\n\n\t\t /*\n\t\t Update the position of the given item in the heap.\n\t\t This function should be called every time the item is being modified.\n\t\t */\n\n\t\t updateItem = function(array, item, cmp) {\n\t\t var pos;\n\t\t if (cmp == null) {\n\t\t cmp = defaultCmp;\n\t\t }\n\t\t pos = array.indexOf(item);\n\t\t if (pos === -1) {\n\t\t return;\n\t\t }\n\t\t _siftdown(array, 0, pos, cmp);\n\t\t return _siftup(array, pos, cmp);\n\t\t };\n\n\n\t\t /*\n\t\t Find the n largest elements in a dataset.\n\t\t */\n\n\t\t nlargest = function(array, n, cmp) {\n\t\t var elem, result, _i, _len, _ref;\n\t\t if (cmp == null) {\n\t\t cmp = defaultCmp;\n\t\t }\n\t\t result = array.slice(0, n);\n\t\t if (!result.length) {\n\t\t return result;\n\t\t }\n\t\t heapify(result, cmp);\n\t\t _ref = array.slice(n);\n\t\t for (_i = 0, _len = _ref.length; _i < _len; _i++) {\n\t\t elem = _ref[_i];\n\t\t heappushpop(result, elem, cmp);\n\t\t }\n\t\t return result.sort(cmp).reverse();\n\t\t };\n\n\n\t\t /*\n\t\t Find the n smallest elements in a dataset.\n\t\t */\n\n\t\t nsmallest = function(array, n, cmp) {\n\t\t var elem, los, result, _i, _j, _len, _ref, _ref1, _results;\n\t\t if (cmp == null) {\n\t\t cmp = defaultCmp;\n\t\t }\n\t\t if (n * 10 <= array.length) {\n\t\t result = array.slice(0, n).sort(cmp);\n\t\t if (!result.length) {\n\t\t return result;\n\t\t }\n\t\t los = result[result.length - 1];\n\t\t _ref = array.slice(n);\n\t\t for (_i = 0, _len = _ref.length; _i < _len; _i++) {\n\t\t elem = _ref[_i];\n\t\t if (cmp(elem, los) < 0) {\n\t\t insort(result, elem, 0, null, cmp);\n\t\t result.pop();\n\t\t los = result[result.length - 1];\n\t\t }\n\t\t }\n\t\t return result;\n\t\t }\n\t\t heapify(array, cmp);\n\t\t _results = [];\n\t\t for (_j = 0, _ref1 = min(n, array.length); 0 <= _ref1 ? _j < _ref1 : _j > _ref1; 0 <= _ref1 ? ++_j : --_j) {\n\t\t _results.push(heappop(array, cmp));\n\t\t }\n\t\t return _results;\n\t\t };\n\n\t\t _siftdown = function(array, startpos, pos, cmp) {\n\t\t var newitem, parent, parentpos;\n\t\t if (cmp == null) {\n\t\t cmp = defaultCmp;\n\t\t }\n\t\t newitem = array[pos];\n\t\t while (pos > startpos) {\n\t\t parentpos = (pos - 1) >> 1;\n\t\t parent = array[parentpos];\n\t\t if (cmp(newitem, parent) < 0) {\n\t\t array[pos] = parent;\n\t\t pos = parentpos;\n\t\t continue;\n\t\t }\n\t\t break;\n\t\t }\n\t\t return array[pos] = newitem;\n\t\t };\n\n\t\t _siftup = function(array, pos, cmp) {\n\t\t var childpos, endpos, newitem, rightpos, startpos;\n\t\t if (cmp == null) {\n\t\t cmp = defaultCmp;\n\t\t }\n\t\t endpos = array.length;\n\t\t startpos = pos;\n\t\t newitem = array[pos];\n\t\t childpos = 2 * pos + 1;\n\t\t while (childpos < endpos) {\n\t\t rightpos = childpos + 1;\n\t\t if (rightpos < endpos && !(cmp(array[childpos], array[rightpos]) < 0)) {\n\t\t childpos = rightpos;\n\t\t }\n\t\t array[pos] = array[childpos];\n\t\t pos = childpos;\n\t\t childpos = 2 * pos + 1;\n\t\t }\n\t\t array[pos] = newitem;\n\t\t return _siftdown(array, startpos, pos, cmp);\n\t\t };\n\n\t\t Heap = (function() {\n\t\t Heap.push = heappush;\n\n\t\t Heap.pop = heappop;\n\n\t\t Heap.replace = heapreplace;\n\n\t\t Heap.pushpop = heappushpop;\n\n\t\t Heap.heapify = heapify;\n\n\t\t Heap.updateItem = updateItem;\n\n\t\t Heap.nlargest = nlargest;\n\n\t\t Heap.nsmallest = nsmallest;\n\n\t\t function Heap(cmp) {\n\t\t this.cmp = cmp != null ? cmp : defaultCmp;\n\t\t this.nodes = [];\n\t\t }\n\n\t\t Heap.prototype.push = function(x) {\n\t\t return heappush(this.nodes, x, this.cmp);\n\t\t };\n\n\t\t Heap.prototype.pop = function() {\n\t\t return heappop(this.nodes, this.cmp);\n\t\t };\n\n\t\t Heap.prototype.peek = function() {\n\t\t return this.nodes[0];\n\t\t };\n\n\t\t Heap.prototype.contains = function(x) {\n\t\t return this.nodes.indexOf(x) !== -1;\n\t\t };\n\n\t\t Heap.prototype.replace = function(x) {\n\t\t return heapreplace(this.nodes, x, this.cmp);\n\t\t };\n\n\t\t Heap.prototype.pushpop = function(x) {\n\t\t return heappushpop(this.nodes, x, this.cmp);\n\t\t };\n\n\t\t Heap.prototype.heapify = function() {\n\t\t return heapify(this.nodes, this.cmp);\n\t\t };\n\n\t\t Heap.prototype.updateItem = function(x) {\n\t\t return updateItem(this.nodes, x, this.cmp);\n\t\t };\n\n\t\t Heap.prototype.clear = function() {\n\t\t return this.nodes = [];\n\t\t };\n\n\t\t Heap.prototype.empty = function() {\n\t\t return this.nodes.length === 0;\n\t\t };\n\n\t\t Heap.prototype.size = function() {\n\t\t return this.nodes.length;\n\t\t };\n\n\t\t Heap.prototype.clone = function() {\n\t\t var heap;\n\t\t heap = new Heap();\n\t\t heap.nodes = this.nodes.slice(0);\n\t\t return heap;\n\t\t };\n\n\t\t Heap.prototype.toArray = function() {\n\t\t return this.nodes.slice(0);\n\t\t };\n\n\t\t Heap.prototype.insert = Heap.prototype.push;\n\n\t\t Heap.prototype.top = Heap.prototype.peek;\n\n\t\t Heap.prototype.front = Heap.prototype.peek;\n\n\t\t Heap.prototype.has = Heap.prototype.contains;\n\n\t\t Heap.prototype.copy = Heap.prototype.clone;\n\n\t\t return Heap;\n\n\t\t })();\n\n\t\t (function(root, factory) {\n\t\t {\n\t\t return module.exports = factory();\n\t\t }\n\t\t })(this, function() {\n\t\t return Heap;\n\t\t });\n\n\t\t}).call(heap$1); \n\t} (heap$2));\n\treturn heap$2.exports;\n}\n\nvar heap;\nvar hasRequiredHeap;\n\nfunction requireHeap () {\n\tif (hasRequiredHeap) return heap;\n\thasRequiredHeap = 1;\n\theap = requireHeap$1();\n\treturn heap;\n}\n\nvar heapExports = requireHeap();\nvar Heap = /*@__PURE__*/getDefaultExportFromCjs(heapExports);\n\nvar dijkstraDefaults = defaults$g({\n root: null,\n weight: function weight(edge) {\n return 1;\n },\n directed: false\n});\nvar elesfn$u = {\n dijkstra: function dijkstra(options) {\n if (!plainObject(options)) {\n var args = arguments;\n options = {\n root: args[0],\n weight: args[1],\n directed: args[2]\n };\n }\n var _dijkstraDefaults = dijkstraDefaults(options),\n root = _dijkstraDefaults.root,\n weight = _dijkstraDefaults.weight,\n directed = _dijkstraDefaults.directed;\n var eles = this;\n var weightFn = weight;\n var source = string(root) ? this.filter(root)[0] : root[0];\n var dist = {};\n var prev = {};\n var knownDist = {};\n var _this$byGroup = this.byGroup(),\n nodes = _this$byGroup.nodes,\n edges = _this$byGroup.edges;\n edges.unmergeBy(function (ele) {\n return ele.isLoop();\n });\n var getDist = function getDist(node) {\n return dist[node.id()];\n };\n var setDist = function setDist(node, d) {\n dist[node.id()] = d;\n Q.updateItem(node);\n };\n var Q = new Heap(function (a, b) {\n return getDist(a) - getDist(b);\n });\n for (var i = 0; i < nodes.length; i++) {\n var node = nodes[i];\n dist[node.id()] = node.same(source) ? 0 : Infinity;\n Q.push(node);\n }\n var distBetween = function distBetween(u, v) {\n var uvs = (directed ? u.edgesTo(v) : u.edgesWith(v)).intersect(edges);\n var smallestDistance = Infinity;\n var smallestEdge;\n for (var _i = 0; _i < uvs.length; _i++) {\n var edge = uvs[_i];\n var _weight = weightFn(edge);\n if (_weight < smallestDistance || !smallestEdge) {\n smallestDistance = _weight;\n smallestEdge = edge;\n }\n }\n return {\n edge: smallestEdge,\n dist: smallestDistance\n };\n };\n while (Q.size() > 0) {\n var u = Q.pop();\n var smalletsDist = getDist(u);\n var uid = u.id();\n knownDist[uid] = smalletsDist;\n if (smalletsDist === Infinity) {\n continue;\n }\n var neighbors = u.neighborhood().intersect(nodes);\n for (var _i2 = 0; _i2 < neighbors.length; _i2++) {\n var v = neighbors[_i2];\n var vid = v.id();\n var vDist = distBetween(u, v);\n var alt = smalletsDist + vDist.dist;\n if (alt < getDist(v)) {\n setDist(v, alt);\n prev[vid] = {\n node: u,\n edge: vDist.edge\n };\n }\n } // for\n } // while\n\n return {\n distanceTo: function distanceTo(node) {\n var target = string(node) ? nodes.filter(node)[0] : node[0];\n return knownDist[target.id()];\n },\n pathTo: function pathTo(node) {\n var target = string(node) ? nodes.filter(node)[0] : node[0];\n var S = [];\n var u = target;\n var uid = u.id();\n if (target.length > 0) {\n S.unshift(target);\n while (prev[uid]) {\n var p = prev[uid];\n S.unshift(p.edge);\n S.unshift(p.node);\n u = p.node;\n uid = u.id();\n }\n }\n return eles.spawn(S);\n }\n };\n }\n};\n\nvar elesfn$t = {\n // kruskal's algorithm (finds min spanning tree, assuming undirected graph)\n // implemented from pseudocode from wikipedia\n kruskal: function kruskal(weightFn) {\n weightFn = weightFn || function (edge) {\n return 1;\n };\n var _this$byGroup = this.byGroup(),\n nodes = _this$byGroup.nodes,\n edges = _this$byGroup.edges;\n var numNodes = nodes.length;\n var forest = new Array(numNodes);\n var A = nodes; // assumes byGroup() creates new collections that can be safely mutated\n\n var findSetIndex = function findSetIndex(ele) {\n for (var i = 0; i < forest.length; i++) {\n var eles = forest[i];\n if (eles.has(ele)) {\n return i;\n }\n }\n };\n\n // start with one forest per node\n for (var i = 0; i < numNodes; i++) {\n forest[i] = this.spawn(nodes[i]);\n }\n var S = edges.sort(function (a, b) {\n return weightFn(a) - weightFn(b);\n });\n for (var _i = 0; _i < S.length; _i++) {\n var edge = S[_i];\n var u = edge.source()[0];\n var v = edge.target()[0];\n var setUIndex = findSetIndex(u);\n var setVIndex = findSetIndex(v);\n var setU = forest[setUIndex];\n var setV = forest[setVIndex];\n if (setUIndex !== setVIndex) {\n A.merge(edge);\n\n // combine forests for u and v\n setU.merge(setV);\n forest.splice(setVIndex, 1);\n }\n }\n return A;\n }\n};\n\nvar aStarDefaults = defaults$g({\n root: null,\n goal: null,\n weight: function weight(edge) {\n return 1;\n },\n heuristic: function heuristic(edge) {\n return 0;\n },\n directed: false\n});\nvar elesfn$s = {\n // Implemented from pseudocode from wikipedia\n aStar: function aStar(options) {\n var cy = this.cy();\n var _aStarDefaults = aStarDefaults(options),\n root = _aStarDefaults.root,\n goal = _aStarDefaults.goal,\n heuristic = _aStarDefaults.heuristic,\n directed = _aStarDefaults.directed,\n weight = _aStarDefaults.weight;\n root = cy.collection(root)[0];\n goal = cy.collection(goal)[0];\n var sid = root.id();\n var tid = goal.id();\n var gScore = {};\n var fScore = {};\n var closedSetIds = {};\n var openSet = new Heap(function (a, b) {\n return fScore[a.id()] - fScore[b.id()];\n });\n var openSetIds = new Set$1();\n var cameFrom = {};\n var cameFromEdge = {};\n var addToOpenSet = function addToOpenSet(ele, id) {\n openSet.push(ele);\n openSetIds.add(id);\n };\n var cMin, cMinId;\n var popFromOpenSet = function popFromOpenSet() {\n cMin = openSet.pop();\n cMinId = cMin.id();\n openSetIds[\"delete\"](cMinId);\n };\n var isInOpenSet = function isInOpenSet(id) {\n return openSetIds.has(id);\n };\n addToOpenSet(root, sid);\n gScore[sid] = 0;\n fScore[sid] = heuristic(root);\n\n // Counter\n var steps = 0;\n\n // Main loop\n while (openSet.size() > 0) {\n popFromOpenSet();\n steps++;\n\n // If we've found our goal, then we are done\n if (cMinId === tid) {\n var path = [];\n var pathNode = goal;\n var pathNodeId = tid;\n var pathEdge = cameFromEdge[pathNodeId];\n for (;;) {\n path.unshift(pathNode);\n if (pathEdge != null) {\n path.unshift(pathEdge);\n }\n pathNode = cameFrom[pathNodeId];\n if (pathNode == null) {\n break;\n }\n pathNodeId = pathNode.id();\n pathEdge = cameFromEdge[pathNodeId];\n }\n return {\n found: true,\n distance: gScore[cMinId],\n path: this.spawn(path),\n steps: steps\n };\n }\n\n // Add cMin to processed nodes\n closedSetIds[cMinId] = true;\n\n // Update scores for neighbors of cMin\n // Take into account if graph is directed or not\n var vwEdges = cMin._private.edges;\n for (var i = 0; i < vwEdges.length; i++) {\n var e = vwEdges[i];\n\n // edge must be in set of calling eles\n if (!this.hasElementWithId(e.id())) {\n continue;\n }\n\n // cMin must be the source of edge if directed\n if (directed && e.data('source') !== cMinId) {\n continue;\n }\n var wSrc = e.source();\n var wTgt = e.target();\n var w = wSrc.id() !== cMinId ? wSrc : wTgt;\n var wid = w.id();\n\n // node must be in set of calling eles\n if (!this.hasElementWithId(wid)) {\n continue;\n }\n\n // if node is in closedSet, ignore it\n if (closedSetIds[wid]) {\n continue;\n }\n\n // New tentative score for node w\n var tempScore = gScore[cMinId] + weight(e);\n\n // Update gScore for node w if:\n // w not present in openSet\n // OR\n // tentative gScore is less than previous value\n\n // w not in openSet\n if (!isInOpenSet(wid)) {\n gScore[wid] = tempScore;\n fScore[wid] = tempScore + heuristic(w);\n addToOpenSet(w, wid);\n cameFrom[wid] = cMin;\n cameFromEdge[wid] = e;\n continue;\n }\n\n // w already in openSet, but with greater gScore\n if (tempScore < gScore[wid]) {\n gScore[wid] = tempScore;\n fScore[wid] = tempScore + heuristic(w);\n cameFrom[wid] = cMin;\n cameFromEdge[wid] = e;\n }\n } // End of neighbors update\n } // End of main loop\n\n // If we've reached here, then we've not reached our goal\n return {\n found: false,\n distance: undefined,\n path: undefined,\n steps: steps\n };\n }\n}; // elesfn\n\nvar floydWarshallDefaults = defaults$g({\n weight: function weight(edge) {\n return 1;\n },\n directed: false\n});\nvar elesfn$r = {\n // Implemented from pseudocode from wikipedia\n floydWarshall: function floydWarshall(options) {\n var cy = this.cy();\n var _floydWarshallDefault = floydWarshallDefaults(options),\n weight = _floydWarshallDefault.weight,\n directed = _floydWarshallDefault.directed;\n var weightFn = weight;\n var _this$byGroup = this.byGroup(),\n nodes = _this$byGroup.nodes,\n edges = _this$byGroup.edges;\n var N = nodes.length;\n var Nsq = N * N;\n var indexOf = function indexOf(node) {\n return nodes.indexOf(node);\n };\n var atIndex = function atIndex(i) {\n return nodes[i];\n };\n\n // Initialize distance matrix\n var dist = new Array(Nsq);\n for (var n = 0; n < Nsq; n++) {\n var j = n % N;\n var i = (n - j) / N;\n if (i === j) {\n dist[n] = 0;\n } else {\n dist[n] = Infinity;\n }\n }\n\n // Initialize matrix used for path reconstruction\n // Initialize distance matrix\n var next = new Array(Nsq);\n var edgeNext = new Array(Nsq);\n\n // Process edges\n for (var _i = 0; _i < edges.length; _i++) {\n var edge = edges[_i];\n var src = edge.source()[0];\n var tgt = edge.target()[0];\n if (src === tgt) {\n continue;\n } // exclude loops\n\n var s = indexOf(src);\n var t = indexOf(tgt);\n var st = s * N + t; // source to target index\n var _weight = weightFn(edge);\n\n // Check if already process another edge between same 2 nodes\n if (dist[st] > _weight) {\n dist[st] = _weight;\n next[st] = t;\n edgeNext[st] = edge;\n }\n\n // If undirected graph, process 'reversed' edge\n if (!directed) {\n var ts = t * N + s; // target to source index\n\n if (!directed && dist[ts] > _weight) {\n dist[ts] = _weight;\n next[ts] = s;\n edgeNext[ts] = edge;\n }\n }\n }\n\n // Main loop\n for (var k = 0; k < N; k++) {\n for (var _i2 = 0; _i2 < N; _i2++) {\n var ik = _i2 * N + k;\n for (var _j = 0; _j < N; _j++) {\n var ij = _i2 * N + _j;\n var kj = k * N + _j;\n if (dist[ik] + dist[kj] < dist[ij]) {\n dist[ij] = dist[ik] + dist[kj];\n next[ij] = next[ik];\n }\n }\n }\n }\n var getArgEle = function getArgEle(ele) {\n return (string(ele) ? cy.filter(ele) : ele)[0];\n };\n var indexOfArgEle = function indexOfArgEle(ele) {\n return indexOf(getArgEle(ele));\n };\n var res = {\n distance: function distance(from, to) {\n var i = indexOfArgEle(from);\n var j = indexOfArgEle(to);\n return dist[i * N + j];\n },\n path: function path(from, to) {\n var i = indexOfArgEle(from);\n var j = indexOfArgEle(to);\n var fromNode = atIndex(i);\n if (i === j) {\n return fromNode.collection();\n }\n if (next[i * N + j] == null) {\n return cy.collection();\n }\n var path = cy.collection();\n var prev = i;\n var edge;\n path.merge(fromNode);\n while (i !== j) {\n prev = i;\n i = next[i * N + j];\n edge = edgeNext[prev * N + i];\n path.merge(edge);\n path.merge(atIndex(i));\n }\n return path;\n }\n };\n return res;\n } // floydWarshall\n}; // elesfn\n\nvar bellmanFordDefaults = defaults$g({\n weight: function weight(edge) {\n return 1;\n },\n directed: false,\n root: null\n});\nvar elesfn$q = {\n // Implemented from pseudocode from wikipedia\n bellmanFord: function bellmanFord(options) {\n var _this = this;\n var _bellmanFordDefaults = bellmanFordDefaults(options),\n weight = _bellmanFordDefaults.weight,\n directed = _bellmanFordDefaults.directed,\n root = _bellmanFordDefaults.root;\n var weightFn = weight;\n var eles = this;\n var cy = this.cy();\n var _this$byGroup = this.byGroup(),\n edges = _this$byGroup.edges,\n nodes = _this$byGroup.nodes;\n var numNodes = nodes.length;\n var infoMap = new Map$1();\n var hasNegativeWeightCycle = false;\n var negativeWeightCycles = [];\n root = cy.collection(root)[0]; // in case selector passed\n\n edges.unmergeBy(function (edge) {\n return edge.isLoop();\n });\n var numEdges = edges.length;\n var getInfo = function getInfo(node) {\n var obj = infoMap.get(node.id());\n if (!obj) {\n obj = {};\n infoMap.set(node.id(), obj);\n }\n return obj;\n };\n var getNodeFromTo = function getNodeFromTo(to) {\n return (string(to) ? cy.$(to) : to)[0];\n };\n var distanceTo = function distanceTo(to) {\n return getInfo(getNodeFromTo(to)).dist;\n };\n var pathTo = function pathTo(to) {\n var thisStart = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : root;\n var end = getNodeFromTo(to);\n var path = [];\n var node = end;\n for (;;) {\n if (node == null) {\n return _this.spawn();\n }\n var _getInfo = getInfo(node),\n edge = _getInfo.edge,\n pred = _getInfo.pred;\n path.unshift(node[0]);\n if (node.same(thisStart) && path.length > 0) {\n break;\n }\n if (edge != null) {\n path.unshift(edge);\n }\n node = pred;\n }\n return eles.spawn(path);\n };\n\n // Initializations { dist, pred, edge }\n for (var i = 0; i < numNodes; i++) {\n var node = nodes[i];\n var info = getInfo(node);\n if (node.same(root)) {\n info.dist = 0;\n } else {\n info.dist = Infinity;\n }\n info.pred = null;\n info.edge = null;\n }\n\n // Edges relaxation\n var replacedEdge = false;\n var checkForEdgeReplacement = function checkForEdgeReplacement(node1, node2, edge, info1, info2, weight) {\n var dist = info1.dist + weight;\n if (dist < info2.dist && !edge.same(info1.edge)) {\n info2.dist = dist;\n info2.pred = node1;\n info2.edge = edge;\n replacedEdge = true;\n }\n };\n for (var _i = 1; _i < numNodes; _i++) {\n replacedEdge = false;\n for (var e = 0; e < numEdges; e++) {\n var edge = edges[e];\n var src = edge.source();\n var tgt = edge.target();\n var _weight = weightFn(edge);\n var srcInfo = getInfo(src);\n var tgtInfo = getInfo(tgt);\n checkForEdgeReplacement(src, tgt, edge, srcInfo, tgtInfo, _weight);\n\n // If undirected graph, we need to take into account the 'reverse' edge\n if (!directed) {\n checkForEdgeReplacement(tgt, src, edge, tgtInfo, srcInfo, _weight);\n }\n }\n if (!replacedEdge) {\n break;\n }\n }\n if (replacedEdge) {\n // Check for negative weight cycles\n var negativeWeightCycleIds = [];\n for (var _e = 0; _e < numEdges; _e++) {\n var _edge = edges[_e];\n var _src = _edge.source();\n var _tgt = _edge.target();\n var _weight2 = weightFn(_edge);\n var srcDist = getInfo(_src).dist;\n var tgtDist = getInfo(_tgt).dist;\n if (srcDist + _weight2 < tgtDist || !directed && tgtDist + _weight2 < srcDist) {\n if (!hasNegativeWeightCycle) {\n warn('Graph contains a negative weight cycle for Bellman-Ford');\n hasNegativeWeightCycle = true;\n }\n if (options.findNegativeWeightCycles !== false) {\n var negativeNodes = [];\n if (srcDist + _weight2 < tgtDist) {\n negativeNodes.push(_src);\n }\n if (!directed && tgtDist + _weight2 < srcDist) {\n negativeNodes.push(_tgt);\n }\n var numNegativeNodes = negativeNodes.length;\n for (var n = 0; n < numNegativeNodes; n++) {\n var start = negativeNodes[n];\n var cycle = [start];\n cycle.push(getInfo(start).edge);\n var _node = getInfo(start).pred;\n while (cycle.indexOf(_node) === -1) {\n cycle.push(_node);\n cycle.push(getInfo(_node).edge);\n _node = getInfo(_node).pred;\n }\n cycle = cycle.slice(cycle.indexOf(_node));\n var smallestId = cycle[0].id();\n var smallestIndex = 0;\n for (var c = 2; c < cycle.length; c += 2) {\n if (cycle[c].id() < smallestId) {\n smallestId = cycle[c].id();\n smallestIndex = c;\n }\n }\n cycle = cycle.slice(smallestIndex).concat(cycle.slice(0, smallestIndex));\n cycle.push(cycle[0]);\n var cycleId = cycle.map(function (el) {\n return el.id();\n }).join(\",\");\n if (negativeWeightCycleIds.indexOf(cycleId) === -1) {\n negativeWeightCycles.push(eles.spawn(cycle));\n negativeWeightCycleIds.push(cycleId);\n }\n }\n } else {\n break;\n }\n }\n }\n }\n return {\n distanceTo: distanceTo,\n pathTo: pathTo,\n hasNegativeWeightCycle: hasNegativeWeightCycle,\n negativeWeightCycles: negativeWeightCycles\n };\n } // bellmanFord\n}; // elesfn\n\nvar sqrt2 = Math.sqrt(2);\n\n// Function which colapses 2 (meta) nodes into one\n// Updates the remaining edge lists\n// Receives as a paramater the edge which causes the collapse\nvar collapse = function collapse(edgeIndex, nodeMap, remainingEdges) {\n if (remainingEdges.length === 0) {\n error(\"Karger-Stein must be run on a connected (sub)graph\");\n }\n var edgeInfo = remainingEdges[edgeIndex];\n var sourceIn = edgeInfo[1];\n var targetIn = edgeInfo[2];\n var partition1 = nodeMap[sourceIn];\n var partition2 = nodeMap[targetIn];\n var newEdges = remainingEdges; // re-use array\n\n // Delete all edges between partition1 and partition2\n for (var i = newEdges.length - 1; i >= 0; i--) {\n var edge = newEdges[i];\n var src = edge[1];\n var tgt = edge[2];\n if (nodeMap[src] === partition1 && nodeMap[tgt] === partition2 || nodeMap[src] === partition2 && nodeMap[tgt] === partition1) {\n newEdges.splice(i, 1);\n }\n }\n\n // All edges pointing to partition2 should now point to partition1\n for (var _i = 0; _i < newEdges.length; _i++) {\n var _edge = newEdges[_i];\n if (_edge[1] === partition2) {\n // Check source\n newEdges[_i] = _edge.slice(); // copy\n newEdges[_i][1] = partition1;\n } else if (_edge[2] === partition2) {\n // Check target\n newEdges[_i] = _edge.slice(); // copy\n newEdges[_i][2] = partition1;\n }\n }\n\n // Move all nodes from partition2 to partition1\n for (var _i2 = 0; _i2 < nodeMap.length; _i2++) {\n if (nodeMap[_i2] === partition2) {\n nodeMap[_i2] = partition1;\n }\n }\n return newEdges;\n};\n\n// Contracts a graph until we reach a certain number of meta nodes\nvar contractUntil = function contractUntil(metaNodeMap, remainingEdges, size, sizeLimit) {\n while (size > sizeLimit) {\n // Choose an edge randomly\n var edgeIndex = Math.floor(Math.random() * remainingEdges.length);\n\n // Collapse graph based on edge\n remainingEdges = collapse(edgeIndex, metaNodeMap, remainingEdges);\n size--;\n }\n return remainingEdges;\n};\nvar elesfn$p = {\n // Computes the minimum cut of an undirected graph\n // Returns the correct answer with high probability\n kargerStein: function kargerStein() {\n var _this = this;\n var _this$byGroup = this.byGroup(),\n nodes = _this$byGroup.nodes,\n edges = _this$byGroup.edges;\n edges.unmergeBy(function (edge) {\n return edge.isLoop();\n });\n var numNodes = nodes.length;\n var numEdges = edges.length;\n var numIter = Math.ceil(Math.pow(Math.log(numNodes) / Math.LN2, 2));\n var stopSize = Math.floor(numNodes / sqrt2);\n if (numNodes < 2) {\n error('At least 2 nodes are required for Karger-Stein algorithm');\n return undefined;\n }\n\n // Now store edge destination as indexes\n // Format for each edge (edge index, source node index, target node index)\n var edgeIndexes = [];\n for (var i = 0; i < numEdges; i++) {\n var e = edges[i];\n edgeIndexes.push([i, nodes.indexOf(e.source()), nodes.indexOf(e.target())]);\n }\n\n // We will store the best cut found here\n var minCutSize = Infinity;\n var minCutEdgeIndexes = [];\n var minCutNodeMap = new Array(numNodes);\n\n // Initial meta node partition\n var metaNodeMap = new Array(numNodes);\n var metaNodeMap2 = new Array(numNodes);\n var copyNodesMap = function copyNodesMap(from, to) {\n for (var _i3 = 0; _i3 < numNodes; _i3++) {\n to[_i3] = from[_i3];\n }\n };\n\n // Main loop\n for (var iter = 0; iter <= numIter; iter++) {\n // Reset meta node partition\n for (var _i4 = 0; _i4 < numNodes; _i4++) {\n metaNodeMap[_i4] = _i4;\n }\n\n // Contract until stop point (stopSize nodes)\n var edgesState = contractUntil(metaNodeMap, edgeIndexes.slice(), numNodes, stopSize);\n var edgesState2 = edgesState.slice(); // copy\n\n // Create a copy of the colapsed nodes state\n copyNodesMap(metaNodeMap, metaNodeMap2);\n\n // Run 2 iterations starting in the stop state\n var res1 = contractUntil(metaNodeMap, edgesState, stopSize, 2);\n var res2 = contractUntil(metaNodeMap2, edgesState2, stopSize, 2);\n\n // Is any of the 2 results the best cut so far?\n if (res1.length <= res2.length && res1.length < minCutSize) {\n minCutSize = res1.length;\n minCutEdgeIndexes = res1;\n copyNodesMap(metaNodeMap, minCutNodeMap);\n } else if (res2.length <= res1.length && res2.length < minCutSize) {\n minCutSize = res2.length;\n minCutEdgeIndexes = res2;\n copyNodesMap(metaNodeMap2, minCutNodeMap);\n }\n } // end of main loop\n\n // Construct result\n var cut = this.spawn(minCutEdgeIndexes.map(function (e) {\n return edges[e[0]];\n }));\n var partition1 = this.spawn();\n var partition2 = this.spawn();\n\n // traverse metaNodeMap for best cut\n var witnessNodePartition = minCutNodeMap[0];\n for (var _i5 = 0; _i5 < minCutNodeMap.length; _i5++) {\n var partitionId = minCutNodeMap[_i5];\n var node = nodes[_i5];\n if (partitionId === witnessNodePartition) {\n partition1.merge(node);\n } else {\n partition2.merge(node);\n }\n }\n\n // construct components corresponding to each disjoint subset of nodes\n var constructComponent = function constructComponent(subset) {\n var component = _this.spawn();\n subset.forEach(function (node) {\n component.merge(node);\n node.connectedEdges().forEach(function (edge) {\n // ensure edge is within calling collection and edge is not in cut\n if (_this.contains(edge) && !cut.contains(edge)) {\n component.merge(edge);\n }\n });\n });\n return component;\n };\n var components = [constructComponent(partition1), constructComponent(partition2)];\n var ret = {\n cut: cut,\n components: components,\n // n.b. partitions are included to be compatible with the old api spec\n // (could be removed in a future major version)\n partition1: partition1,\n partition2: partition2\n };\n return ret;\n }\n}; // elesfn\n\nvar _Math$hypot;\nvar copyPosition = function copyPosition(p) {\n return {\n x: p.x,\n y: p.y\n };\n};\nvar modelToRenderedPosition$1 = function modelToRenderedPosition(p, zoom, pan) {\n return {\n x: p.x * zoom + pan.x,\n y: p.y * zoom + pan.y\n };\n};\nvar renderedToModelPosition = function renderedToModelPosition(p, zoom, pan) {\n return {\n x: (p.x - pan.x) / zoom,\n y: (p.y - pan.y) / zoom\n };\n};\nvar array2point = function array2point(arr) {\n return {\n x: arr[0],\n y: arr[1]\n };\n};\nvar min = function min(arr) {\n var begin = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 0;\n var end = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : arr.length;\n var min = Infinity;\n for (var i = begin; i < end; i++) {\n var val = arr[i];\n if (isFinite(val)) {\n min = Math.min(val, min);\n }\n }\n return min;\n};\nvar max = function max(arr) {\n var begin = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 0;\n var end = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : arr.length;\n var max = -Infinity;\n for (var i = begin; i < end; i++) {\n var val = arr[i];\n if (isFinite(val)) {\n max = Math.max(val, max);\n }\n }\n return max;\n};\nvar mean = function mean(arr) {\n var begin = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 0;\n var end = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : arr.length;\n var total = 0;\n var n = 0;\n for (var i = begin; i < end; i++) {\n var val = arr[i];\n if (isFinite(val)) {\n total += val;\n n++;\n }\n }\n return total / n;\n};\nvar median = function median(arr) {\n var begin = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 0;\n var end = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : arr.length;\n var copy = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : true;\n var sort = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : true;\n var includeHoles = arguments.length > 5 && arguments[5] !== undefined ? arguments[5] : true;\n if (copy) {\n arr = arr.slice(begin, end);\n } else {\n if (end < arr.length) {\n arr.splice(end, arr.length - end);\n }\n if (begin > 0) {\n arr.splice(0, begin);\n }\n }\n\n // all non finite (e.g. Infinity, NaN) elements must be -Infinity so they go to the start\n var off = 0; // offset from non-finite values\n for (var i = arr.length - 1; i >= 0; i--) {\n var v = arr[i];\n if (includeHoles) {\n if (!isFinite(v)) {\n arr[i] = -Infinity;\n off++;\n }\n } else {\n // just remove it if we don't want to consider holes\n arr.splice(i, 1);\n }\n }\n if (sort) {\n arr.sort(function (a, b) {\n return a - b;\n }); // requires copy = true if you don't want to change the orig\n }\n var len = arr.length;\n var mid = Math.floor(len / 2);\n if (len % 2 !== 0) {\n return arr[mid + 1 + off];\n } else {\n return (arr[mid - 1 + off] + arr[mid + off]) / 2;\n }\n};\nvar deg2rad = function deg2rad(deg) {\n return Math.PI * deg / 180;\n};\nvar getAngleFromDisp = function getAngleFromDisp(dispX, dispY) {\n return Math.atan2(dispY, dispX) - Math.PI / 2;\n};\nvar log2 = Math.log2 || function (n) {\n return Math.log(n) / Math.log(2);\n};\nvar signum = function signum(x) {\n if (x > 0) {\n return 1;\n } else if (x < 0) {\n return -1;\n } else {\n return 0;\n }\n};\nvar dist = function dist(p1, p2) {\n return Math.sqrt(sqdist(p1, p2));\n};\nvar sqdist = function sqdist(p1, p2) {\n var dx = p2.x - p1.x;\n var dy = p2.y - p1.y;\n return dx * dx + dy * dy;\n};\nvar inPlaceSumNormalize = function inPlaceSumNormalize(v) {\n var length = v.length;\n\n // First, get sum of all elements\n var total = 0;\n for (var i = 0; i < length; i++) {\n total += v[i];\n }\n\n // Now, divide each by the sum of all elements\n for (var _i = 0; _i < length; _i++) {\n v[_i] = v[_i] / total;\n }\n return v;\n};\n\n// from http://en.wikipedia.org/wiki/Bézier_curve#Quadratic_curves\nvar qbezierAt = function qbezierAt(p0, p1, p2, t) {\n return (1 - t) * (1 - t) * p0 + 2 * (1 - t) * t * p1 + t * t * p2;\n};\nvar qbezierPtAt = function qbezierPtAt(p0, p1, p2, t) {\n return {\n x: qbezierAt(p0.x, p1.x, p2.x, t),\n y: qbezierAt(p0.y, p1.y, p2.y, t)\n };\n};\nvar lineAt = function lineAt(p0, p1, t, d) {\n var vec = {\n x: p1.x - p0.x,\n y: p1.y - p0.y\n };\n var vecDist = dist(p0, p1);\n var normVec = {\n x: vec.x / vecDist,\n y: vec.y / vecDist\n };\n t = t == null ? 0 : t;\n d = d != null ? d : t * vecDist;\n return {\n x: p0.x + normVec.x * d,\n y: p0.y + normVec.y * d\n };\n};\nvar bound = function bound(min, val, max) {\n return Math.max(min, Math.min(max, val));\n};\n\n// makes a full bb (x1, y1, x2, y2, w, h) from implicit params\nvar makeBoundingBox = function makeBoundingBox(bb) {\n if (bb == null) {\n return {\n x1: Infinity,\n y1: Infinity,\n x2: -Infinity,\n y2: -Infinity,\n w: 0,\n h: 0\n };\n } else if (bb.x1 != null && bb.y1 != null) {\n if (bb.x2 != null && bb.y2 != null && bb.x2 >= bb.x1 && bb.y2 >= bb.y1) {\n return {\n x1: bb.x1,\n y1: bb.y1,\n x2: bb.x2,\n y2: bb.y2,\n w: bb.x2 - bb.x1,\n h: bb.y2 - bb.y1\n };\n } else if (bb.w != null && bb.h != null && bb.w >= 0 && bb.h >= 0) {\n return {\n x1: bb.x1,\n y1: bb.y1,\n x2: bb.x1 + bb.w,\n y2: bb.y1 + bb.h,\n w: bb.w,\n h: bb.h\n };\n }\n }\n};\nvar copyBoundingBox = function copyBoundingBox(bb) {\n return {\n x1: bb.x1,\n x2: bb.x2,\n w: bb.w,\n y1: bb.y1,\n y2: bb.y2,\n h: bb.h\n };\n};\nvar clearBoundingBox = function clearBoundingBox(bb) {\n bb.x1 = Infinity;\n bb.y1 = Infinity;\n bb.x2 = -Infinity;\n bb.y2 = -Infinity;\n bb.w = 0;\n bb.h = 0;\n};\nvar updateBoundingBox = function updateBoundingBox(bb1, bb2) {\n // update bb1 with bb2 bounds\n\n bb1.x1 = Math.min(bb1.x1, bb2.x1);\n bb1.x2 = Math.max(bb1.x2, bb2.x2);\n bb1.w = bb1.x2 - bb1.x1;\n bb1.y1 = Math.min(bb1.y1, bb2.y1);\n bb1.y2 = Math.max(bb1.y2, bb2.y2);\n bb1.h = bb1.y2 - bb1.y1;\n};\nvar expandBoundingBoxByPoint = function expandBoundingBoxByPoint(bb, x, y) {\n bb.x1 = Math.min(bb.x1, x);\n bb.x2 = Math.max(bb.x2, x);\n bb.w = bb.x2 - bb.x1;\n bb.y1 = Math.min(bb.y1, y);\n bb.y2 = Math.max(bb.y2, y);\n bb.h = bb.y2 - bb.y1;\n};\nvar expandBoundingBox = function expandBoundingBox(bb) {\n var padding = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : 0;\n bb.x1 -= padding;\n bb.x2 += padding;\n bb.y1 -= padding;\n bb.y2 += padding;\n bb.w = bb.x2 - bb.x1;\n bb.h = bb.y2 - bb.y1;\n return bb;\n};\nvar expandBoundingBoxSides = function expandBoundingBoxSides(bb) {\n var padding = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : [0];\n var top, right, bottom, left;\n if (padding.length === 1) {\n top = right = bottom = left = padding[0];\n } else if (padding.length === 2) {\n top = bottom = padding[0];\n left = right = padding[1];\n } else if (padding.length === 4) {\n var _padding = _slicedToArray(padding, 4);\n top = _padding[0];\n right = _padding[1];\n bottom = _padding[2];\n left = _padding[3];\n }\n bb.x1 -= left;\n bb.x2 += right;\n bb.y1 -= top;\n bb.y2 += bottom;\n bb.w = bb.x2 - bb.x1;\n bb.h = bb.y2 - bb.y1;\n return bb;\n};\n\n// assign the values of bb2 into bb1\nvar assignBoundingBox = function assignBoundingBox(bb1, bb2) {\n bb1.x1 = bb2.x1;\n bb1.y1 = bb2.y1;\n bb1.x2 = bb2.x2;\n bb1.y2 = bb2.y2;\n bb1.w = bb1.x2 - bb1.x1;\n bb1.h = bb1.y2 - bb1.y1;\n};\nvar boundingBoxesIntersect = function boundingBoxesIntersect(bb1, bb2) {\n // case: one bb to right of other\n if (bb1.x1 > bb2.x2) {\n return false;\n }\n if (bb2.x1 > bb1.x2) {\n return false;\n }\n\n // case: one bb to left of other\n if (bb1.x2 < bb2.x1) {\n return false;\n }\n if (bb2.x2 < bb1.x1) {\n return false;\n }\n\n // case: one bb above other\n if (bb1.y2 < bb2.y1) {\n return false;\n }\n if (bb2.y2 < bb1.y1) {\n return false;\n }\n\n // case: one bb below other\n if (bb1.y1 > bb2.y2) {\n return false;\n }\n if (bb2.y1 > bb1.y2) {\n return false;\n }\n\n // otherwise, must have some overlap\n return true;\n};\nvar inBoundingBox = function inBoundingBox(bb, x, y) {\n return bb.x1 <= x && x <= bb.x2 && bb.y1 <= y && y <= bb.y2;\n};\nvar pointInBoundingBox = function pointInBoundingBox(bb, pt) {\n return inBoundingBox(bb, pt.x, pt.y);\n};\nvar boundingBoxInBoundingBox = function boundingBoxInBoundingBox(bb1, bb2) {\n return inBoundingBox(bb1, bb2.x1, bb2.y1) && inBoundingBox(bb1, bb2.x2, bb2.y2);\n};\nvar hypot = (_Math$hypot = Math.hypot) !== null && _Math$hypot !== undefined ? _Math$hypot : function (x, y) {\n return Math.sqrt(x * x + y * y);\n};\nfunction inflatePolygon(polygon, d) {\n if (polygon.length < 3) {\n throw new Error('Need at least 3 vertices');\n }\n // Helpers\n var add = function add(a, b) {\n return {\n x: a.x + b.x,\n y: a.y + b.y\n };\n };\n var sub = function sub(a, b) {\n return {\n x: a.x - b.x,\n y: a.y - b.y\n };\n };\n var scale = function scale(v, s) {\n return {\n x: v.x * s,\n y: v.y * s\n };\n };\n var cross = function cross(u, v) {\n return u.x * v.y - u.y * v.x;\n };\n var normalize = function normalize(v) {\n var len = hypot(v.x, v.y);\n return len === 0 ? {\n x: 0,\n y: 0\n } : {\n x: v.x / len,\n y: v.y / len\n };\n };\n // Signed area (positive = CCW)\n var signedArea = function signedArea(pts) {\n var A = 0;\n for (var i = 0; i < pts.length; i++) {\n var p = pts[i],\n q = pts[(i + 1) % pts.length];\n A += p.x * q.y - q.x * p.y;\n }\n return A / 2;\n };\n // Line–line intersection (infinite lines)\n var intersectLines = function intersectLines(p1, p2, p3, p4) {\n var r = sub(p2, p1);\n var s = sub(p4, p3);\n var denom = cross(r, s);\n if (Math.abs(denom) < 1e-9) {\n // Parallel or nearly so — fallback to midpoint\n return add(p1, scale(r, 0.5));\n }\n var t = cross(sub(p3, p1), s) / denom;\n return add(p1, scale(r, t));\n };\n\n // Make a shallow copy and enforce CCW\n var pts = polygon.map(function (p) {\n return {\n x: p.x,\n y: p.y\n };\n });\n if (signedArea(pts) < 0) pts.reverse();\n var n = pts.length;\n // Compute outward normals for each edge\n var normals = [];\n for (var i = 0; i < n; i++) {\n var p = pts[i],\n q = pts[(i + 1) % n];\n var edge = sub(q, p);\n // For CCW polygon, inward normal = (-edge.y, edge.x)\n // so outward normal = (edge.y, -edge.x)\n var out = normalize({\n x: edge.y,\n y: -edge.x\n });\n normals.push(out);\n }\n\n // Build offset edges\n var offsetEdges = normals.map(function (nrm, i) {\n var p1 = add(pts[i], scale(nrm, d));\n var p2 = add(pts[(i + 1) % n], scale(nrm, d));\n return {\n p1: p1,\n p2: p2\n };\n });\n\n // Intersect consecutive offset edges\n var inflated = [];\n for (var _i2 = 0; _i2 < n; _i2++) {\n var prevEdge = offsetEdges[(_i2 - 1 + n) % n];\n var currEdge = offsetEdges[_i2];\n var ip = intersectLines(prevEdge.p1, prevEdge.p2, currEdge.p1, currEdge.p2);\n inflated.push(ip);\n }\n return inflated;\n}\nfunction miterBox(pts, centerX, centerY, width, height, strokeWidth) {\n var tpts = transformPoints(pts, centerX, centerY, width, height);\n var offsetPoints = inflatePolygon(tpts, strokeWidth);\n var bb = makeBoundingBox();\n offsetPoints.forEach(function (pt) {\n return expandBoundingBoxByPoint(bb, pt.x, pt.y);\n });\n return bb;\n}\nvar roundRectangleIntersectLine = function roundRectangleIntersectLine(x, y, nodeX, nodeY, width, height, padding) {\n var radius = arguments.length > 7 && arguments[7] !== undefined ? arguments[7] : 'auto';\n var cornerRadius = radius === 'auto' ? getRoundRectangleRadius(width, height) : radius;\n var halfWidth = width / 2;\n var halfHeight = height / 2;\n cornerRadius = Math.min(cornerRadius, halfWidth, halfHeight);\n var doWidth = cornerRadius !== halfWidth,\n doHeight = cornerRadius !== halfHeight;\n\n // Check intersections with straight line segments\n var straightLineIntersections;\n\n // Top segment, left to right\n if (doWidth) {\n var topStartX = nodeX - halfWidth + cornerRadius - padding;\n var topStartY = nodeY - halfHeight - padding;\n var topEndX = nodeX + halfWidth - cornerRadius + padding;\n var topEndY = topStartY;\n straightLineIntersections = finiteLinesIntersect(x, y, nodeX, nodeY, topStartX, topStartY, topEndX, topEndY, false);\n if (straightLineIntersections.length > 0) {\n return straightLineIntersections;\n }\n }\n\n // Right segment, top to bottom\n if (doHeight) {\n var rightStartX = nodeX + halfWidth + padding;\n var rightStartY = nodeY - halfHeight + cornerRadius - padding;\n var rightEndX = rightStartX;\n var rightEndY = nodeY + halfHeight - cornerRadius + padding;\n straightLineIntersections = finiteLinesIntersect(x, y, nodeX, nodeY, rightStartX, rightStartY, rightEndX, rightEndY, false);\n if (straightLineIntersections.length > 0) {\n return straightLineIntersections;\n }\n }\n\n // Bottom segment, left to right\n if (doWidth) {\n var bottomStartX = nodeX - halfWidth + cornerRadius - padding;\n var bottomStartY = nodeY + halfHeight + padding;\n var bottomEndX = nodeX + halfWidth - cornerRadius + padding;\n var bottomEndY = bottomStartY;\n straightLineIntersections = finiteLinesIntersect(x, y, nodeX, nodeY, bottomStartX, bottomStartY, bottomEndX, bottomEndY, false);\n if (straightLineIntersections.length > 0) {\n return straightLineIntersections;\n }\n }\n\n // Left segment, top to bottom\n if (doHeight) {\n var leftStartX = nodeX - halfWidth - padding;\n var leftStartY = nodeY - halfHeight + cornerRadius - padding;\n var leftEndX = leftStartX;\n var leftEndY = nodeY + halfHeight - cornerRadius + padding;\n straightLineIntersections = finiteLinesIntersect(x, y, nodeX, nodeY, leftStartX, leftStartY, leftEndX, leftEndY, false);\n if (straightLineIntersections.length > 0) {\n return straightLineIntersections;\n }\n }\n\n // Check intersections with arc segments\n var arcIntersections;\n\n // Top Left\n {\n var topLeftCenterX = nodeX - halfWidth + cornerRadius;\n var topLeftCenterY = nodeY - halfHeight + cornerRadius;\n arcIntersections = intersectLineCircle(x, y, nodeX, nodeY, topLeftCenterX, topLeftCenterY, cornerRadius + padding);\n\n // Ensure the intersection is on the desired quarter of the circle\n if (arcIntersections.length > 0 && arcIntersections[0] <= topLeftCenterX && arcIntersections[1] <= topLeftCenterY) {\n return [arcIntersections[0], arcIntersections[1]];\n }\n }\n\n // Top Right\n {\n var topRightCenterX = nodeX + halfWidth - cornerRadius;\n var topRightCenterY = nodeY - halfHeight + cornerRadius;\n arcIntersections = intersectLineCircle(x, y, nodeX, nodeY, topRightCenterX, topRightCenterY, cornerRadius + padding);\n\n // Ensure the intersection is on the desired quarter of the circle\n if (arcIntersections.length > 0 && arcIntersections[0] >= topRightCenterX && arcIntersections[1] <= topRightCenterY) {\n return [arcIntersections[0], arcIntersections[1]];\n }\n }\n\n // Bottom Right\n {\n var bottomRightCenterX = nodeX + halfWidth - cornerRadius;\n var bottomRightCenterY = nodeY + halfHeight - cornerRadius;\n arcIntersections = intersectLineCircle(x, y, nodeX, nodeY, bottomRightCenterX, bottomRightCenterY, cornerRadius + padding);\n\n // Ensure the intersection is on the desired quarter of the circle\n if (arcIntersections.length > 0 && arcIntersections[0] >= bottomRightCenterX && arcIntersections[1] >= bottomRightCenterY) {\n return [arcIntersections[0], arcIntersections[1]];\n }\n }\n\n // Bottom Left\n {\n var bottomLeftCenterX = nodeX - halfWidth + cornerRadius;\n var bottomLeftCenterY = nodeY + halfHeight - cornerRadius;\n arcIntersections = intersectLineCircle(x, y, nodeX, nodeY, bottomLeftCenterX, bottomLeftCenterY, cornerRadius + padding);\n\n // Ensure the intersection is on the desired quarter of the circle\n if (arcIntersections.length > 0 && arcIntersections[0] <= bottomLeftCenterX && arcIntersections[1] >= bottomLeftCenterY) {\n return [arcIntersections[0], arcIntersections[1]];\n }\n }\n return []; // if nothing\n};\nvar inLineVicinity = function inLineVicinity(x, y, lx1, ly1, lx2, ly2, tolerance) {\n var t = tolerance;\n var x1 = Math.min(lx1, lx2);\n var x2 = Math.max(lx1, lx2);\n var y1 = Math.min(ly1, ly2);\n var y2 = Math.max(ly1, ly2);\n return x1 - t <= x && x <= x2 + t && y1 - t <= y && y <= y2 + t;\n};\nvar inBezierVicinity = function inBezierVicinity(x, y, x1, y1, x2, y2, x3, y3, tolerance) {\n var bb = {\n x1: Math.min(x1, x3, x2) - tolerance,\n x2: Math.max(x1, x3, x2) + tolerance,\n y1: Math.min(y1, y3, y2) - tolerance,\n y2: Math.max(y1, y3, y2) + tolerance\n };\n\n // if outside the rough bounding box for the bezier, then it can't be a hit\n if (x < bb.x1 || x > bb.x2 || y < bb.y1 || y > bb.y2) {\n // console.log('bezier out of rough bb')\n return false;\n } else {\n // console.log('do more expensive check');\n return true;\n }\n};\nvar solveQuadratic = function solveQuadratic(a, b, c, val) {\n c -= val;\n var r = b * b - 4 * a * c;\n if (r < 0) {\n return [];\n }\n var sqrtR = Math.sqrt(r);\n var denom = 2 * a;\n var root1 = (-b + sqrtR) / denom;\n var root2 = (-b - sqrtR) / denom;\n return [root1, root2];\n};\nvar solveCubic = function solveCubic(a, b, c, d, result) {\n // Solves a cubic function, returns root in form [r1, i1, r2, i2, r3, i3], where\n // r is the real component, i is the imaginary component\n\n // An implementation of the Cardano method from the year 1545\n // http://en.wikipedia.org/wiki/Cubic_function#The_nature_of_the_roots\n\n var epsilon = 0.00001;\n\n // avoid division by zero while keeping the overall expression close in value\n if (a === 0) {\n a = epsilon;\n }\n b /= a;\n c /= a;\n d /= a;\n var discriminant, q, r, dum1, s, t, term1, r13;\n q = (3.0 * c - b * b) / 9.0;\n r = -(27.0 * d) + b * (9.0 * c - 2.0 * (b * b));\n r /= 54.0;\n discriminant = q * q * q + r * r;\n result[1] = 0;\n term1 = b / 3.0;\n if (discriminant > 0) {\n s = r + Math.sqrt(discriminant);\n s = s < 0 ? -Math.pow(-s, 1.0 / 3.0) : Math.pow(s, 1.0 / 3.0);\n t = r - Math.sqrt(discriminant);\n t = t < 0 ? -Math.pow(-t, 1.0 / 3.0) : Math.pow(t, 1.0 / 3.0);\n result[0] = -term1 + s + t;\n term1 += (s + t) / 2.0;\n result[4] = result[2] = -term1;\n term1 = Math.sqrt(3.0) * (-t + s) / 2;\n result[3] = term1;\n result[5] = -term1;\n return;\n }\n result[5] = result[3] = 0;\n if (discriminant === 0) {\n r13 = r < 0 ? -Math.pow(-r, 1.0 / 3.0) : Math.pow(r, 1.0 / 3.0);\n result[0] = -term1 + 2.0 * r13;\n result[4] = result[2] = -(r13 + term1);\n return;\n }\n q = -q;\n dum1 = q * q * q;\n dum1 = Math.acos(r / Math.sqrt(dum1));\n r13 = 2.0 * Math.sqrt(q);\n result[0] = -term1 + r13 * Math.cos(dum1 / 3.0);\n result[2] = -term1 + r13 * Math.cos((dum1 + 2.0 * Math.PI) / 3.0);\n result[4] = -term1 + r13 * Math.cos((dum1 + 4.0 * Math.PI) / 3.0);\n return;\n};\nvar sqdistToQuadraticBezier = function sqdistToQuadraticBezier(x, y, x1, y1, x2, y2, x3, y3) {\n // Find minimum distance by using the minimum of the distance\n // function between the given point and the curve\n\n // This gives the coefficients of the resulting cubic equation\n // whose roots tell us where a possible minimum is\n // (Coefficients are divided by 4)\n\n var a = 1.0 * x1 * x1 - 4 * x1 * x2 + 2 * x1 * x3 + 4 * x2 * x2 - 4 * x2 * x3 + x3 * x3 + y1 * y1 - 4 * y1 * y2 + 2 * y1 * y3 + 4 * y2 * y2 - 4 * y2 * y3 + y3 * y3;\n var b = 1.0 * 9 * x1 * x2 - 3 * x1 * x1 - 3 * x1 * x3 - 6 * x2 * x2 + 3 * x2 * x3 + 9 * y1 * y2 - 3 * y1 * y1 - 3 * y1 * y3 - 6 * y2 * y2 + 3 * y2 * y3;\n var c = 1.0 * 3 * x1 * x1 - 6 * x1 * x2 + x1 * x3 - x1 * x + 2 * x2 * x2 + 2 * x2 * x - x3 * x + 3 * y1 * y1 - 6 * y1 * y2 + y1 * y3 - y1 * y + 2 * y2 * y2 + 2 * y2 * y - y3 * y;\n var d = 1.0 * x1 * x2 - x1 * x1 + x1 * x - x2 * x + y1 * y2 - y1 * y1 + y1 * y - y2 * y;\n\n // debug(\"coefficients: \" + a / a + \", \" + b / a + \", \" + c / a + \", \" + d / a);\n\n var roots = [];\n\n // Use the cubic solving algorithm\n solveCubic(a, b, c, d, roots);\n var zeroThreshold = 0.0000001;\n var params = [];\n for (var index = 0; index < 6; index += 2) {\n if (Math.abs(roots[index + 1]) < zeroThreshold && roots[index] >= 0 && roots[index] <= 1.0) {\n params.push(roots[index]);\n }\n }\n params.push(1.0);\n params.push(0.0);\n var minDistanceSquared = -1;\n var curX, curY, distSquared;\n for (var i = 0; i < params.length; i++) {\n curX = Math.pow(1.0 - params[i], 2.0) * x1 + 2.0 * (1 - params[i]) * params[i] * x2 + params[i] * params[i] * x3;\n curY = Math.pow(1 - params[i], 2.0) * y1 + 2 * (1.0 - params[i]) * params[i] * y2 + params[i] * params[i] * y3;\n distSquared = Math.pow(curX - x, 2) + Math.pow(curY - y, 2);\n // debug('distance for param ' + params[i] + \": \" + Math.sqrt(distSquared));\n if (minDistanceSquared >= 0) {\n if (distSquared < minDistanceSquared) {\n minDistanceSquared = distSquared;\n }\n } else {\n minDistanceSquared = distSquared;\n }\n }\n return minDistanceSquared;\n};\nvar sqdistToFiniteLine = function sqdistToFiniteLine(x, y, x1, y1, x2, y2) {\n var offset = [x - x1, y - y1];\n var line = [x2 - x1, y2 - y1];\n var lineSq = line[0] * line[0] + line[1] * line[1];\n var hypSq = offset[0] * offset[0] + offset[1] * offset[1];\n var dotProduct = offset[0] * line[0] + offset[1] * line[1];\n var adjSq = dotProduct * dotProduct / lineSq;\n if (dotProduct < 0) {\n return hypSq;\n }\n if (adjSq > lineSq) {\n return (x - x2) * (x - x2) + (y - y2) * (y - y2);\n }\n return hypSq - adjSq;\n};\nvar pointInsidePolygonPoints = function pointInsidePolygonPoints(x, y, points) {\n var x1, y1, x2, y2;\n var y3;\n\n // Intersect with vertical line through (x, y)\n var up = 0;\n // let down = 0;\n for (var i = 0; i < points.length / 2; i++) {\n x1 = points[i * 2];\n y1 = points[i * 2 + 1];\n if (i + 1 < points.length / 2) {\n x2 = points[(i + 1) * 2];\n y2 = points[(i + 1) * 2 + 1];\n } else {\n x2 = points[(i + 1 - points.length / 2) * 2];\n y2 = points[(i + 1 - points.length / 2) * 2 + 1];\n }\n if (x1 == x && x2 == x) ; else if (x1 >= x && x >= x2 || x1 <= x && x <= x2) {\n y3 = (x - x1) / (x2 - x1) * (y2 - y1) + y1;\n if (y3 > y) {\n up++;\n }\n\n // if( y3 < y ){\n // down++;\n // }\n } else {\n continue;\n }\n }\n if (up % 2 === 0) {\n return false;\n } else {\n return true;\n }\n};\nvar pointInsidePolygon = function pointInsidePolygon(x, y, basePoints, centerX, centerY, width, height, direction, padding) {\n var transformedPoints = new Array(basePoints.length);\n\n // Gives negative angle\n var angle;\n if (direction[0] != null) {\n angle = Math.atan(direction[1] / direction[0]);\n if (direction[0] < 0) {\n angle = angle + Math.PI / 2;\n } else {\n angle = -angle - Math.PI / 2;\n }\n } else {\n angle = direction;\n }\n var cos = Math.cos(-angle);\n var sin = Math.sin(-angle);\n\n // console.log(\"base: \" + basePoints);\n for (var i = 0; i < transformedPoints.length / 2; i++) {\n transformedPoints[i * 2] = width / 2 * (basePoints[i * 2] * cos - basePoints[i * 2 + 1] * sin);\n transformedPoints[i * 2 + 1] = height / 2 * (basePoints[i * 2 + 1] * cos + basePoints[i * 2] * sin);\n transformedPoints[i * 2] += centerX;\n transformedPoints[i * 2 + 1] += centerY;\n }\n var points;\n if (padding > 0) {\n var expandedLineSet = expandPolygon(transformedPoints, -padding);\n points = joinLines(expandedLineSet);\n } else {\n points = transformedPoints;\n }\n return pointInsidePolygonPoints(x, y, points);\n};\nvar pointInsideRoundPolygon = function pointInsideRoundPolygon(x, y, basePoints, centerX, centerY, width, height, corners) {\n var cutPolygonPoints = new Array(basePoints.length * 2);\n for (var i = 0; i < corners.length; i++) {\n var corner = corners[i];\n cutPolygonPoints[i * 4 + 0] = corner.startX;\n cutPolygonPoints[i * 4 + 1] = corner.startY;\n cutPolygonPoints[i * 4 + 2] = corner.stopX;\n cutPolygonPoints[i * 4 + 3] = corner.stopY;\n var squaredDistance = Math.pow(corner.cx - x, 2) + Math.pow(corner.cy - y, 2);\n if (squaredDistance <= Math.pow(corner.radius, 2)) {\n return true;\n }\n }\n return pointInsidePolygonPoints(x, y, cutPolygonPoints);\n};\nvar joinLines = function joinLines(lineSet) {\n var vertices = new Array(lineSet.length / 2);\n var currentLineStartX, currentLineStartY, currentLineEndX, currentLineEndY;\n var nextLineStartX, nextLineStartY, nextLineEndX, nextLineEndY;\n for (var i = 0; i < lineSet.length / 4; i++) {\n currentLineStartX = lineSet[i * 4];\n currentLineStartY = lineSet[i * 4 + 1];\n currentLineEndX = lineSet[i * 4 + 2];\n currentLineEndY = lineSet[i * 4 + 3];\n if (i < lineSet.length / 4 - 1) {\n nextLineStartX = lineSet[(i + 1) * 4];\n nextLineStartY = lineSet[(i + 1) * 4 + 1];\n nextLineEndX = lineSet[(i + 1) * 4 + 2];\n nextLineEndY = lineSet[(i + 1) * 4 + 3];\n } else {\n nextLineStartX = lineSet[0];\n nextLineStartY = lineSet[1];\n nextLineEndX = lineSet[2];\n nextLineEndY = lineSet[3];\n }\n var intersection = finiteLinesIntersect(currentLineStartX, currentLineStartY, currentLineEndX, currentLineEndY, nextLineStartX, nextLineStartY, nextLineEndX, nextLineEndY, true);\n vertices[i * 2] = intersection[0];\n vertices[i * 2 + 1] = intersection[1];\n }\n return vertices;\n};\nvar expandPolygon = function expandPolygon(points, pad) {\n var expandedLineSet = new Array(points.length * 2);\n var currentPointX, currentPointY, nextPointX, nextPointY;\n for (var i = 0; i < points.length / 2; i++) {\n currentPointX = points[i * 2];\n currentPointY = points[i * 2 + 1];\n if (i < points.length / 2 - 1) {\n nextPointX = points[(i + 1) * 2];\n nextPointY = points[(i + 1) * 2 + 1];\n } else {\n nextPointX = points[0];\n nextPointY = points[1];\n }\n\n // Current line: [currentPointX, currentPointY] to [nextPointX, nextPointY]\n\n // Assume CCW polygon winding\n\n var offsetX = nextPointY - currentPointY;\n var offsetY = -(nextPointX - currentPointX);\n\n // Normalize\n var offsetLength = Math.sqrt(offsetX * offsetX + offsetY * offsetY);\n var normalizedOffsetX = offsetX / offsetLength;\n var normalizedOffsetY = offsetY / offsetLength;\n expandedLineSet[i * 4] = currentPointX + normalizedOffsetX * pad;\n expandedLineSet[i * 4 + 1] = currentPointY + normalizedOffsetY * pad;\n expandedLineSet[i * 4 + 2] = nextPointX + normalizedOffsetX * pad;\n expandedLineSet[i * 4 + 3] = nextPointY + normalizedOffsetY * pad;\n }\n return expandedLineSet;\n};\nvar intersectLineEllipse = function intersectLineEllipse(x, y, centerX, centerY, ellipseWradius, ellipseHradius) {\n var dispX = centerX - x;\n var dispY = centerY - y;\n dispX /= ellipseWradius;\n dispY /= ellipseHradius;\n var len = Math.sqrt(dispX * dispX + dispY * dispY);\n var newLength = len - 1;\n if (newLength < 0) {\n return [];\n }\n var lenProportion = newLength / len;\n return [(centerX - x) * lenProportion + x, (centerY - y) * lenProportion + y];\n};\nvar checkInEllipse = function checkInEllipse(x, y, width, height, centerX, centerY, padding) {\n x -= centerX;\n y -= centerY;\n x /= width / 2 + padding;\n y /= height / 2 + padding;\n return x * x + y * y <= 1;\n};\n\n// Returns intersections of increasing distance from line's start point\nvar intersectLineCircle = function intersectLineCircle(x1, y1, x2, y2, centerX, centerY, radius) {\n // Calculate d, direction vector of line\n var d = [x2 - x1, y2 - y1]; // Direction vector of line\n var f = [x1 - centerX, y1 - centerY];\n var a = d[0] * d[0] + d[1] * d[1];\n var b = 2 * (f[0] * d[0] + f[1] * d[1]);\n var c = f[0] * f[0] + f[1] * f[1] - radius * radius;\n var discriminant = b * b - 4 * a * c;\n if (discriminant < 0) {\n return [];\n }\n var t1 = (-b + Math.sqrt(discriminant)) / (2 * a);\n var t2 = (-b - Math.sqrt(discriminant)) / (2 * a);\n var tMin = Math.min(t1, t2);\n var tMax = Math.max(t1, t2);\n var inRangeParams = [];\n if (tMin >= 0 && tMin <= 1) {\n inRangeParams.push(tMin);\n }\n if (tMax >= 0 && tMax <= 1) {\n inRangeParams.push(tMax);\n }\n if (inRangeParams.length === 0) {\n return [];\n }\n var nearIntersectionX = inRangeParams[0] * d[0] + x1;\n var nearIntersectionY = inRangeParams[0] * d[1] + y1;\n if (inRangeParams.length > 1) {\n if (inRangeParams[0] == inRangeParams[1]) {\n return [nearIntersectionX, nearIntersectionY];\n } else {\n var farIntersectionX = inRangeParams[1] * d[0] + x1;\n var farIntersectionY = inRangeParams[1] * d[1] + y1;\n return [nearIntersectionX, nearIntersectionY, farIntersectionX, farIntersectionY];\n }\n } else {\n return [nearIntersectionX, nearIntersectionY];\n }\n};\nvar midOfThree = function midOfThree(a, b, c) {\n if (b <= a && a <= c || c <= a && a <= b) {\n return a;\n } else if (a <= b && b <= c || c <= b && b <= a) {\n return b;\n } else {\n return c;\n }\n};\n\n// (x1,y1)=>(x2,y2) intersect with (x3,y3)=>(x4,y4)\nvar finiteLinesIntersect = function finiteLinesIntersect(x1, y1, x2, y2, x3, y3, x4, y4, infiniteLines) {\n var dx13 = x1 - x3;\n var dx21 = x2 - x1;\n var dx43 = x4 - x3;\n var dy13 = y1 - y3;\n var dy21 = y2 - y1;\n var dy43 = y4 - y3;\n var ua_t = dx43 * dy13 - dy43 * dx13;\n var ub_t = dx21 * dy13 - dy21 * dx13;\n var u_b = dy43 * dx21 - dx43 * dy21;\n if (u_b !== 0) {\n var ua = ua_t / u_b;\n var ub = ub_t / u_b;\n var flptThreshold = 0.001;\n var _min = 0 - flptThreshold;\n var _max = 1 + flptThreshold;\n if (_min <= ua && ua <= _max && _min <= ub && ub <= _max) {\n return [x1 + ua * dx21, y1 + ua * dy21];\n } else {\n if (!infiniteLines) {\n return [];\n } else {\n return [x1 + ua * dx21, y1 + ua * dy21];\n }\n }\n } else {\n if (ua_t === 0 || ub_t === 0) {\n // Parallel, coincident lines. Check if overlap\n\n // Check endpoint of second line\n if (midOfThree(x1, x2, x4) === x4) {\n return [x4, y4];\n }\n\n // Check start point of second line\n if (midOfThree(x1, x2, x3) === x3) {\n return [x3, y3];\n }\n\n // Endpoint of first line\n if (midOfThree(x3, x4, x2) === x2) {\n return [x2, y2];\n }\n return [];\n } else {\n // Parallel, non-coincident\n return [];\n }\n }\n};\nvar transformPoints = function transformPoints(points, centerX, centerY, width, height) {\n var ret = [];\n var halfW = width / 2;\n var halfH = height / 2;\n var x = centerX;\n var y = centerY;\n ret.push({\n x: x + halfW * points[0],\n y: y + halfH * points[1]\n });\n for (var i = 1; i < points.length / 2; i++) {\n ret.push({\n x: x + halfW * points[i * 2],\n y: y + halfH * points[i * 2 + 1]\n });\n }\n return ret;\n};\n\n// math.polygonIntersectLine( x, y, basePoints, centerX, centerY, width, height, padding )\n// intersect a node polygon (pts transformed)\n//\n// math.polygonIntersectLine( x, y, basePoints, centerX, centerY )\n// intersect the points (no transform)\nvar polygonIntersectLine = function polygonIntersectLine(x, y, basePoints, centerX, centerY, width, height, padding) {\n var intersections = [];\n var intersection;\n var transformedPoints = new Array(basePoints.length);\n var doTransform = true;\n if (width == null) {\n doTransform = false;\n }\n var points;\n if (doTransform) {\n for (var i = 0; i < transformedPoints.length / 2; i++) {\n transformedPoints[i * 2] = basePoints[i * 2] * width + centerX;\n transformedPoints[i * 2 + 1] = basePoints[i * 2 + 1] * height + centerY;\n }\n if (padding > 0) {\n var expandedLineSet = expandPolygon(transformedPoints, -padding);\n points = joinLines(expandedLineSet);\n } else {\n points = transformedPoints;\n }\n } else {\n points = basePoints;\n }\n var currentX, currentY, nextX, nextY;\n for (var _i3 = 0; _i3 < points.length / 2; _i3++) {\n currentX = points[_i3 * 2];\n currentY = points[_i3 * 2 + 1];\n if (_i3 < points.length / 2 - 1) {\n nextX = points[(_i3 + 1) * 2];\n nextY = points[(_i3 + 1) * 2 + 1];\n } else {\n nextX = points[0];\n nextY = points[1];\n }\n intersection = finiteLinesIntersect(x, y, centerX, centerY, currentX, currentY, nextX, nextY);\n if (intersection.length !== 0) {\n intersections.push(intersection[0], intersection[1]);\n }\n }\n return intersections;\n};\nvar roundPolygonIntersectLine = function roundPolygonIntersectLine(x, y, basePoints, centerX, centerY, width, height, padding, corners) {\n var intersections = [];\n var intersection;\n var lines = new Array(basePoints.length * 2);\n corners.forEach(function (corner, i) {\n if (i === 0) {\n lines[lines.length - 2] = corner.startX;\n lines[lines.length - 1] = corner.startY;\n } else {\n lines[i * 4 - 2] = corner.startX;\n lines[i * 4 - 1] = corner.startY;\n }\n lines[i * 4] = corner.stopX;\n lines[i * 4 + 1] = corner.stopY;\n intersection = intersectLineCircle(x, y, centerX, centerY, corner.cx, corner.cy, corner.radius);\n if (intersection.length !== 0) {\n intersections.push(intersection[0], intersection[1]);\n }\n });\n for (var i = 0; i < lines.length / 4; i++) {\n intersection = finiteLinesIntersect(x, y, centerX, centerY, lines[i * 4], lines[i * 4 + 1], lines[i * 4 + 2], lines[i * 4 + 3], false);\n if (intersection.length !== 0) {\n intersections.push(intersection[0], intersection[1]);\n }\n }\n if (intersections.length > 2) {\n var lowestIntersection = [intersections[0], intersections[1]];\n var lowestSquaredDistance = Math.pow(lowestIntersection[0] - x, 2) + Math.pow(lowestIntersection[1] - y, 2);\n for (var _i4 = 1; _i4 < intersections.length / 2; _i4++) {\n var squaredDistance = Math.pow(intersections[_i4 * 2] - x, 2) + Math.pow(intersections[_i4 * 2 + 1] - y, 2);\n if (squaredDistance <= lowestSquaredDistance) {\n lowestIntersection[0] = intersections[_i4 * 2];\n lowestIntersection[1] = intersections[_i4 * 2 + 1];\n lowestSquaredDistance = squaredDistance;\n }\n }\n return lowestIntersection;\n }\n return intersections;\n};\nvar shortenIntersection = function shortenIntersection(intersection, offset, amount) {\n var disp = [intersection[0] - offset[0], intersection[1] - offset[1]];\n var length = Math.sqrt(disp[0] * disp[0] + disp[1] * disp[1]);\n var lenRatio = (length - amount) / length;\n if (lenRatio < 0) {\n lenRatio = 0.00001;\n }\n return [offset[0] + lenRatio * disp[0], offset[1] + lenRatio * disp[1]];\n};\nvar generateUnitNgonPointsFitToSquare = function generateUnitNgonPointsFitToSquare(sides, rotationRadians) {\n var points = generateUnitNgonPoints(sides, rotationRadians);\n points = fitPolygonToSquare(points);\n return points;\n};\nvar fitPolygonToSquare = function fitPolygonToSquare(points) {\n var x, y;\n var sides = points.length / 2;\n var minX = Infinity,\n minY = Infinity,\n maxX = -Infinity,\n maxY = -Infinity;\n for (var i = 0; i < sides; i++) {\n x = points[2 * i];\n y = points[2 * i + 1];\n minX = Math.min(minX, x);\n maxX = Math.max(maxX, x);\n minY = Math.min(minY, y);\n maxY = Math.max(maxY, y);\n }\n\n // stretch factors\n var sx = 2 / (maxX - minX);\n var sy = 2 / (maxY - minY);\n for (var _i5 = 0; _i5 < sides; _i5++) {\n x = points[2 * _i5] = points[2 * _i5] * sx;\n y = points[2 * _i5 + 1] = points[2 * _i5 + 1] * sy;\n minX = Math.min(minX, x);\n maxX = Math.max(maxX, x);\n minY = Math.min(minY, y);\n maxY = Math.max(maxY, y);\n }\n if (minY < -1) {\n for (var _i6 = 0; _i6 < sides; _i6++) {\n y = points[2 * _i6 + 1] = points[2 * _i6 + 1] + (-1 - minY);\n }\n }\n return points;\n};\nvar generateUnitNgonPoints = function generateUnitNgonPoints(sides, rotationRadians) {\n var increment = 1.0 / sides * 2 * Math.PI;\n var startAngle = sides % 2 === 0 ? Math.PI / 2.0 + increment / 2.0 : Math.PI / 2.0;\n startAngle += rotationRadians;\n var points = new Array(sides * 2);\n var currentAngle;\n for (var i = 0; i < sides; i++) {\n currentAngle = i * increment + startAngle;\n points[2 * i] = Math.cos(currentAngle); // x\n points[2 * i + 1] = Math.sin(-currentAngle); // y\n }\n return points;\n};\n\n// Set the default radius, unless half of width or height is smaller than default\nvar getRoundRectangleRadius = function getRoundRectangleRadius(width, height) {\n return Math.min(width / 4, height / 4, 8);\n};\n\n// Set the default radius\nvar getRoundPolygonRadius = function getRoundPolygonRadius(width, height) {\n return Math.min(width / 10, height / 10, 8);\n};\nvar getCutRectangleCornerLength = function getCutRectangleCornerLength() {\n return 8;\n};\nvar bezierPtsToQuadCoeff = function bezierPtsToQuadCoeff(p0, p1, p2) {\n return [p0 - 2 * p1 + p2, 2 * (p1 - p0), p0];\n};\n\n// get curve width, height, and control point position offsets as a percentage of node height / width\nvar getBarrelCurveConstants = function getBarrelCurveConstants(width, height) {\n return {\n heightOffset: Math.min(15, 0.05 * height),\n widthOffset: Math.min(100, 0.25 * width),\n ctrlPtOffsetPct: 0.05\n };\n};\n\n// Separating Axis Theorem (SAT) to determine if two polygons intersect. \n// The function takes two polygons as input and returns a boolean value indicating \n// whether the two polygons intersect.\nfunction satPolygonIntersection(poly1, poly2) {\n function getAxes(polygon) {\n var axes = [];\n for (var i = 0; i < polygon.length; i++) {\n var p1 = polygon[i];\n var p2 = polygon[(i + 1) % polygon.length];\n var edge = {\n x: p2.x - p1.x,\n y: p2.y - p1.y\n };\n var normal = {\n x: -edge.y,\n y: edge.x\n };\n var length = Math.sqrt(normal.x * normal.x + normal.y * normal.y);\n axes.push({\n x: normal.x / length,\n y: normal.y / length\n });\n }\n return axes;\n }\n function project(polygon, axis) {\n var min = Infinity;\n var max = -Infinity;\n var _iterator = _createForOfIteratorHelper(polygon),\n _step;\n try {\n for (_iterator.s(); !(_step = _iterator.n()).done;) {\n var point = _step.value;\n var projection = point.x * axis.x + point.y * axis.y;\n min = Math.min(min, projection);\n max = Math.max(max, projection);\n }\n } catch (err) {\n _iterator.e(err);\n } finally {\n _iterator.f();\n }\n return {\n min: min,\n max: max\n };\n }\n function overlaps(proj1, proj2) {\n return !(proj1.max < proj2.min || proj2.max < proj1.min);\n }\n var axes = [].concat(_toConsumableArray(getAxes(poly1)), _toConsumableArray(getAxes(poly2)));\n var _iterator2 = _createForOfIteratorHelper(axes),\n _step2;\n try {\n for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {\n var axis = _step2.value;\n var proj1 = project(poly1, axis);\n var proj2 = project(poly2, axis);\n if (!overlaps(proj1, proj2)) {\n return false; // No overlap, so the polygons do not intersect\n }\n }\n } catch (err) {\n _iterator2.e(err);\n } finally {\n _iterator2.f();\n }\n return true; // polygons intersect\n}\n\nvar pageRankDefaults = defaults$g({\n dampingFactor: 0.8,\n precision: 0.000001,\n iterations: 200,\n weight: function weight(edge) {\n return 1;\n }\n});\nvar elesfn$o = {\n pageRank: function pageRank(options) {\n var _pageRankDefaults = pageRankDefaults(options),\n dampingFactor = _pageRankDefaults.dampingFactor,\n precision = _pageRankDefaults.precision,\n iterations = _pageRankDefaults.iterations,\n weight = _pageRankDefaults.weight;\n var cy = this._private.cy;\n var _this$byGroup = this.byGroup(),\n nodes = _this$byGroup.nodes,\n edges = _this$byGroup.edges;\n var numNodes = nodes.length;\n var numNodesSqd = numNodes * numNodes;\n var numEdges = edges.length;\n\n // Construct transposed adjacency matrix\n // First lets have a zeroed matrix of the right size\n // We'll also keep track of the sum of each column\n var matrix = new Array(numNodesSqd);\n var columnSum = new Array(numNodes);\n var additionalProb = (1 - dampingFactor) / numNodes;\n\n // Create null matrix\n for (var i = 0; i < numNodes; i++) {\n for (var j = 0; j < numNodes; j++) {\n var n = i * numNodes + j;\n matrix[n] = 0;\n }\n columnSum[i] = 0;\n }\n\n // Now, process edges\n for (var _i = 0; _i < numEdges; _i++) {\n var edge = edges[_i];\n var srcId = edge.data('source');\n var tgtId = edge.data('target');\n\n // Don't include loops in the matrix\n if (srcId === tgtId) {\n continue;\n }\n var s = nodes.indexOfId(srcId);\n var t = nodes.indexOfId(tgtId);\n var w = weight(edge);\n var _n = t * numNodes + s;\n\n // Update matrix\n matrix[_n] += w;\n\n // Update column sum\n columnSum[s] += w;\n }\n\n // Add additional probability based on damping factor\n // Also, take into account columns that have sum = 0\n var p = 1.0 / numNodes + additionalProb; // Shorthand\n\n // Traverse matrix, column by column\n for (var _j = 0; _j < numNodes; _j++) {\n if (columnSum[_j] === 0) {\n // No 'links' out from node jth, assume equal probability for each possible node\n for (var _i2 = 0; _i2 < numNodes; _i2++) {\n var _n2 = _i2 * numNodes + _j;\n matrix[_n2] = p;\n }\n } else {\n // Node jth has outgoing link, compute normalized probabilities\n for (var _i3 = 0; _i3 < numNodes; _i3++) {\n var _n3 = _i3 * numNodes + _j;\n matrix[_n3] = matrix[_n3] / columnSum[_j] + additionalProb;\n }\n }\n }\n\n // Compute dominant eigenvector using power method\n var eigenvector = new Array(numNodes);\n var temp = new Array(numNodes);\n var previous;\n\n // Start with a vector of all 1's\n // Also, initialize a null vector which will be used as shorthand\n for (var _i4 = 0; _i4 < numNodes; _i4++) {\n eigenvector[_i4] = 1;\n }\n for (var iter = 0; iter < iterations; iter++) {\n // Temp array with all 0's\n for (var _i5 = 0; _i5 < numNodes; _i5++) {\n temp[_i5] = 0;\n }\n\n // Multiply matrix with previous result\n for (var _i6 = 0; _i6 < numNodes; _i6++) {\n for (var _j2 = 0; _j2 < numNodes; _j2++) {\n var _n4 = _i6 * numNodes + _j2;\n temp[_i6] += matrix[_n4] * eigenvector[_j2];\n }\n }\n inPlaceSumNormalize(temp);\n previous = eigenvector;\n eigenvector = temp;\n temp = previous;\n var diff = 0;\n // Compute difference (squared module) of both vectors\n for (var _i7 = 0; _i7 < numNodes; _i7++) {\n var delta = previous[_i7] - eigenvector[_i7];\n diff += delta * delta;\n }\n\n // If difference is less than the desired threshold, stop iterating\n if (diff < precision) {\n break;\n }\n }\n\n // Construct result\n var res = {\n rank: function rank(node) {\n node = cy.collection(node)[0];\n return eigenvector[nodes.indexOf(node)];\n }\n };\n return res;\n } // pageRank\n}; // elesfn\n\nvar defaults$f = defaults$g({\n root: null,\n weight: function weight(edge) {\n return 1;\n },\n directed: false,\n alpha: 0\n});\nvar elesfn$n = {\n degreeCentralityNormalized: function degreeCentralityNormalized(options) {\n options = defaults$f(options);\n var cy = this.cy();\n var nodes = this.nodes();\n var numNodes = nodes.length;\n if (!options.directed) {\n var degrees = {};\n var maxDegree = 0;\n for (var i = 0; i < numNodes; i++) {\n var node = nodes[i];\n\n // add current node to the current options object and call degreeCentrality\n options.root = node;\n var currDegree = this.degreeCentrality(options);\n if (maxDegree < currDegree.degree) {\n maxDegree = currDegree.degree;\n }\n degrees[node.id()] = currDegree.degree;\n }\n return {\n degree: function degree(node) {\n if (maxDegree === 0) {\n return 0;\n }\n if (string(node)) {\n // from is a selector string\n node = cy.filter(node);\n }\n return degrees[node.id()] / maxDegree;\n }\n };\n } else {\n var indegrees = {};\n var outdegrees = {};\n var maxIndegree = 0;\n var maxOutdegree = 0;\n for (var _i = 0; _i < numNodes; _i++) {\n var _node = nodes[_i];\n var id = _node.id();\n\n // add current node to the current options object and call degreeCentrality\n options.root = _node;\n var _currDegree = this.degreeCentrality(options);\n if (maxIndegree < _currDegree.indegree) maxIndegree = _currDegree.indegree;\n if (maxOutdegree < _currDegree.outdegree) maxOutdegree = _currDegree.outdegree;\n indegrees[id] = _currDegree.indegree;\n outdegrees[id] = _currDegree.outdegree;\n }\n return {\n indegree: function indegree(node) {\n if (maxIndegree == 0) {\n return 0;\n }\n if (string(node)) {\n // from is a selector string\n node = cy.filter(node);\n }\n return indegrees[node.id()] / maxIndegree;\n },\n outdegree: function outdegree(node) {\n if (maxOutdegree === 0) {\n return 0;\n }\n if (string(node)) {\n // from is a selector string\n node = cy.filter(node);\n }\n return outdegrees[node.id()] / maxOutdegree;\n }\n };\n }\n },\n // degreeCentralityNormalized\n\n // Implemented from the algorithm in Opsahl's paper\n // \"Node centrality in weighted networks: Generalizing degree and shortest paths\"\n // check the heading 2 \"Degree\"\n degreeCentrality: function degreeCentrality(options) {\n options = defaults$f(options);\n var cy = this.cy();\n var callingEles = this;\n var _options = options,\n root = _options.root,\n weight = _options.weight,\n directed = _options.directed,\n alpha = _options.alpha;\n root = cy.collection(root)[0];\n if (!directed) {\n var connEdges = root.connectedEdges().intersection(callingEles);\n var k = connEdges.length;\n var s = 0;\n\n // Now, sum edge weights\n for (var i = 0; i < connEdges.length; i++) {\n s += weight(connEdges[i]);\n }\n return {\n degree: Math.pow(k, 1 - alpha) * Math.pow(s, alpha)\n };\n } else {\n var edges = root.connectedEdges();\n var incoming = edges.filter(function (edge) {\n return edge.target().same(root) && callingEles.has(edge);\n });\n var outgoing = edges.filter(function (edge) {\n return edge.source().same(root) && callingEles.has(edge);\n });\n var k_in = incoming.length;\n var k_out = outgoing.length;\n var s_in = 0;\n var s_out = 0;\n\n // Now, sum incoming edge weights\n for (var _i2 = 0; _i2 < incoming.length; _i2++) {\n s_in += weight(incoming[_i2]);\n }\n\n // Now, sum outgoing edge weights\n for (var _i3 = 0; _i3 < outgoing.length; _i3++) {\n s_out += weight(outgoing[_i3]);\n }\n return {\n indegree: Math.pow(k_in, 1 - alpha) * Math.pow(s_in, alpha),\n outdegree: Math.pow(k_out, 1 - alpha) * Math.pow(s_out, alpha)\n };\n }\n } // degreeCentrality\n}; // elesfn\n\n// nice, short mathematical alias\nelesfn$n.dc = elesfn$n.degreeCentrality;\nelesfn$n.dcn = elesfn$n.degreeCentralityNormalised = elesfn$n.degreeCentralityNormalized;\n\nvar defaults$e = defaults$g({\n harmonic: true,\n weight: function weight() {\n return 1;\n },\n directed: false,\n root: null\n});\nvar elesfn$m = {\n closenessCentralityNormalized: function closenessCentralityNormalized(options) {\n var _defaults = defaults$e(options),\n harmonic = _defaults.harmonic,\n weight = _defaults.weight,\n directed = _defaults.directed;\n var cy = this.cy();\n var closenesses = {};\n var maxCloseness = 0;\n var nodes = this.nodes();\n var fw = this.floydWarshall({\n weight: weight,\n directed: directed\n });\n\n // Compute closeness for every node and find the maximum closeness\n for (var i = 0; i < nodes.length; i++) {\n var currCloseness = 0;\n var node_i = nodes[i];\n for (var j = 0; j < nodes.length; j++) {\n if (i !== j) {\n var d = fw.distance(node_i, nodes[j]);\n if (harmonic) {\n currCloseness += 1 / d;\n } else {\n currCloseness += d;\n }\n }\n }\n if (!harmonic) {\n currCloseness = 1 / currCloseness;\n }\n if (maxCloseness < currCloseness) {\n maxCloseness = currCloseness;\n }\n closenesses[node_i.id()] = currCloseness;\n }\n return {\n closeness: function closeness(node) {\n if (maxCloseness == 0) {\n return 0;\n }\n if (string(node)) {\n // from is a selector string\n node = cy.filter(node)[0].id();\n } else {\n // from is a node\n node = node.id();\n }\n return closenesses[node] / maxCloseness;\n }\n };\n },\n // Implemented from pseudocode from wikipedia\n closenessCentrality: function closenessCentrality(options) {\n var _defaults2 = defaults$e(options),\n root = _defaults2.root,\n weight = _defaults2.weight,\n directed = _defaults2.directed,\n harmonic = _defaults2.harmonic;\n root = this.filter(root)[0];\n\n // we need distance from this node to every other node\n var dijkstra = this.dijkstra({\n root: root,\n weight: weight,\n directed: directed\n });\n var totalDistance = 0;\n var nodes = this.nodes();\n for (var i = 0; i < nodes.length; i++) {\n var n = nodes[i];\n if (!n.same(root)) {\n var d = dijkstra.distanceTo(n);\n if (harmonic) {\n totalDistance += 1 / d;\n } else {\n totalDistance += d;\n }\n }\n }\n return harmonic ? totalDistance : 1 / totalDistance;\n } // closenessCentrality\n}; // elesfn\n\n// nice, short mathematical alias\nelesfn$m.cc = elesfn$m.closenessCentrality;\nelesfn$m.ccn = elesfn$m.closenessCentralityNormalised = elesfn$m.closenessCentralityNormalized;\n\nvar defaults$d = defaults$g({\n weight: null,\n directed: false\n});\nvar elesfn$l = {\n // Implemented from the algorithm in the paper \"On Variants of Shortest-Path Betweenness Centrality and their Generic Computation\" by Ulrik Brandes\n betweennessCentrality: function betweennessCentrality(options) {\n var _defaults = defaults$d(options),\n directed = _defaults.directed,\n weight = _defaults.weight;\n var weighted = weight != null;\n var cy = this.cy();\n\n // starting\n var V = this.nodes();\n var A = {};\n var _C = {};\n var max = 0;\n var C = {\n set: function set(key, val) {\n _C[key] = val;\n if (val > max) {\n max = val;\n }\n },\n get: function get(key) {\n return _C[key];\n }\n };\n\n // A contains the neighborhoods of every node\n for (var i = 0; i < V.length; i++) {\n var v = V[i];\n var vid = v.id();\n if (directed) {\n A[vid] = v.outgoers().nodes(); // get outgoers of every node\n } else {\n A[vid] = v.openNeighborhood().nodes(); // get neighbors of every node\n }\n C.set(vid, 0);\n }\n var _loop = function _loop() {\n var sid = V[s].id();\n var S = []; // stack\n var P = {};\n var g = {};\n var d = {};\n var Q = new Heap(function (a, b) {\n return d[a] - d[b];\n }); // queue\n\n // init dictionaries\n for (var _i = 0; _i < V.length; _i++) {\n var _vid = V[_i].id();\n P[_vid] = [];\n g[_vid] = 0;\n d[_vid] = Infinity;\n }\n g[sid] = 1; // sigma\n d[sid] = 0; // distance to s\n\n Q.push(sid);\n while (!Q.empty()) {\n var _v = Q.pop();\n S.push(_v);\n if (weighted) {\n for (var j = 0; j < A[_v].length; j++) {\n var w = A[_v][j];\n var vEle = cy.getElementById(_v);\n var edge = undefined;\n if (vEle.edgesTo(w).length > 0) {\n edge = vEle.edgesTo(w)[0];\n } else {\n edge = w.edgesTo(vEle)[0];\n }\n var edgeWeight = weight(edge);\n w = w.id();\n if (d[w] > d[_v] + edgeWeight) {\n d[w] = d[_v] + edgeWeight;\n if (Q.nodes.indexOf(w) < 0) {\n //if w is not in Q\n Q.push(w);\n } else {\n // update position if w is in Q\n Q.updateItem(w);\n }\n g[w] = 0;\n P[w] = [];\n }\n if (d[w] == d[_v] + edgeWeight) {\n g[w] = g[w] + g[_v];\n P[w].push(_v);\n }\n }\n } else {\n for (var _j = 0; _j < A[_v].length; _j++) {\n var _w = A[_v][_j].id();\n if (d[_w] == Infinity) {\n Q.push(_w);\n d[_w] = d[_v] + 1;\n }\n if (d[_w] == d[_v] + 1) {\n g[_w] = g[_w] + g[_v];\n P[_w].push(_v);\n }\n }\n }\n }\n var e = {};\n for (var _i2 = 0; _i2 < V.length; _i2++) {\n e[V[_i2].id()] = 0;\n }\n while (S.length > 0) {\n var _w2 = S.pop();\n for (var _j2 = 0; _j2 < P[_w2].length; _j2++) {\n var _v2 = P[_w2][_j2];\n e[_v2] = e[_v2] + g[_v2] / g[_w2] * (1 + e[_w2]);\n }\n if (_w2 != V[s].id()) {\n C.set(_w2, C.get(_w2) + e[_w2]);\n }\n }\n };\n for (var s = 0; s < V.length; s++) {\n _loop();\n }\n var ret = {\n betweenness: function betweenness(node) {\n var id = cy.collection(node).id();\n return C.get(id);\n },\n betweennessNormalized: function betweennessNormalized(node) {\n if (max == 0) {\n return 0;\n }\n var id = cy.collection(node).id();\n return C.get(id) / max;\n }\n };\n\n // alias\n ret.betweennessNormalised = ret.betweennessNormalized;\n return ret;\n } // betweennessCentrality\n}; // elesfn\n\n// nice, short mathematical alias\nelesfn$l.bc = elesfn$l.betweennessCentrality;\n\n// Implemented by Zoe Xi @zoexi for GSOC 2016\n// https://github.com/cytoscape/cytoscape.js-markov-cluster\n\n\n/* eslint-disable no-unused-vars */\nvar defaults$c = defaults$g({\n expandFactor: 2,\n // affects time of computation and cluster granularity to some extent: M * M\n inflateFactor: 2,\n // affects cluster granularity (the greater the value, the more clusters): M(i,j) / E(j)\n multFactor: 1,\n // optional self loops for each node. Use a neutral value to improve cluster computations.\n maxIterations: 20,\n // maximum number of iterations of the MCL algorithm in a single run\n attributes: [\n // attributes/features used to group nodes, ie. similarity values between nodes\n function (edge) {\n return 1;\n }]\n});\n/* eslint-enable */\n\nvar setOptions$3 = function setOptions(options) {\n return defaults$c(options);\n};\n/* eslint-enable */\n\nvar getSimilarity$1 = function getSimilarity(edge, attributes) {\n var total = 0;\n for (var i = 0; i < attributes.length; i++) {\n total += attributes[i](edge);\n }\n return total;\n};\nvar addLoops = function addLoops(M, n, val) {\n for (var i = 0; i < n; i++) {\n M[i * n + i] = val;\n }\n};\nvar normalize = function normalize(M, n) {\n var sum;\n for (var col = 0; col < n; col++) {\n sum = 0;\n for (var row = 0; row < n; row++) {\n sum += M[row * n + col];\n }\n for (var _row = 0; _row < n; _row++) {\n M[_row * n + col] = M[_row * n + col] / sum;\n }\n }\n};\n\n// TODO: blocked matrix multiplication?\nvar mmult = function mmult(A, B, n) {\n var C = new Array(n * n);\n for (var i = 0; i < n; i++) {\n for (var j = 0; j < n; j++) {\n C[i * n + j] = 0;\n }\n for (var k = 0; k < n; k++) {\n for (var _j = 0; _j < n; _j++) {\n C[i * n + _j] += A[i * n + k] * B[k * n + _j];\n }\n }\n }\n return C;\n};\nvar expand = function expand(M, n, expandFactor /** power **/) {\n var _M = M.slice(0);\n for (var p = 1; p < expandFactor; p++) {\n M = mmult(M, _M, n);\n }\n return M;\n};\nvar inflate = function inflate(M, n, inflateFactor /** r **/) {\n var _M = new Array(n * n);\n\n // M(i,j) ^ inflatePower\n for (var i = 0; i < n * n; i++) {\n _M[i] = Math.pow(M[i], inflateFactor);\n }\n normalize(_M, n);\n return _M;\n};\nvar hasConverged = function hasConverged(M, _M, n2, roundFactor) {\n // Check that both matrices have the same elements (i,j)\n for (var i = 0; i < n2; i++) {\n var v1 = Math.round(M[i] * Math.pow(10, roundFactor)) / Math.pow(10, roundFactor); // truncate to 'roundFactor' decimal places\n var v2 = Math.round(_M[i] * Math.pow(10, roundFactor)) / Math.pow(10, roundFactor);\n if (v1 !== v2) {\n return false;\n }\n }\n return true;\n};\nvar assign$2 = function assign(M, n, nodes, cy) {\n var clusters = [];\n for (var i = 0; i < n; i++) {\n var cluster = [];\n for (var j = 0; j < n; j++) {\n // Row-wise attractors and elements that they attract belong in same cluster\n if (Math.round(M[i * n + j] * 1000) / 1000 > 0) {\n cluster.push(nodes[j]);\n }\n }\n if (cluster.length !== 0) {\n clusters.push(cy.collection(cluster));\n }\n }\n return clusters;\n};\nvar isDuplicate = function isDuplicate(c1, c2) {\n for (var i = 0; i < c1.length; i++) {\n if (!c2[i] || c1[i].id() !== c2[i].id()) {\n return false;\n }\n }\n return true;\n};\nvar removeDuplicates = function removeDuplicates(clusters) {\n for (var i = 0; i < clusters.length; i++) {\n for (var j = 0; j < clusters.length; j++) {\n if (i != j && isDuplicate(clusters[i], clusters[j])) {\n clusters.splice(j, 1);\n }\n }\n }\n return clusters;\n};\nvar markovClustering = function markovClustering(options) {\n var nodes = this.nodes();\n var edges = this.edges();\n var cy = this.cy();\n\n // Set parameters of algorithm:\n var opts = setOptions$3(options);\n\n // Map each node to its position in node array\n var id2position = {};\n for (var i = 0; i < nodes.length; i++) {\n id2position[nodes[i].id()] = i;\n }\n\n // Generate stochastic matrix M from input graph G (should be symmetric/undirected)\n var n = nodes.length,\n n2 = n * n;\n var M = new Array(n2),\n _M;\n for (var _i = 0; _i < n2; _i++) {\n M[_i] = 0;\n }\n for (var e = 0; e < edges.length; e++) {\n var edge = edges[e];\n var _i2 = id2position[edge.source().id()];\n var j = id2position[edge.target().id()];\n var sim = getSimilarity$1(edge, opts.attributes);\n M[_i2 * n + j] += sim; // G should be symmetric and undirected\n M[j * n + _i2] += sim;\n }\n\n // Begin Markov cluster algorithm\n\n // Step 1: Add self loops to each node, ie. add multFactor to matrix diagonal\n addLoops(M, n, opts.multFactor);\n\n // Step 2: M = normalize( M );\n normalize(M, n);\n var isStillMoving = true;\n var iterations = 0;\n while (isStillMoving && iterations < opts.maxIterations) {\n isStillMoving = false;\n\n // Step 3:\n _M = expand(M, n, opts.expandFactor);\n\n // Step 4:\n M = inflate(_M, n, opts.inflateFactor);\n\n // Step 5: check to see if ~steady state has been reached\n if (!hasConverged(M, _M, n2, 4)) {\n isStillMoving = true;\n }\n iterations++;\n }\n\n // Build clusters from matrix\n var clusters = assign$2(M, n, nodes, cy);\n\n // Remove duplicate clusters due to symmetry of graph and M matrix\n clusters = removeDuplicates(clusters);\n return clusters;\n};\nvar markovClustering$1 = {\n markovClustering: markovClustering,\n mcl: markovClustering\n};\n\n// Common distance metrics for clustering algorithms\n// https://en.wikipedia.org/wiki/Hierarchical_clustering#Metric\n\nvar identity$1 = function identity(x) {\n return x;\n};\nvar absDiff = function absDiff(p, q) {\n return Math.abs(q - p);\n};\nvar addAbsDiff = function addAbsDiff(total, p, q) {\n return total + absDiff(p, q);\n};\nvar addSquaredDiff = function addSquaredDiff(total, p, q) {\n return total + Math.pow(q - p, 2);\n};\nvar sqrt = function sqrt(x) {\n return Math.sqrt(x);\n};\nvar maxAbsDiff = function maxAbsDiff(currentMax, p, q) {\n return Math.max(currentMax, absDiff(p, q));\n};\nvar getDistance = function getDistance(length, getP, getQ, init, visit) {\n var post = arguments.length > 5 && arguments[5] !== undefined ? arguments[5] : identity$1;\n var ret = init;\n var p, q;\n for (var dim = 0; dim < length; dim++) {\n p = getP(dim);\n q = getQ(dim);\n ret = visit(ret, p, q);\n }\n return post(ret);\n};\nvar distances = {\n euclidean: function euclidean(length, getP, getQ) {\n if (length >= 2) {\n return getDistance(length, getP, getQ, 0, addSquaredDiff, sqrt);\n } else {\n // for single attr case, more efficient to avoid sqrt\n return getDistance(length, getP, getQ, 0, addAbsDiff);\n }\n },\n squaredEuclidean: function squaredEuclidean(length, getP, getQ) {\n return getDistance(length, getP, getQ, 0, addSquaredDiff);\n },\n manhattan: function manhattan(length, getP, getQ) {\n return getDistance(length, getP, getQ, 0, addAbsDiff);\n },\n max: function max(length, getP, getQ) {\n return getDistance(length, getP, getQ, -Infinity, maxAbsDiff);\n }\n};\n\n// in case the user accidentally doesn't use camel case\ndistances['squared-euclidean'] = distances['squaredEuclidean'];\ndistances['squaredeuclidean'] = distances['squaredEuclidean'];\nfunction clusteringDistance (method, length, getP, getQ, nodeP, nodeQ) {\n var impl;\n if (fn$6(method)) {\n impl = method;\n } else {\n impl = distances[method] || distances.euclidean;\n }\n if (length === 0 && fn$6(method)) {\n return impl(nodeP, nodeQ);\n } else {\n return impl(length, getP, getQ, nodeP, nodeQ);\n }\n}\n\nvar defaults$b = defaults$g({\n k: 2,\n m: 2,\n sensitivityThreshold: 0.0001,\n distance: 'euclidean',\n maxIterations: 10,\n attributes: [],\n testMode: false,\n testCentroids: null\n});\nvar setOptions$2 = function setOptions(options) {\n return defaults$b(options);\n};\n\nvar getDist = function getDist(type, node, centroid, attributes, mode) {\n var noNodeP = mode !== 'kMedoids';\n var getP = noNodeP ? function (i) {\n return centroid[i];\n } : function (i) {\n return attributes[i](centroid);\n };\n var getQ = function getQ(i) {\n return attributes[i](node);\n };\n var nodeP = centroid;\n var nodeQ = node;\n return clusteringDistance(type, attributes.length, getP, getQ, nodeP, nodeQ);\n};\nvar randomCentroids = function randomCentroids(nodes, k, attributes) {\n var ndim = attributes.length;\n var min = new Array(ndim);\n var max = new Array(ndim);\n var centroids = new Array(k);\n var centroid = null;\n\n // Find min, max values for each attribute dimension\n for (var i = 0; i < ndim; i++) {\n min[i] = nodes.min(attributes[i]).value;\n max[i] = nodes.max(attributes[i]).value;\n }\n\n // Build k centroids, each represented as an n-dim feature vector\n for (var c = 0; c < k; c++) {\n centroid = [];\n for (var _i = 0; _i < ndim; _i++) {\n centroid[_i] = Math.random() * (max[_i] - min[_i]) + min[_i]; // random initial value\n }\n centroids[c] = centroid;\n }\n return centroids;\n};\nvar classify = function classify(node, centroids, distance, attributes, type) {\n var min = Infinity;\n var index = 0;\n for (var i = 0; i < centroids.length; i++) {\n var dist = getDist(distance, node, centroids[i], attributes, type);\n if (dist < min) {\n min = dist;\n index = i;\n }\n }\n return index;\n};\nvar buildCluster = function buildCluster(centroid, nodes, assignment) {\n var cluster = [];\n var node = null;\n for (var n = 0; n < nodes.length; n++) {\n node = nodes[n];\n if (assignment[node.id()] === centroid) {\n //console.log(\"Node \" + node.id() + \" is associated with medoid #: \" + m);\n cluster.push(node);\n }\n }\n return cluster;\n};\nvar haveValuesConverged = function haveValuesConverged(v1, v2, sensitivityThreshold) {\n return Math.abs(v2 - v1) <= sensitivityThreshold;\n};\nvar haveMatricesConverged = function haveMatricesConverged(v1, v2, sensitivityThreshold) {\n for (var i = 0; i < v1.length; i++) {\n for (var j = 0; j < v1[i].length; j++) {\n var diff = Math.abs(v1[i][j] - v2[i][j]);\n if (diff > sensitivityThreshold) {\n return false;\n }\n }\n }\n return true;\n};\nvar seenBefore = function seenBefore(node, medoids, n) {\n for (var i = 0; i < n; i++) {\n if (node === medoids[i]) return true;\n }\n return false;\n};\nvar randomMedoids = function randomMedoids(nodes, k) {\n var medoids = new Array(k);\n\n // For small data sets, the probability of medoid conflict is greater,\n // so we need to check to see if we've already seen or chose this node before.\n if (nodes.length < 50) {\n // Randomly select k medoids from the n nodes\n for (var i = 0; i < k; i++) {\n var node = nodes[Math.floor(Math.random() * nodes.length)];\n\n // If we've already chosen this node to be a medoid, don't choose it again (for small data sets).\n // Instead choose a different random node.\n while (seenBefore(node, medoids, i)) {\n node = nodes[Math.floor(Math.random() * nodes.length)];\n }\n medoids[i] = node;\n }\n } else {\n // Relatively large data set, so pretty safe to not check and just select random nodes\n for (var _i2 = 0; _i2 < k; _i2++) {\n medoids[_i2] = nodes[Math.floor(Math.random() * nodes.length)];\n }\n }\n return medoids;\n};\nvar findCost = function findCost(potentialNewMedoid, cluster, attributes) {\n var cost = 0;\n for (var n = 0; n < cluster.length; n++) {\n cost += getDist('manhattan', cluster[n], potentialNewMedoid, attributes, 'kMedoids');\n }\n return cost;\n};\nvar kMeans = function kMeans(options) {\n var cy = this.cy();\n var nodes = this.nodes();\n var node = null;\n\n // Set parameters of algorithm: # of clusters, distance metric, etc.\n var opts = setOptions$2(options);\n\n // Begin k-means algorithm\n var clusters = new Array(opts.k);\n var assignment = {};\n var centroids;\n\n // Step 1: Initialize centroid positions\n if (opts.testMode) {\n if (typeof opts.testCentroids === 'number') {\n // TODO: implement a seeded random number generator.\n opts.testCentroids;\n centroids = randomCentroids(nodes, opts.k, opts.attributes);\n } else if (_typeof(opts.testCentroids) === 'object') {\n centroids = opts.testCentroids;\n } else {\n centroids = randomCentroids(nodes, opts.k, opts.attributes);\n }\n } else {\n centroids = randomCentroids(nodes, opts.k, opts.attributes);\n }\n var isStillMoving = true;\n var iterations = 0;\n while (isStillMoving && iterations < opts.maxIterations) {\n // Step 2: Assign nodes to the nearest centroid\n for (var n = 0; n < nodes.length; n++) {\n node = nodes[n];\n // Determine which cluster this node belongs to: node id => cluster #\n assignment[node.id()] = classify(node, centroids, opts.distance, opts.attributes, 'kMeans');\n }\n\n // Step 3: For each of the k clusters, update its centroid\n isStillMoving = false;\n for (var c = 0; c < opts.k; c++) {\n // Get all nodes that belong to this cluster\n var cluster = buildCluster(c, nodes, assignment);\n if (cluster.length === 0) {\n // If cluster is empty, break out early & move to next cluster\n continue;\n }\n\n // Update centroids by calculating avg of all nodes within the cluster.\n var ndim = opts.attributes.length;\n var centroid = centroids[c]; // [ dim_1, dim_2, dim_3, ... , dim_n ]\n var newCentroid = new Array(ndim);\n var sum = new Array(ndim);\n for (var d = 0; d < ndim; d++) {\n sum[d] = 0.0;\n for (var i = 0; i < cluster.length; i++) {\n node = cluster[i];\n sum[d] += opts.attributes[d](node);\n }\n newCentroid[d] = sum[d] / cluster.length;\n\n // Check to see if algorithm has converged, i.e. when centroids no longer change\n if (!haveValuesConverged(newCentroid[d], centroid[d], opts.sensitivityThreshold)) {\n isStillMoving = true;\n }\n }\n centroids[c] = newCentroid;\n clusters[c] = cy.collection(cluster);\n }\n iterations++;\n }\n return clusters;\n};\nvar kMedoids = function kMedoids(options) {\n var cy = this.cy();\n var nodes = this.nodes();\n var node = null;\n var opts = setOptions$2(options);\n\n // Begin k-medoids algorithm\n var clusters = new Array(opts.k);\n var medoids;\n var assignment = {};\n var curCost;\n var minCosts = new Array(opts.k); // minimum cost configuration for each cluster\n\n // Step 1: Initialize k medoids\n if (opts.testMode) {\n if (typeof opts.testCentroids === 'number') ; else if (_typeof(opts.testCentroids) === 'object') {\n medoids = opts.testCentroids;\n } else {\n medoids = randomMedoids(nodes, opts.k);\n }\n } else {\n medoids = randomMedoids(nodes, opts.k);\n }\n var isStillMoving = true;\n var iterations = 0;\n while (isStillMoving && iterations < opts.maxIterations) {\n // Step 2: Assign nodes to the nearest medoid\n for (var n = 0; n < nodes.length; n++) {\n node = nodes[n];\n // Determine which cluster this node belongs to: node id => cluster #\n assignment[node.id()] = classify(node, medoids, opts.distance, opts.attributes, 'kMedoids');\n }\n isStillMoving = false;\n // Step 3: For each medoid m, and for each node associated with mediod m,\n // select the node with the lowest configuration cost as new medoid.\n for (var m = 0; m < medoids.length; m++) {\n // Get all nodes that belong to this medoid\n var cluster = buildCluster(m, nodes, assignment);\n if (cluster.length === 0) {\n // If cluster is empty, break out early & move to next cluster\n continue;\n }\n minCosts[m] = findCost(medoids[m], cluster, opts.attributes); // original cost\n\n // Select different medoid if its configuration has the lowest cost\n for (var _n = 0; _n < cluster.length; _n++) {\n curCost = findCost(cluster[_n], cluster, opts.attributes);\n if (curCost < minCosts[m]) {\n minCosts[m] = curCost;\n medoids[m] = cluster[_n];\n isStillMoving = true;\n }\n }\n clusters[m] = cy.collection(cluster);\n }\n iterations++;\n }\n return clusters;\n};\nvar updateCentroids = function updateCentroids(centroids, nodes, U, weight, opts) {\n var numerator, denominator;\n for (var n = 0; n < nodes.length; n++) {\n for (var c = 0; c < centroids.length; c++) {\n weight[n][c] = Math.pow(U[n][c], opts.m);\n }\n }\n for (var _c = 0; _c < centroids.length; _c++) {\n for (var dim = 0; dim < opts.attributes.length; dim++) {\n numerator = 0;\n denominator = 0;\n for (var _n2 = 0; _n2 < nodes.length; _n2++) {\n numerator += weight[_n2][_c] * opts.attributes[dim](nodes[_n2]);\n denominator += weight[_n2][_c];\n }\n centroids[_c][dim] = numerator / denominator;\n }\n }\n};\nvar updateMembership = function updateMembership(U, _U, centroids, nodes, opts) {\n // Save previous step\n for (var i = 0; i < U.length; i++) {\n _U[i] = U[i].slice();\n }\n var sum, numerator, denominator;\n var pow = 2 / (opts.m - 1);\n for (var c = 0; c < centroids.length; c++) {\n for (var n = 0; n < nodes.length; n++) {\n sum = 0;\n for (var k = 0; k < centroids.length; k++) {\n // against all other centroids\n numerator = getDist(opts.distance, nodes[n], centroids[c], opts.attributes, 'cmeans');\n denominator = getDist(opts.distance, nodes[n], centroids[k], opts.attributes, 'cmeans');\n sum += Math.pow(numerator / denominator, pow);\n }\n U[n][c] = 1 / sum;\n }\n }\n};\nvar assign$1 = function assign(nodes, U, opts, cy) {\n var clusters = new Array(opts.k);\n for (var c = 0; c < clusters.length; c++) {\n clusters[c] = [];\n }\n var max;\n var index;\n for (var n = 0; n < U.length; n++) {\n // for each node (U is N x C matrix)\n max = -Infinity;\n index = -1;\n // Determine which cluster the node is most likely to belong in\n for (var _c2 = 0; _c2 < U[0].length; _c2++) {\n if (U[n][_c2] > max) {\n max = U[n][_c2];\n index = _c2;\n }\n }\n clusters[index].push(nodes[n]);\n }\n\n // Turn every array into a collection of nodes\n for (var _c3 = 0; _c3 < clusters.length; _c3++) {\n clusters[_c3] = cy.collection(clusters[_c3]);\n }\n return clusters;\n};\nvar fuzzyCMeans = function fuzzyCMeans(options) {\n var cy = this.cy();\n var nodes = this.nodes();\n var opts = setOptions$2(options);\n\n // Begin fuzzy c-means algorithm\n var clusters;\n var centroids;\n var U;\n var _U;\n var weight;\n\n // Step 1: Initialize letiables.\n _U = new Array(nodes.length);\n for (var i = 0; i < nodes.length; i++) {\n // N x C matrix\n _U[i] = new Array(opts.k);\n }\n U = new Array(nodes.length);\n for (var _i3 = 0; _i3 < nodes.length; _i3++) {\n // N x C matrix\n U[_i3] = new Array(opts.k);\n }\n for (var _i4 = 0; _i4 < nodes.length; _i4++) {\n var total = 0;\n for (var j = 0; j < opts.k; j++) {\n U[_i4][j] = Math.random();\n total += U[_i4][j];\n }\n for (var _j = 0; _j < opts.k; _j++) {\n U[_i4][_j] = U[_i4][_j] / total;\n }\n }\n centroids = new Array(opts.k);\n for (var _i5 = 0; _i5 < opts.k; _i5++) {\n centroids[_i5] = new Array(opts.attributes.length);\n }\n weight = new Array(nodes.length);\n for (var _i6 = 0; _i6 < nodes.length; _i6++) {\n // N x C matrix\n weight[_i6] = new Array(opts.k);\n }\n // end init FCM\n\n var isStillMoving = true;\n var iterations = 0;\n while (isStillMoving && iterations < opts.maxIterations) {\n isStillMoving = false;\n\n // Step 2: Calculate the centroids for each step.\n updateCentroids(centroids, nodes, U, weight, opts);\n\n // Step 3: Update the partition matrix U.\n updateMembership(U, _U, centroids, nodes, opts);\n\n // Step 4: Check for convergence.\n if (!haveMatricesConverged(U, _U, opts.sensitivityThreshold)) {\n isStillMoving = true;\n }\n iterations++;\n }\n\n // Assign nodes to clusters with highest probability.\n clusters = assign$1(nodes, U, opts, cy);\n return {\n clusters: clusters,\n degreeOfMembership: U\n };\n};\nvar kClustering = {\n kMeans: kMeans,\n kMedoids: kMedoids,\n fuzzyCMeans: fuzzyCMeans,\n fcm: fuzzyCMeans\n};\n\n// Implemented by Zoe Xi @zoexi for GSOC 2016\n// https://github.com/cytoscape/cytoscape.js-hierarchical\n\nvar defaults$a = defaults$g({\n distance: 'euclidean',\n // distance metric to compare nodes\n linkage: 'min',\n // linkage criterion : how to determine the distance between clusters of nodes\n mode: 'threshold',\n // mode:'threshold' => clusters must be threshold distance apart\n threshold: Infinity,\n // the distance threshold\n // mode:'dendrogram' => the nodes are organised as leaves in a tree (siblings are close), merging makes clusters\n addDendrogram: false,\n // whether to add the dendrogram to the graph for viz\n dendrogramDepth: 0,\n // depth at which dendrogram branches are merged into the returned clusters\n attributes: [] // array of attr functions\n});\nvar linkageAliases = {\n 'single': 'min',\n 'complete': 'max'\n};\nvar setOptions$1 = function setOptions(options) {\n var opts = defaults$a(options);\n var preferredAlias = linkageAliases[opts.linkage];\n if (preferredAlias != null) {\n opts.linkage = preferredAlias;\n }\n return opts;\n};\nvar mergeClosest = function mergeClosest(clusters, index, dists, mins, opts) {\n // Find two closest clusters from cached mins\n var minKey = 0;\n var min = Infinity;\n var dist;\n var attrs = opts.attributes;\n var getDist = function getDist(n1, n2) {\n return clusteringDistance(opts.distance, attrs.length, function (i) {\n return attrs[i](n1);\n }, function (i) {\n return attrs[i](n2);\n }, n1, n2);\n };\n for (var i = 0; i < clusters.length; i++) {\n var key = clusters[i].key;\n var _dist = dists[key][mins[key]];\n if (_dist < min) {\n minKey = key;\n min = _dist;\n }\n }\n if (opts.mode === 'threshold' && min >= opts.threshold || opts.mode === 'dendrogram' && clusters.length === 1) {\n return false;\n }\n var c1 = index[minKey];\n var c2 = index[mins[minKey]];\n var merged;\n\n // Merge two closest clusters\n if (opts.mode === 'dendrogram') {\n merged = {\n left: c1,\n right: c2,\n key: c1.key\n };\n } else {\n merged = {\n value: c1.value.concat(c2.value),\n key: c1.key\n };\n }\n clusters[c1.index] = merged;\n clusters.splice(c2.index, 1);\n index[c1.key] = merged;\n\n // Update distances with new merged cluster\n for (var _i = 0; _i < clusters.length; _i++) {\n var cur = clusters[_i];\n if (c1.key === cur.key) {\n dist = Infinity;\n } else if (opts.linkage === 'min') {\n dist = dists[c1.key][cur.key];\n if (dists[c1.key][cur.key] > dists[c2.key][cur.key]) {\n dist = dists[c2.key][cur.key];\n }\n } else if (opts.linkage === 'max') {\n dist = dists[c1.key][cur.key];\n if (dists[c1.key][cur.key] < dists[c2.key][cur.key]) {\n dist = dists[c2.key][cur.key];\n }\n } else if (opts.linkage === 'mean') {\n dist = (dists[c1.key][cur.key] * c1.size + dists[c2.key][cur.key] * c2.size) / (c1.size + c2.size);\n } else {\n if (opts.mode === 'dendrogram') dist = getDist(cur.value, c1.value);else dist = getDist(cur.value[0], c1.value[0]);\n }\n dists[c1.key][cur.key] = dists[cur.key][c1.key] = dist; // distance matrix is symmetric\n }\n\n // Update cached mins\n for (var _i2 = 0; _i2 < clusters.length; _i2++) {\n var key1 = clusters[_i2].key;\n if (mins[key1] === c1.key || mins[key1] === c2.key) {\n var _min = key1;\n for (var j = 0; j < clusters.length; j++) {\n var key2 = clusters[j].key;\n if (dists[key1][key2] < dists[key1][_min]) {\n _min = key2;\n }\n }\n mins[key1] = _min;\n }\n clusters[_i2].index = _i2;\n }\n\n // Clean up meta data used for clustering\n c1.key = c2.key = c1.index = c2.index = null;\n return true;\n};\nvar _getAllChildren = function getAllChildren(root, arr, cy) {\n if (!root) return;\n if (root.value) {\n arr.push(root.value);\n } else {\n if (root.left) _getAllChildren(root.left, arr);\n if (root.right) _getAllChildren(root.right, arr);\n }\n};\nvar _buildDendrogram = function buildDendrogram(root, cy) {\n if (!root) return '';\n if (root.left && root.right) {\n var leftStr = _buildDendrogram(root.left, cy);\n var rightStr = _buildDendrogram(root.right, cy);\n var node = cy.add({\n group: 'nodes',\n data: {\n id: leftStr + ',' + rightStr\n }\n });\n cy.add({\n group: 'edges',\n data: {\n source: leftStr,\n target: node.id()\n }\n });\n cy.add({\n group: 'edges',\n data: {\n source: rightStr,\n target: node.id()\n }\n });\n return node.id();\n } else if (root.value) {\n return root.value.id();\n }\n};\nvar _buildClustersFromTree = function buildClustersFromTree(root, k, cy) {\n if (!root) return [];\n var left = [],\n right = [],\n leaves = [];\n if (k === 0) {\n // don't cut tree, simply return all nodes as 1 single cluster\n if (root.left) _getAllChildren(root.left, left);\n if (root.right) _getAllChildren(root.right, right);\n leaves = left.concat(right);\n return [cy.collection(leaves)];\n } else if (k === 1) {\n // cut at root\n\n if (root.value) {\n // leaf node\n return [cy.collection(root.value)];\n } else {\n if (root.left) _getAllChildren(root.left, left);\n if (root.right) _getAllChildren(root.right, right);\n return [cy.collection(left), cy.collection(right)];\n }\n } else {\n if (root.value) {\n return [cy.collection(root.value)];\n } else {\n if (root.left) left = _buildClustersFromTree(root.left, k - 1, cy);\n if (root.right) right = _buildClustersFromTree(root.right, k - 1, cy);\n return left.concat(right);\n }\n }\n};\n\nvar hierarchicalClustering = function hierarchicalClustering(options) {\n var cy = this.cy();\n var nodes = this.nodes();\n\n // Set parameters of algorithm: linkage type, distance metric, etc.\n var opts = setOptions$1(options);\n var attrs = opts.attributes;\n var getDist = function getDist(n1, n2) {\n return clusteringDistance(opts.distance, attrs.length, function (i) {\n return attrs[i](n1);\n }, function (i) {\n return attrs[i](n2);\n }, n1, n2);\n };\n\n // Begin hierarchical algorithm\n var clusters = [];\n var dists = []; // distances between each pair of clusters\n var mins = []; // closest cluster for each cluster\n var index = []; // hash of all clusters by key\n\n // In agglomerative (bottom-up) clustering, each node starts as its own cluster\n for (var n = 0; n < nodes.length; n++) {\n var cluster = {\n value: opts.mode === 'dendrogram' ? nodes[n] : [nodes[n]],\n key: n,\n index: n\n };\n clusters[n] = cluster;\n index[n] = cluster;\n dists[n] = [];\n mins[n] = 0;\n }\n\n // Calculate the distance between each pair of clusters\n for (var i = 0; i < clusters.length; i++) {\n for (var j = 0; j <= i; j++) {\n var dist = undefined;\n if (opts.mode === 'dendrogram') {\n // modes store cluster values differently\n dist = i === j ? Infinity : getDist(clusters[i].value, clusters[j].value);\n } else {\n dist = i === j ? Infinity : getDist(clusters[i].value[0], clusters[j].value[0]);\n }\n dists[i][j] = dist;\n dists[j][i] = dist;\n if (dist < dists[i][mins[i]]) {\n mins[i] = j; // Cache mins: closest cluster to cluster i is cluster j\n }\n }\n }\n\n // Find the closest pair of clusters and merge them into a single cluster.\n // Update distances between new cluster and each of the old clusters, and loop until threshold reached.\n var merged = mergeClosest(clusters, index, dists, mins, opts);\n while (merged) {\n merged = mergeClosest(clusters, index, dists, mins, opts);\n }\n var retClusters;\n\n // Dendrogram mode builds the hierarchy and adds intermediary nodes + edges\n // in addition to returning the clusters.\n if (opts.mode === 'dendrogram') {\n retClusters = _buildClustersFromTree(clusters[0], opts.dendrogramDepth, cy);\n if (opts.addDendrogram) _buildDendrogram(clusters[0], cy);\n } else {\n // Regular mode simply returns the clusters\n\n retClusters = new Array(clusters.length);\n clusters.forEach(function (cluster, i) {\n // Clean up meta data used for clustering\n cluster.key = cluster.index = null;\n retClusters[i] = cy.collection(cluster.value);\n });\n }\n return retClusters;\n};\nvar hierarchicalClustering$1 = {\n hierarchicalClustering: hierarchicalClustering,\n hca: hierarchicalClustering\n};\n\n// Implemented by Zoe Xi @zoexi for GSOC 2016\n// https://github.com/cytoscape/cytoscape.js-affinity-propagation\n\nvar defaults$9 = defaults$g({\n distance: 'euclidean',\n // distance metric to compare attributes between two nodes\n preference: 'median',\n // suitability of a data point to serve as an exemplar\n damping: 0.8,\n // damping factor between [0.5, 1)\n maxIterations: 1000,\n // max number of iterations to run\n minIterations: 100,\n // min number of iterations to run in order for clustering to stop\n attributes: [// functions to quantify the similarity between any two points\n // e.g. node => node.data('weight')\n ]\n});\nvar setOptions = function setOptions(options) {\n var dmp = options.damping;\n var pref = options.preference;\n if (!(0.5 <= dmp && dmp < 1)) {\n error(\"Damping must range on [0.5, 1). Got: \".concat(dmp));\n }\n var validPrefs = ['median', 'mean', 'min', 'max'];\n if (!(validPrefs.some(function (v) {\n return v === pref;\n }) || number$1(pref))) {\n error(\"Preference must be one of [\".concat(validPrefs.map(function (p) {\n return \"'\".concat(p, \"'\");\n }).join(', '), \"] or a number. Got: \").concat(pref));\n }\n return defaults$9(options);\n};\n\nvar getSimilarity = function getSimilarity(type, n1, n2, attributes) {\n var attr = function attr(n, i) {\n return attributes[i](n);\n };\n\n // nb negative because similarity should have an inverse relationship to distance\n return -clusteringDistance(type, attributes.length, function (i) {\n return attr(n1, i);\n }, function (i) {\n return attr(n2, i);\n }, n1, n2);\n};\nvar getPreference = function getPreference(S, preference) {\n // larger preference = greater # of clusters\n var p = null;\n if (preference === 'median') {\n p = median(S);\n } else if (preference === 'mean') {\n p = mean(S);\n } else if (preference === 'min') {\n p = min(S);\n } else if (preference === 'max') {\n p = max(S);\n } else {\n // Custom preference number, as set by user\n p = preference;\n }\n return p;\n};\nvar findExemplars = function findExemplars(n, R, A) {\n var indices = [];\n for (var i = 0; i < n; i++) {\n if (R[i * n + i] + A[i * n + i] > 0) {\n indices.push(i);\n }\n }\n return indices;\n};\nvar assignClusters = function assignClusters(n, S, exemplars) {\n var clusters = [];\n for (var i = 0; i < n; i++) {\n var index = -1;\n var max = -Infinity;\n for (var ei = 0; ei < exemplars.length; ei++) {\n var e = exemplars[ei];\n if (S[i * n + e] > max) {\n index = e;\n max = S[i * n + e];\n }\n }\n if (index > 0) {\n clusters.push(index);\n }\n }\n for (var _ei = 0; _ei < exemplars.length; _ei++) {\n clusters[exemplars[_ei]] = exemplars[_ei];\n }\n return clusters;\n};\nvar assign = function assign(n, S, exemplars) {\n var clusters = assignClusters(n, S, exemplars);\n for (var ei = 0; ei < exemplars.length; ei++) {\n var ii = [];\n for (var c = 0; c < clusters.length; c++) {\n if (clusters[c] === exemplars[ei]) {\n ii.push(c);\n }\n }\n var maxI = -1;\n var maxSum = -Infinity;\n for (var i = 0; i < ii.length; i++) {\n var sum = 0;\n for (var j = 0; j < ii.length; j++) {\n sum += S[ii[j] * n + ii[i]];\n }\n if (sum > maxSum) {\n maxI = i;\n maxSum = sum;\n }\n }\n exemplars[ei] = ii[maxI];\n }\n clusters = assignClusters(n, S, exemplars);\n return clusters;\n};\nvar affinityPropagation = function affinityPropagation(options) {\n var cy = this.cy();\n var nodes = this.nodes();\n var opts = setOptions(options);\n\n // Map each node to its position in node array\n var id2position = {};\n for (var i = 0; i < nodes.length; i++) {\n id2position[nodes[i].id()] = i;\n }\n\n // Begin affinity propagation algorithm\n\n var n; // number of data points\n var n2; // size of matrices\n var S; // similarity matrix (1D array)\n var p; // preference/suitability of a data point to serve as an exemplar\n var R; // responsibility matrix (1D array)\n var A; // availability matrix (1D array)\n\n n = nodes.length;\n n2 = n * n;\n\n // Initialize and build S similarity matrix\n S = new Array(n2);\n for (var _i = 0; _i < n2; _i++) {\n S[_i] = -Infinity; // for cases where two data points shouldn't be linked together\n }\n for (var _i2 = 0; _i2 < n; _i2++) {\n for (var j = 0; j < n; j++) {\n if (_i2 !== j) {\n S[_i2 * n + j] = getSimilarity(opts.distance, nodes[_i2], nodes[j], opts.attributes);\n }\n }\n }\n\n // Place preferences on the diagonal of S\n p = getPreference(S, opts.preference);\n for (var _i3 = 0; _i3 < n; _i3++) {\n S[_i3 * n + _i3] = p;\n }\n\n // Initialize R responsibility matrix\n R = new Array(n2);\n for (var _i4 = 0; _i4 < n2; _i4++) {\n R[_i4] = 0.0;\n }\n\n // Initialize A availability matrix\n A = new Array(n2);\n for (var _i5 = 0; _i5 < n2; _i5++) {\n A[_i5] = 0.0;\n }\n var old = new Array(n);\n var Rp = new Array(n);\n var se = new Array(n);\n for (var _i6 = 0; _i6 < n; _i6++) {\n old[_i6] = 0.0;\n Rp[_i6] = 0.0;\n se[_i6] = 0;\n }\n var e = new Array(n * opts.minIterations);\n for (var _i7 = 0; _i7 < e.length; _i7++) {\n e[_i7] = 0;\n }\n var iter;\n for (iter = 0; iter < opts.maxIterations; iter++) {\n // main algorithmic loop\n\n // Update R responsibility matrix\n for (var _i8 = 0; _i8 < n; _i8++) {\n var max = -Infinity,\n max2 = -Infinity,\n maxI = -1,\n AS = 0.0;\n for (var _j = 0; _j < n; _j++) {\n old[_j] = R[_i8 * n + _j];\n AS = A[_i8 * n + _j] + S[_i8 * n + _j];\n if (AS >= max) {\n max2 = max;\n max = AS;\n maxI = _j;\n } else if (AS > max2) {\n max2 = AS;\n }\n }\n for (var _j2 = 0; _j2 < n; _j2++) {\n R[_i8 * n + _j2] = (1 - opts.damping) * (S[_i8 * n + _j2] - max) + opts.damping * old[_j2];\n }\n R[_i8 * n + maxI] = (1 - opts.damping) * (S[_i8 * n + maxI] - max2) + opts.damping * old[maxI];\n }\n\n // Update A availability matrix\n for (var _i9 = 0; _i9 < n; _i9++) {\n var sum = 0;\n for (var _j3 = 0; _j3 < n; _j3++) {\n old[_j3] = A[_j3 * n + _i9];\n Rp[_j3] = Math.max(0, R[_j3 * n + _i9]);\n sum += Rp[_j3];\n }\n sum -= Rp[_i9];\n Rp[_i9] = R[_i9 * n + _i9];\n sum += Rp[_i9];\n for (var _j4 = 0; _j4 < n; _j4++) {\n A[_j4 * n + _i9] = (1 - opts.damping) * Math.min(0, sum - Rp[_j4]) + opts.damping * old[_j4];\n }\n A[_i9 * n + _i9] = (1 - opts.damping) * (sum - Rp[_i9]) + opts.damping * old[_i9];\n }\n\n // Check for convergence\n var K = 0;\n for (var _i10 = 0; _i10 < n; _i10++) {\n var E = A[_i10 * n + _i10] + R[_i10 * n + _i10] > 0 ? 1 : 0;\n e[iter % opts.minIterations * n + _i10] = E;\n K += E;\n }\n if (K > 0 && (iter >= opts.minIterations - 1 || iter == opts.maxIterations - 1)) {\n var _sum = 0;\n for (var _i11 = 0; _i11 < n; _i11++) {\n se[_i11] = 0;\n for (var _j5 = 0; _j5 < opts.minIterations; _j5++) {\n se[_i11] += e[_j5 * n + _i11];\n }\n if (se[_i11] === 0 || se[_i11] === opts.minIterations) {\n _sum++;\n }\n }\n if (_sum === n) {\n // then we have convergence\n break;\n }\n }\n }\n\n // Identify exemplars (cluster centers)\n var exemplarsIndices = findExemplars(n, R, A);\n\n // Assign nodes to clusters\n var clusterIndices = assign(n, S, exemplarsIndices);\n var clusters = {};\n for (var c = 0; c < exemplarsIndices.length; c++) {\n clusters[exemplarsIndices[c]] = [];\n }\n for (var _i12 = 0; _i12 < nodes.length; _i12++) {\n var pos = id2position[nodes[_i12].id()];\n var clusterIndex = clusterIndices[pos];\n if (clusterIndex != null) {\n // the node may have not been assigned a cluster if no valid attributes were specified\n clusters[clusterIndex].push(nodes[_i12]);\n }\n }\n var retClusters = new Array(exemplarsIndices.length);\n for (var _c = 0; _c < exemplarsIndices.length; _c++) {\n retClusters[_c] = cy.collection(clusters[exemplarsIndices[_c]]);\n }\n return retClusters;\n};\nvar affinityPropagation$1 = {\n affinityPropagation: affinityPropagation,\n ap: affinityPropagation\n};\n\nvar hierholzerDefaults = defaults$g({\n root: undefined,\n directed: false\n});\nvar elesfn$k = {\n hierholzer: function hierholzer(options) {\n if (!plainObject(options)) {\n var args = arguments;\n options = {\n root: args[0],\n directed: args[1]\n };\n }\n var _hierholzerDefaults = hierholzerDefaults(options),\n root = _hierholzerDefaults.root,\n directed = _hierholzerDefaults.directed;\n var eles = this;\n var dflag = false;\n var oddIn;\n var oddOut;\n var startVertex;\n if (root) startVertex = string(root) ? this.filter(root)[0].id() : root[0].id();\n var nodes = {};\n var edges = {};\n if (directed) {\n eles.forEach(function (ele) {\n var id = ele.id();\n if (ele.isNode()) {\n var ind = ele.indegree(true);\n var outd = ele.outdegree(true);\n var d1 = ind - outd;\n var d2 = outd - ind;\n if (d1 == 1) {\n if (oddIn) dflag = true;else oddIn = id;\n } else if (d2 == 1) {\n if (oddOut) dflag = true;else oddOut = id;\n } else if (d2 > 1 || d1 > 1) {\n dflag = true;\n }\n nodes[id] = [];\n ele.outgoers().forEach(function (e) {\n if (e.isEdge()) nodes[id].push(e.id());\n });\n } else {\n edges[id] = [undefined, ele.target().id()];\n }\n });\n } else {\n eles.forEach(function (ele) {\n var id = ele.id();\n if (ele.isNode()) {\n var d = ele.degree(true);\n if (d % 2) {\n if (!oddIn) oddIn = id;else if (!oddOut) oddOut = id;else dflag = true;\n }\n nodes[id] = [];\n ele.connectedEdges().forEach(function (e) {\n return nodes[id].push(e.id());\n });\n } else {\n edges[id] = [ele.source().id(), ele.target().id()];\n }\n });\n }\n var result = {\n found: false,\n trail: undefined\n };\n if (dflag) return result;else if (oddOut && oddIn) {\n if (directed) {\n if (startVertex && oddOut != startVertex) {\n return result;\n }\n startVertex = oddOut;\n } else {\n if (startVertex && oddOut != startVertex && oddIn != startVertex) {\n return result;\n } else if (!startVertex) {\n startVertex = oddOut;\n }\n }\n } else {\n if (!startVertex) startVertex = eles[0].id();\n }\n var walk = function walk(v) {\n var currentNode = v;\n var subtour = [v];\n var adj, adjTail, adjHead;\n while (nodes[currentNode].length) {\n adj = nodes[currentNode].shift();\n adjTail = edges[adj][0];\n adjHead = edges[adj][1];\n if (currentNode != adjHead) {\n nodes[adjHead] = nodes[adjHead].filter(function (e) {\n return e != adj;\n });\n currentNode = adjHead;\n } else if (!directed && currentNode != adjTail) {\n nodes[adjTail] = nodes[adjTail].filter(function (e) {\n return e != adj;\n });\n currentNode = adjTail;\n }\n subtour.unshift(adj);\n subtour.unshift(currentNode);\n }\n return subtour;\n };\n var trail = [];\n var subtour = [];\n subtour = walk(startVertex);\n while (subtour.length != 1) {\n if (nodes[subtour[0]].length == 0) {\n trail.unshift(eles.getElementById(subtour.shift()));\n trail.unshift(eles.getElementById(subtour.shift()));\n } else {\n subtour = walk(subtour.shift()).concat(subtour);\n }\n }\n trail.unshift(eles.getElementById(subtour.shift())); // final node\n\n for (var d in nodes) {\n if (nodes[d].length) {\n return result;\n }\n }\n result.found = true;\n result.trail = this.spawn(trail, true);\n return result;\n }\n};\n\nvar hopcroftTarjanBiconnected = function hopcroftTarjanBiconnected() {\n var eles = this;\n var nodes = {};\n var id = 0;\n var edgeCount = 0;\n var components = [];\n var stack = [];\n var visitedEdges = {};\n var buildComponent = function buildComponent(x, y) {\n var i = stack.length - 1;\n var cutset = [];\n var component = eles.spawn();\n while (stack[i].x != x || stack[i].y != y) {\n cutset.push(stack.pop().edge);\n i--;\n }\n cutset.push(stack.pop().edge);\n cutset.forEach(function (edge) {\n var connectedNodes = edge.connectedNodes().intersection(eles);\n component.merge(edge);\n connectedNodes.forEach(function (node) {\n var nodeId = node.id();\n var connectedEdges = node.connectedEdges().intersection(eles);\n component.merge(node);\n if (!nodes[nodeId].cutVertex) {\n component.merge(connectedEdges);\n } else {\n component.merge(connectedEdges.filter(function (edge) {\n return edge.isLoop();\n }));\n }\n });\n });\n components.push(component);\n };\n var _biconnectedSearch = function biconnectedSearch(root, currentNode, parent) {\n if (root === parent) edgeCount += 1;\n nodes[currentNode] = {\n id: id,\n low: id++,\n cutVertex: false\n };\n var edges = eles.getElementById(currentNode).connectedEdges().intersection(eles);\n if (edges.size() === 0) {\n components.push(eles.spawn(eles.getElementById(currentNode)));\n } else {\n var sourceId, targetId, otherNodeId, edgeId;\n edges.forEach(function (edge) {\n sourceId = edge.source().id();\n targetId = edge.target().id();\n otherNodeId = sourceId === currentNode ? targetId : sourceId;\n if (otherNodeId !== parent) {\n edgeId = edge.id();\n if (!visitedEdges[edgeId]) {\n visitedEdges[edgeId] = true;\n stack.push({\n x: currentNode,\n y: otherNodeId,\n edge: edge\n });\n }\n if (!(otherNodeId in nodes)) {\n _biconnectedSearch(root, otherNodeId, currentNode);\n nodes[currentNode].low = Math.min(nodes[currentNode].low, nodes[otherNodeId].low);\n if (nodes[currentNode].id <= nodes[otherNodeId].low) {\n nodes[currentNode].cutVertex = true;\n buildComponent(currentNode, otherNodeId);\n }\n } else {\n nodes[currentNode].low = Math.min(nodes[currentNode].low, nodes[otherNodeId].id);\n }\n }\n });\n }\n };\n eles.forEach(function (ele) {\n if (ele.isNode()) {\n var nodeId = ele.id();\n if (!(nodeId in nodes)) {\n edgeCount = 0;\n _biconnectedSearch(nodeId, nodeId);\n nodes[nodeId].cutVertex = edgeCount > 1;\n }\n }\n });\n var cutVertices = Object.keys(nodes).filter(function (id) {\n return nodes[id].cutVertex;\n }).map(function (id) {\n return eles.getElementById(id);\n });\n return {\n cut: eles.spawn(cutVertices),\n components: components\n };\n};\nvar hopcroftTarjanBiconnected$1 = {\n hopcroftTarjanBiconnected: hopcroftTarjanBiconnected,\n htbc: hopcroftTarjanBiconnected,\n htb: hopcroftTarjanBiconnected,\n hopcroftTarjanBiconnectedComponents: hopcroftTarjanBiconnected\n};\n\nvar tarjanStronglyConnected = function tarjanStronglyConnected() {\n var eles = this;\n var nodes = {};\n var index = 0;\n var components = [];\n var stack = [];\n var cut = eles.spawn(eles);\n var _stronglyConnectedSearch = function stronglyConnectedSearch(sourceNodeId) {\n stack.push(sourceNodeId);\n nodes[sourceNodeId] = {\n index: index,\n low: index++,\n explored: false\n };\n var connectedEdges = eles.getElementById(sourceNodeId).connectedEdges().intersection(eles);\n connectedEdges.forEach(function (edge) {\n var targetNodeId = edge.target().id();\n if (targetNodeId !== sourceNodeId) {\n if (!(targetNodeId in nodes)) {\n _stronglyConnectedSearch(targetNodeId);\n }\n if (!nodes[targetNodeId].explored) {\n nodes[sourceNodeId].low = Math.min(nodes[sourceNodeId].low, nodes[targetNodeId].low);\n }\n }\n });\n if (nodes[sourceNodeId].index === nodes[sourceNodeId].low) {\n var componentNodes = eles.spawn();\n for (;;) {\n var nodeId = stack.pop();\n componentNodes.merge(eles.getElementById(nodeId));\n nodes[nodeId].low = nodes[sourceNodeId].index;\n nodes[nodeId].explored = true;\n if (nodeId === sourceNodeId) {\n break;\n }\n }\n var componentEdges = componentNodes.edgesWith(componentNodes);\n var component = componentNodes.merge(componentEdges);\n components.push(component);\n cut = cut.difference(component);\n }\n };\n eles.forEach(function (ele) {\n if (ele.isNode()) {\n var nodeId = ele.id();\n if (!(nodeId in nodes)) {\n _stronglyConnectedSearch(nodeId);\n }\n }\n });\n return {\n cut: cut,\n components: components\n };\n};\nvar tarjanStronglyConnected$1 = {\n tarjanStronglyConnected: tarjanStronglyConnected,\n tsc: tarjanStronglyConnected,\n tscc: tarjanStronglyConnected,\n tarjanStronglyConnectedComponents: tarjanStronglyConnected\n};\n\nvar elesfn$j = {};\n[elesfn$v, elesfn$u, elesfn$t, elesfn$s, elesfn$r, elesfn$q, elesfn$p, elesfn$o, elesfn$n, elesfn$m, elesfn$l, markovClustering$1, kClustering, hierarchicalClustering$1, affinityPropagation$1, elesfn$k, hopcroftTarjanBiconnected$1, tarjanStronglyConnected$1].forEach(function (props) {\n extend(elesfn$j, props);\n});\n\n/*!\nEmbeddable Minimum Strictly-Compliant Promises/A+ 1.1.1 Thenable\nCopyright (c) 2013-2014 Ralf S. Engelschall (http://engelschall.com)\nLicensed under The MIT License (http://opensource.org/licenses/MIT)\n*/\n\n/* promise states [Promises/A+ 2.1] */\nvar STATE_PENDING = 0; /* [Promises/A+ 2.1.1] */\nvar STATE_FULFILLED = 1; /* [Promises/A+ 2.1.2] */\nvar STATE_REJECTED = 2; /* [Promises/A+ 2.1.3] */\n\n/* promise object constructor */\nvar _api = function api(executor) {\n /* optionally support non-constructor/plain-function call */\n if (!(this instanceof _api)) return new _api(executor);\n\n /* initialize object */\n this.id = 'Thenable/1.0.7';\n this.state = STATE_PENDING; /* initial state */\n this.fulfillValue = undefined; /* initial value */ /* [Promises/A+ 1.3, 2.1.2.2] */\n this.rejectReason = undefined; /* initial reason */ /* [Promises/A+ 1.5, 2.1.3.2] */\n this.onFulfilled = []; /* initial handlers */\n this.onRejected = []; /* initial handlers */\n\n /* provide optional information-hiding proxy */\n this.proxy = {\n then: this.then.bind(this)\n };\n\n /* support optional executor function */\n if (typeof executor === 'function') executor.call(this, this.fulfill.bind(this), this.reject.bind(this));\n};\n\n/* promise API methods */\n_api.prototype = {\n /* promise resolving methods */\n fulfill: function fulfill(value) {\n return deliver(this, STATE_FULFILLED, 'fulfillValue', value);\n },\n reject: function reject(value) {\n return deliver(this, STATE_REJECTED, 'rejectReason', value);\n },\n /* \"The then Method\" [Promises/A+ 1.1, 1.2, 2.2] */\n then: function then(onFulfilled, onRejected) {\n var curr = this;\n var next = new _api(); /* [Promises/A+ 2.2.7] */\n curr.onFulfilled.push(resolver(onFulfilled, next, 'fulfill')); /* [Promises/A+ 2.2.2/2.2.6] */\n curr.onRejected.push(resolver(onRejected, next, 'reject')); /* [Promises/A+ 2.2.3/2.2.6] */\n execute(curr);\n return next.proxy; /* [Promises/A+ 2.2.7, 3.3] */\n }\n};\n\n/* deliver an action */\nvar deliver = function deliver(curr, state, name, value) {\n if (curr.state === STATE_PENDING) {\n curr.state = state; /* [Promises/A+ 2.1.2.1, 2.1.3.1] */\n curr[name] = value; /* [Promises/A+ 2.1.2.2, 2.1.3.2] */\n execute(curr);\n }\n return curr;\n};\n\n/* execute all handlers */\nvar execute = function execute(curr) {\n if (curr.state === STATE_FULFILLED) execute_handlers(curr, 'onFulfilled', curr.fulfillValue);else if (curr.state === STATE_REJECTED) execute_handlers(curr, 'onRejected', curr.rejectReason);\n};\n\n/* execute particular set of handlers */\nvar execute_handlers = function execute_handlers(curr, name, value) {\n /* global setImmediate: true */\n /* global setTimeout: true */\n\n /* short-circuit processing */\n if (curr[name].length === 0) return;\n\n /* iterate over all handlers, exactly once */\n var handlers = curr[name];\n curr[name] = []; /* [Promises/A+ 2.2.2.3, 2.2.3.3] */\n var func = function func() {\n for (var i = 0; i < handlers.length; i++) handlers[i](value); /* [Promises/A+ 2.2.5] */\n };\n\n /* execute procedure asynchronously */ /* [Promises/A+ 2.2.4, 3.1] */\n if (typeof setImmediate === 'function') setImmediate(func);else setTimeout(func, 0);\n};\n\n/* generate a resolver function */\nvar resolver = function resolver(cb, next, method) {\n return function (value) {\n if (typeof cb !== 'function') /* [Promises/A+ 2.2.1, 2.2.7.3, 2.2.7.4] */\n next[method].call(next, value); /* [Promises/A+ 2.2.7.3, 2.2.7.4] */else {\n var result;\n try {\n result = cb(value);\n } /* [Promises/A+ 2.2.2.1, 2.2.3.1, 2.2.5, 3.2] */ catch (e) {\n next.reject(e); /* [Promises/A+ 2.2.7.2] */\n return;\n }\n _resolve(next, result); /* [Promises/A+ 2.2.7.1] */\n }\n };\n};\n\n/* \"Promise Resolution Procedure\" */ /* [Promises/A+ 2.3] */\nvar _resolve = function resolve(promise, x) {\n /* sanity check arguments */ /* [Promises/A+ 2.3.1] */\n if (promise === x || promise.proxy === x) {\n promise.reject(new TypeError('cannot resolve promise with itself'));\n return;\n }\n\n /* surgically check for a \"then\" method\n (mainly to just call the \"getter\" of \"then\" only once) */\n var then;\n if (_typeof(x) === 'object' && x !== null || typeof x === 'function') {\n try {\n then = x.then;\n } /* [Promises/A+ 2.3.3.1, 3.5] */ catch (e) {\n promise.reject(e); /* [Promises/A+ 2.3.3.2] */\n return;\n }\n }\n\n /* handle own Thenables [Promises/A+ 2.3.2]\n and similar \"thenables\" [Promises/A+ 2.3.3] */\n if (typeof then === 'function') {\n var resolved = false;\n try {\n /* call retrieved \"then\" method */ /* [Promises/A+ 2.3.3.3] */\n then.call(x, /* resolvePromise */ /* [Promises/A+ 2.3.3.3.1] */\n function (y) {\n if (resolved) return;\n resolved = true; /* [Promises/A+ 2.3.3.3.3] */\n if (y === x) /* [Promises/A+ 3.6] */\n promise.reject(new TypeError('circular thenable chain'));else _resolve(promise, y);\n }, /* rejectPromise */ /* [Promises/A+ 2.3.3.3.2] */\n function (r) {\n if (resolved) return;\n resolved = true; /* [Promises/A+ 2.3.3.3.3] */\n promise.reject(r);\n });\n } catch (e) {\n if (!resolved) /* [Promises/A+ 2.3.3.3.3] */\n promise.reject(e); /* [Promises/A+ 2.3.3.3.4] */\n }\n return;\n }\n\n /* handle other values */\n promise.fulfill(x); /* [Promises/A+ 2.3.4, 2.3.3.4] */\n};\n\n// so we always have Promise.all()\n_api.all = function (ps) {\n return new _api(function (resolveAll, rejectAll) {\n var vals = new Array(ps.length);\n var doneCount = 0;\n var fulfill = function fulfill(i, val) {\n vals[i] = val;\n doneCount++;\n if (doneCount === ps.length) {\n resolveAll(vals);\n }\n };\n for (var i = 0; i < ps.length; i++) {\n (function (i) {\n var p = ps[i];\n var isPromise = p != null && p.then != null;\n if (isPromise) {\n p.then(function (val) {\n fulfill(i, val);\n }, function (err) {\n rejectAll(err);\n });\n } else {\n var val = p;\n fulfill(i, val);\n }\n })(i);\n }\n });\n};\n_api.resolve = function (val) {\n return new _api(function (resolve, reject) {\n resolve(val);\n });\n};\n_api.reject = function (val) {\n return new _api(function (resolve, reject) {\n reject(val);\n });\n};\nvar Promise$1 = typeof Promise !== 'undefined' ? Promise : _api; // eslint-disable-line no-undef\n\nvar Animation = function Animation(target, opts, opts2) {\n var isCore = core(target);\n var isEle = !isCore;\n var _p = this._private = extend({\n duration: 1000\n }, opts, opts2);\n _p.target = target;\n _p.style = _p.style || _p.css;\n _p.started = false;\n _p.playing = false;\n _p.hooked = false;\n _p.applying = false;\n _p.progress = 0;\n _p.completes = [];\n _p.frames = [];\n if (_p.complete && fn$6(_p.complete)) {\n _p.completes.push(_p.complete);\n }\n if (isEle) {\n var pos = target.position();\n _p.startPosition = _p.startPosition || {\n x: pos.x,\n y: pos.y\n };\n _p.startStyle = _p.startStyle || target.cy().style().getAnimationStartStyle(target, _p.style);\n }\n if (isCore) {\n var pan = target.pan();\n _p.startPan = {\n x: pan.x,\n y: pan.y\n };\n _p.startZoom = target.zoom();\n }\n\n // for future timeline/animations impl\n this.length = 1;\n this[0] = this;\n};\nvar anifn = Animation.prototype;\nextend(anifn, {\n instanceString: function instanceString() {\n return 'animation';\n },\n hook: function hook() {\n var _p = this._private;\n if (!_p.hooked) {\n // add to target's animation queue\n var q;\n var tAni = _p.target._private.animation;\n if (_p.queue) {\n q = tAni.queue;\n } else {\n q = tAni.current;\n }\n q.push(this);\n\n // add to the animation loop pool\n if (elementOrCollection(_p.target)) {\n _p.target.cy().addToAnimationPool(_p.target);\n }\n _p.hooked = true;\n }\n return this;\n },\n play: function play() {\n var _p = this._private;\n\n // autorewind\n if (_p.progress === 1) {\n _p.progress = 0;\n }\n _p.playing = true;\n _p.started = false; // needs to be started by animation loop\n _p.stopped = false;\n this.hook();\n\n // the animation loop will start the animation...\n\n return this;\n },\n playing: function playing() {\n return this._private.playing;\n },\n apply: function apply() {\n var _p = this._private;\n _p.applying = true;\n _p.started = false; // needs to be started by animation loop\n _p.stopped = false;\n this.hook();\n\n // the animation loop will apply the animation at this progress\n\n return this;\n },\n applying: function applying() {\n return this._private.applying;\n },\n pause: function pause() {\n var _p = this._private;\n _p.playing = false;\n _p.started = false;\n return this;\n },\n stop: function stop() {\n var _p = this._private;\n _p.playing = false;\n _p.started = false;\n _p.stopped = true; // to be removed from animation queues\n\n return this;\n },\n rewind: function rewind() {\n return this.progress(0);\n },\n fastforward: function fastforward() {\n return this.progress(1);\n },\n time: function time(t) {\n var _p = this._private;\n if (t === undefined) {\n return _p.progress * _p.duration;\n } else {\n return this.progress(t / _p.duration);\n }\n },\n progress: function progress(p) {\n var _p = this._private;\n var wasPlaying = _p.playing;\n if (p === undefined) {\n return _p.progress;\n } else {\n if (wasPlaying) {\n this.pause();\n }\n _p.progress = p;\n _p.started = false;\n if (wasPlaying) {\n this.play();\n }\n }\n return this;\n },\n completed: function completed() {\n return this._private.progress === 1;\n },\n reverse: function reverse() {\n var _p = this._private;\n var wasPlaying = _p.playing;\n if (wasPlaying) {\n this.pause();\n }\n _p.progress = 1 - _p.progress;\n _p.started = false;\n var swap = function swap(a, b) {\n var _pa = _p[a];\n if (_pa == null) {\n return;\n }\n _p[a] = _p[b];\n _p[b] = _pa;\n };\n swap('zoom', 'startZoom');\n swap('pan', 'startPan');\n swap('position', 'startPosition');\n\n // swap styles\n if (_p.style) {\n for (var i = 0; i < _p.style.length; i++) {\n var prop = _p.style[i];\n var name = prop.name;\n var startStyleProp = _p.startStyle[name];\n _p.startStyle[name] = prop;\n _p.style[i] = startStyleProp;\n }\n }\n if (wasPlaying) {\n this.play();\n }\n return this;\n },\n promise: function promise(type) {\n var _p = this._private;\n var arr;\n switch (type) {\n case 'frame':\n arr = _p.frames;\n break;\n default:\n case 'complete':\n case 'completed':\n arr = _p.completes;\n }\n return new Promise$1(function (resolve, reject) {\n arr.push(function () {\n resolve();\n });\n });\n }\n});\nanifn.complete = anifn.completed;\nanifn.run = anifn.play;\nanifn.running = anifn.playing;\n\nvar define$3 = {\n animated: function animated() {\n return function animatedImpl() {\n var self = this;\n var selfIsArrayLike = self.length !== undefined;\n var all = selfIsArrayLike ? self : [self]; // put in array if not array-like\n var cy = this._private.cy || this;\n if (!cy.styleEnabled()) {\n return false;\n }\n var ele = all[0];\n if (ele) {\n return ele._private.animation.current.length > 0;\n }\n };\n },\n // animated\n\n clearQueue: function clearQueue() {\n return function clearQueueImpl() {\n var self = this;\n var selfIsArrayLike = self.length !== undefined;\n var all = selfIsArrayLike ? self : [self]; // put in array if not array-like\n var cy = this._private.cy || this;\n if (!cy.styleEnabled()) {\n return this;\n }\n for (var i = 0; i < all.length; i++) {\n var ele = all[i];\n ele._private.animation.queue = [];\n }\n return this;\n };\n },\n // clearQueue\n\n delay: function delay() {\n return function delayImpl(time, complete) {\n var cy = this._private.cy || this;\n if (!cy.styleEnabled()) {\n return this;\n }\n return this.animate({\n delay: time,\n duration: time,\n complete: complete\n });\n };\n },\n // delay\n\n delayAnimation: function delayAnimation() {\n return function delayAnimationImpl(time, complete) {\n var cy = this._private.cy || this;\n if (!cy.styleEnabled()) {\n return this;\n }\n return this.animation({\n delay: time,\n duration: time,\n complete: complete\n });\n };\n },\n // delay\n\n animation: function animation() {\n return function animationImpl(properties, params) {\n var self = this;\n var selfIsArrayLike = self.length !== undefined;\n var all = selfIsArrayLike ? self : [self]; // put in array if not array-like\n var cy = this._private.cy || this;\n var isCore = !selfIsArrayLike;\n var isEles = !isCore;\n if (!cy.styleEnabled()) {\n return this;\n }\n var style = cy.style();\n properties = extend({}, properties, params);\n var propertiesEmpty = Object.keys(properties).length === 0;\n if (propertiesEmpty) {\n return new Animation(all[0], properties); // nothing to animate\n }\n if (properties.duration === undefined) {\n properties.duration = 400;\n }\n switch (properties.duration) {\n case 'slow':\n properties.duration = 600;\n break;\n case 'fast':\n properties.duration = 200;\n break;\n }\n if (isEles) {\n properties.style = style.getPropsList(properties.style || properties.css);\n properties.css = undefined;\n }\n if (isEles && properties.renderedPosition != null) {\n var rpos = properties.renderedPosition;\n var pan = cy.pan();\n var zoom = cy.zoom();\n properties.position = renderedToModelPosition(rpos, zoom, pan);\n }\n\n // override pan w/ panBy if set\n if (isCore && properties.panBy != null) {\n var panBy = properties.panBy;\n var cyPan = cy.pan();\n properties.pan = {\n x: cyPan.x + panBy.x,\n y: cyPan.y + panBy.y\n };\n }\n\n // override pan w/ center if set\n var center = properties.center || properties.centre;\n if (isCore && center != null) {\n var centerPan = cy.getCenterPan(center.eles, properties.zoom);\n if (centerPan != null) {\n properties.pan = centerPan;\n }\n }\n\n // override pan & zoom w/ fit if set\n if (isCore && properties.fit != null) {\n var fit = properties.fit;\n var fitVp = cy.getFitViewport(fit.eles || fit.boundingBox, fit.padding);\n if (fitVp != null) {\n properties.pan = fitVp.pan;\n properties.zoom = fitVp.zoom;\n }\n }\n\n // override zoom (& potentially pan) w/ zoom obj if set\n if (isCore && plainObject(properties.zoom)) {\n var vp = cy.getZoomedViewport(properties.zoom);\n if (vp != null) {\n if (vp.zoomed) {\n properties.zoom = vp.zoom;\n }\n if (vp.panned) {\n properties.pan = vp.pan;\n }\n } else {\n properties.zoom = null; // an inavalid zoom (e.g. no delta) gets automatically destroyed\n }\n }\n return new Animation(all[0], properties);\n };\n },\n // animate\n\n animate: function animate() {\n return function animateImpl(properties, params) {\n var self = this;\n var selfIsArrayLike = self.length !== undefined;\n var all = selfIsArrayLike ? self : [self]; // put in array if not array-like\n var cy = this._private.cy || this;\n if (!cy.styleEnabled()) {\n return this;\n }\n if (params) {\n properties = extend({}, properties, params);\n }\n\n // manually hook and run the animation\n for (var i = 0; i < all.length; i++) {\n var ele = all[i];\n var queue = ele.animated() && (properties.queue === undefined || properties.queue);\n var ani = ele.animation(properties, queue ? {\n queue: true\n } : undefined);\n ani.play();\n }\n return this; // chaining\n };\n },\n // animate\n\n stop: function stop() {\n return function stopImpl(clearQueue, jumpToEnd) {\n var self = this;\n var selfIsArrayLike = self.length !== undefined;\n var all = selfIsArrayLike ? self : [self]; // put in array if not array-like\n var cy = this._private.cy || this;\n if (!cy.styleEnabled()) {\n return this;\n }\n for (var i = 0; i < all.length; i++) {\n var ele = all[i];\n var _p = ele._private;\n var anis = _p.animation.current;\n for (var j = 0; j < anis.length; j++) {\n var ani = anis[j];\n var ani_p = ani._private;\n if (jumpToEnd) {\n // next iteration of the animation loop, the animation\n // will go straight to the end and be removed\n ani_p.duration = 0;\n }\n }\n\n // clear the queue of future animations\n if (clearQueue) {\n _p.animation.queue = [];\n }\n if (!jumpToEnd) {\n _p.animation.current = [];\n }\n }\n\n // we have to notify (the animation loop doesn't do it for us on `stop`)\n cy.notify('draw');\n return this;\n };\n } // stop\n}; // define\n\n/**\n * Checks if `value` is classified as an `Array` object.\n *\n * @static\n * @memberOf _\n * @since 0.1.0\n * @category Lang\n * @param {*} value The value to check.\n * @returns {boolean} Returns `true` if `value` is an array, else `false`.\n * @example\n *\n * _.isArray([1, 2, 3]);\n * // => true\n *\n * _.isArray(document.body.children);\n * // => false\n *\n * _.isArray('abc');\n * // => false\n *\n * _.isArray(_.noop);\n * // => false\n */\n\nvar isArray_1;\nvar hasRequiredIsArray;\n\nfunction requireIsArray () {\n\tif (hasRequiredIsArray) return isArray_1;\n\thasRequiredIsArray = 1;\n\tvar isArray = Array.isArray;\n\n\tisArray_1 = isArray;\n\treturn isArray_1;\n}\n\nvar _isKey;\nvar hasRequired_isKey;\n\nfunction require_isKey () {\n\tif (hasRequired_isKey) return _isKey;\n\thasRequired_isKey = 1;\n\tvar isArray = requireIsArray(),\n\t isSymbol = requireIsSymbol();\n\n\t/** Used to match property names within property paths. */\n\tvar reIsDeepProp = /\\.|\\[(?:[^[\\]]*|([\"'])(?:(?!\\1)[^\\\\]|\\\\.)*?\\1)\\]/,\n\t reIsPlainProp = /^\\w*$/;\n\n\t/**\n\t * Checks if `value` is a property name and not a property path.\n\t *\n\t * @private\n\t * @param {*} value The value to check.\n\t * @param {Object} [object] The object to query keys on.\n\t * @returns {boolean} Returns `true` if `value` is a property name, else `false`.\n\t */\n\tfunction isKey(value, object) {\n\t if (isArray(value)) {\n\t return false;\n\t }\n\t var type = typeof value;\n\t if (type == 'number' || type == 'symbol' || type == 'boolean' ||\n\t value == null || isSymbol(value)) {\n\t return true;\n\t }\n\t return reIsPlainProp.test(value) || !reIsDeepProp.test(value) ||\n\t (object != null && value in Object(object));\n\t}\n\n\t_isKey = isKey;\n\treturn _isKey;\n}\n\nvar isFunction_1;\nvar hasRequiredIsFunction;\n\nfunction requireIsFunction () {\n\tif (hasRequiredIsFunction) return isFunction_1;\n\thasRequiredIsFunction = 1;\n\tvar baseGetTag = require_baseGetTag(),\n\t isObject = requireIsObject();\n\n\t/** `Object#toString` result references. */\n\tvar asyncTag = '[object AsyncFunction]',\n\t funcTag = '[object Function]',\n\t genTag = '[object GeneratorFunction]',\n\t proxyTag = '[object Proxy]';\n\n\t/**\n\t * Checks if `value` is classified as a `Function` object.\n\t *\n\t * @static\n\t * @memberOf _\n\t * @since 0.1.0\n\t * @category Lang\n\t * @param {*} value The value to check.\n\t * @returns {boolean} Returns `true` if `value` is a function, else `false`.\n\t * @example\n\t *\n\t * _.isFunction(_);\n\t * // => true\n\t *\n\t * _.isFunction(/abc/);\n\t * // => false\n\t */\n\tfunction isFunction(value) {\n\t if (!isObject(value)) {\n\t return false;\n\t }\n\t // The use of `Object#toString` avoids issues with the `typeof` operator\n\t // in Safari 9 which returns 'object' for typed arrays and other constructors.\n\t var tag = baseGetTag(value);\n\t return tag == funcTag || tag == genTag || tag == asyncTag || tag == proxyTag;\n\t}\n\n\tisFunction_1 = isFunction;\n\treturn isFunction_1;\n}\n\nvar _coreJsData;\nvar hasRequired_coreJsData;\n\nfunction require_coreJsData () {\n\tif (hasRequired_coreJsData) return _coreJsData;\n\thasRequired_coreJsData = 1;\n\tvar root = require_root();\n\n\t/** Used to detect overreaching core-js shims. */\n\tvar coreJsData = root['__core-js_shared__'];\n\n\t_coreJsData = coreJsData;\n\treturn _coreJsData;\n}\n\nvar _isMasked;\nvar hasRequired_isMasked;\n\nfunction require_isMasked () {\n\tif (hasRequired_isMasked) return _isMasked;\n\thasRequired_isMasked = 1;\n\tvar coreJsData = require_coreJsData();\n\n\t/** Used to detect methods masquerading as native. */\n\tvar maskSrcKey = (function() {\n\t var uid = /[^.]+$/.exec(coreJsData && coreJsData.keys && coreJsData.keys.IE_PROTO || '');\n\t return uid ? ('Symbol(src)_1.' + uid) : '';\n\t}());\n\n\t/**\n\t * Checks if `func` has its source masked.\n\t *\n\t * @private\n\t * @param {Function} func The function to check.\n\t * @returns {boolean} Returns `true` if `func` is masked, else `false`.\n\t */\n\tfunction isMasked(func) {\n\t return !!maskSrcKey && (maskSrcKey in func);\n\t}\n\n\t_isMasked = isMasked;\n\treturn _isMasked;\n}\n\n/** Used for built-in method references. */\n\nvar _toSource;\nvar hasRequired_toSource;\n\nfunction require_toSource () {\n\tif (hasRequired_toSource) return _toSource;\n\thasRequired_toSource = 1;\n\tvar funcProto = Function.prototype;\n\n\t/** Used to resolve the decompiled source of functions. */\n\tvar funcToString = funcProto.toString;\n\n\t/**\n\t * Converts `func` to its source code.\n\t *\n\t * @private\n\t * @param {Function} func The function to convert.\n\t * @returns {string} Returns the source code.\n\t */\n\tfunction toSource(func) {\n\t if (func != null) {\n\t try {\n\t return funcToString.call(func);\n\t } catch (e) {}\n\t try {\n\t return (func + '');\n\t } catch (e) {}\n\t }\n\t return '';\n\t}\n\n\t_toSource = toSource;\n\treturn _toSource;\n}\n\nvar _baseIsNative;\nvar hasRequired_baseIsNative;\n\nfunction require_baseIsNative () {\n\tif (hasRequired_baseIsNative) return _baseIsNative;\n\thasRequired_baseIsNative = 1;\n\tvar isFunction = requireIsFunction(),\n\t isMasked = require_isMasked(),\n\t isObject = requireIsObject(),\n\t toSource = require_toSource();\n\n\t/**\n\t * Used to match `RegExp`\n\t * [syntax characters](http://ecma-international.org/ecma-262/7.0/#sec-patterns).\n\t */\n\tvar reRegExpChar = /[\\\\^$.*+?()[\\]{}|]/g;\n\n\t/** Used to detect host constructors (Safari). */\n\tvar reIsHostCtor = /^\\[object .+?Constructor\\]$/;\n\n\t/** Used for built-in method references. */\n\tvar funcProto = Function.prototype,\n\t objectProto = Object.prototype;\n\n\t/** Used to resolve the decompiled source of functions. */\n\tvar funcToString = funcProto.toString;\n\n\t/** Used to check objects for own properties. */\n\tvar hasOwnProperty = objectProto.hasOwnProperty;\n\n\t/** Used to detect if a method is native. */\n\tvar reIsNative = RegExp('^' +\n\t funcToString.call(hasOwnProperty).replace(reRegExpChar, '\\\\$&')\n\t .replace(/hasOwnProperty|(function).*?(?=\\\\\\()| for .+?(?=\\\\\\])/g, '$1.*?') + '$'\n\t);\n\n\t/**\n\t * The base implementation of `_.isNative` without bad shim checks.\n\t *\n\t * @private\n\t * @param {*} value The value to check.\n\t * @returns {boolean} Returns `true` if `value` is a native function,\n\t * else `false`.\n\t */\n\tfunction baseIsNative(value) {\n\t if (!isObject(value) || isMasked(value)) {\n\t return false;\n\t }\n\t var pattern = isFunction(value) ? reIsNative : reIsHostCtor;\n\t return pattern.test(toSource(value));\n\t}\n\n\t_baseIsNative = baseIsNative;\n\treturn _baseIsNative;\n}\n\n/**\n * Gets the value at `key` of `object`.\n *\n * @private\n * @param {Object} [object] The object to query.\n * @param {string} key The key of the property to get.\n * @returns {*} Returns the property value.\n */\n\nvar _getValue;\nvar hasRequired_getValue;\n\nfunction require_getValue () {\n\tif (hasRequired_getValue) return _getValue;\n\thasRequired_getValue = 1;\n\tfunction getValue(object, key) {\n\t return object == null ? undefined : object[key];\n\t}\n\n\t_getValue = getValue;\n\treturn _getValue;\n}\n\nvar _getNative;\nvar hasRequired_getNative;\n\nfunction require_getNative () {\n\tif (hasRequired_getNative) return _getNative;\n\thasRequired_getNative = 1;\n\tvar baseIsNative = require_baseIsNative(),\n\t getValue = require_getValue();\n\n\t/**\n\t * Gets the native function at `key` of `object`.\n\t *\n\t * @private\n\t * @param {Object} object The object to query.\n\t * @param {string} key The key of the method to get.\n\t * @returns {*} Returns the function if it's native, else `undefined`.\n\t */\n\tfunction getNative(object, key) {\n\t var value = getValue(object, key);\n\t return baseIsNative(value) ? value : undefined;\n\t}\n\n\t_getNative = getNative;\n\treturn _getNative;\n}\n\nvar _nativeCreate;\nvar hasRequired_nativeCreate;\n\nfunction require_nativeCreate () {\n\tif (hasRequired_nativeCreate) return _nativeCreate;\n\thasRequired_nativeCreate = 1;\n\tvar getNative = require_getNative();\n\n\t/* Built-in method references that are verified to be native. */\n\tvar nativeCreate = getNative(Object, 'create');\n\n\t_nativeCreate = nativeCreate;\n\treturn _nativeCreate;\n}\n\nvar _hashClear;\nvar hasRequired_hashClear;\n\nfunction require_hashClear () {\n\tif (hasRequired_hashClear) return _hashClear;\n\thasRequired_hashClear = 1;\n\tvar nativeCreate = require_nativeCreate();\n\n\t/**\n\t * Removes all key-value entries from the hash.\n\t *\n\t * @private\n\t * @name clear\n\t * @memberOf Hash\n\t */\n\tfunction hashClear() {\n\t this.__data__ = nativeCreate ? nativeCreate(null) : {};\n\t this.size = 0;\n\t}\n\n\t_hashClear = hashClear;\n\treturn _hashClear;\n}\n\n/**\n * Removes `key` and its value from the hash.\n *\n * @private\n * @name delete\n * @memberOf Hash\n * @param {Object} hash The hash to modify.\n * @param {string} key The key of the value to remove.\n * @returns {boolean} Returns `true` if the entry was removed, else `false`.\n */\n\nvar _hashDelete;\nvar hasRequired_hashDelete;\n\nfunction require_hashDelete () {\n\tif (hasRequired_hashDelete) return _hashDelete;\n\thasRequired_hashDelete = 1;\n\tfunction hashDelete(key) {\n\t var result = this.has(key) && delete this.__data__[key];\n\t this.size -= result ? 1 : 0;\n\t return result;\n\t}\n\n\t_hashDelete = hashDelete;\n\treturn _hashDelete;\n}\n\nvar _hashGet;\nvar hasRequired_hashGet;\n\nfunction require_hashGet () {\n\tif (hasRequired_hashGet) return _hashGet;\n\thasRequired_hashGet = 1;\n\tvar nativeCreate = require_nativeCreate();\n\n\t/** Used to stand-in for `undefined` hash values. */\n\tvar HASH_UNDEFINED = '__lodash_hash_undefined__';\n\n\t/** Used for built-in method references. */\n\tvar objectProto = Object.prototype;\n\n\t/** Used to check objects for own properties. */\n\tvar hasOwnProperty = objectProto.hasOwnProperty;\n\n\t/**\n\t * Gets the hash value for `key`.\n\t *\n\t * @private\n\t * @name get\n\t * @memberOf Hash\n\t * @param {string} key The key of the value to get.\n\t * @returns {*} Returns the entry value.\n\t */\n\tfunction hashGet(key) {\n\t var data = this.__data__;\n\t if (nativeCreate) {\n\t var result = data[key];\n\t return result === HASH_UNDEFINED ? undefined : result;\n\t }\n\t return hasOwnProperty.call(data, key) ? data[key] : undefined;\n\t}\n\n\t_hashGet = hashGet;\n\treturn _hashGet;\n}\n\nvar _hashHas;\nvar hasRequired_hashHas;\n\nfunction require_hashHas () {\n\tif (hasRequired_hashHas) return _hashHas;\n\thasRequired_hashHas = 1;\n\tvar nativeCreate = require_nativeCreate();\n\n\t/** Used for built-in method references. */\n\tvar objectProto = Object.prototype;\n\n\t/** Used to check objects for own properties. */\n\tvar hasOwnProperty = objectProto.hasOwnProperty;\n\n\t/**\n\t * Checks if a hash value for `key` exists.\n\t *\n\t * @private\n\t * @name has\n\t * @memberOf Hash\n\t * @param {string} key The key of the entry to check.\n\t * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`.\n\t */\n\tfunction hashHas(key) {\n\t var data = this.__data__;\n\t return nativeCreate ? (data[key] !== undefined) : hasOwnProperty.call(data, key);\n\t}\n\n\t_hashHas = hashHas;\n\treturn _hashHas;\n}\n\nvar _hashSet;\nvar hasRequired_hashSet;\n\nfunction require_hashSet () {\n\tif (hasRequired_hashSet) return _hashSet;\n\thasRequired_hashSet = 1;\n\tvar nativeCreate = require_nativeCreate();\n\n\t/** Used to stand-in for `undefined` hash values. */\n\tvar HASH_UNDEFINED = '__lodash_hash_undefined__';\n\n\t/**\n\t * Sets the hash `key` to `value`.\n\t *\n\t * @private\n\t * @name set\n\t * @memberOf Hash\n\t * @param {string} key The key of the value to set.\n\t * @param {*} value The value to set.\n\t * @returns {Object} Returns the hash instance.\n\t */\n\tfunction hashSet(key, value) {\n\t var data = this.__data__;\n\t this.size += this.has(key) ? 0 : 1;\n\t data[key] = (nativeCreate && value === undefined) ? HASH_UNDEFINED : value;\n\t return this;\n\t}\n\n\t_hashSet = hashSet;\n\treturn _hashSet;\n}\n\nvar _Hash;\nvar hasRequired_Hash;\n\nfunction require_Hash () {\n\tif (hasRequired_Hash) return _Hash;\n\thasRequired_Hash = 1;\n\tvar hashClear = require_hashClear(),\n\t hashDelete = require_hashDelete(),\n\t hashGet = require_hashGet(),\n\t hashHas = require_hashHas(),\n\t hashSet = require_hashSet();\n\n\t/**\n\t * Creates a hash object.\n\t *\n\t * @private\n\t * @constructor\n\t * @param {Array} [entries] The key-value pairs to cache.\n\t */\n\tfunction Hash(entries) {\n\t var index = -1,\n\t length = entries == null ? 0 : entries.length;\n\n\t this.clear();\n\t while (++index < length) {\n\t var entry = entries[index];\n\t this.set(entry[0], entry[1]);\n\t }\n\t}\n\n\t// Add methods to `Hash`.\n\tHash.prototype.clear = hashClear;\n\tHash.prototype['delete'] = hashDelete;\n\tHash.prototype.get = hashGet;\n\tHash.prototype.has = hashHas;\n\tHash.prototype.set = hashSet;\n\n\t_Hash = Hash;\n\treturn _Hash;\n}\n\n/**\n * Removes all key-value entries from the list cache.\n *\n * @private\n * @name clear\n * @memberOf ListCache\n */\n\nvar _listCacheClear;\nvar hasRequired_listCacheClear;\n\nfunction require_listCacheClear () {\n\tif (hasRequired_listCacheClear) return _listCacheClear;\n\thasRequired_listCacheClear = 1;\n\tfunction listCacheClear() {\n\t this.__data__ = [];\n\t this.size = 0;\n\t}\n\n\t_listCacheClear = listCacheClear;\n\treturn _listCacheClear;\n}\n\n/**\n * Performs a\n * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero)\n * comparison between two values to determine if they are equivalent.\n *\n * @static\n * @memberOf _\n * @since 4.0.0\n * @category Lang\n * @param {*} value The value to compare.\n * @param {*} other The other value to compare.\n * @returns {boolean} Returns `true` if the values are equivalent, else `false`.\n * @example\n *\n * var object = { 'a': 1 };\n * var other = { 'a': 1 };\n *\n * _.eq(object, object);\n * // => true\n *\n * _.eq(object, other);\n * // => false\n *\n * _.eq('a', 'a');\n * // => true\n *\n * _.eq('a', Object('a'));\n * // => false\n *\n * _.eq(NaN, NaN);\n * // => true\n */\n\nvar eq_1;\nvar hasRequiredEq;\n\nfunction requireEq () {\n\tif (hasRequiredEq) return eq_1;\n\thasRequiredEq = 1;\n\tfunction eq(value, other) {\n\t return value === other || (value !== value && other !== other);\n\t}\n\n\teq_1 = eq;\n\treturn eq_1;\n}\n\nvar _assocIndexOf;\nvar hasRequired_assocIndexOf;\n\nfunction require_assocIndexOf () {\n\tif (hasRequired_assocIndexOf) return _assocIndexOf;\n\thasRequired_assocIndexOf = 1;\n\tvar eq = requireEq();\n\n\t/**\n\t * Gets the index at which the `key` is found in `array` of key-value pairs.\n\t *\n\t * @private\n\t * @param {Array} array The array to inspect.\n\t * @param {*} key The key to search for.\n\t * @returns {number} Returns the index of the matched value, else `-1`.\n\t */\n\tfunction assocIndexOf(array, key) {\n\t var length = array.length;\n\t while (length--) {\n\t if (eq(array[length][0], key)) {\n\t return length;\n\t }\n\t }\n\t return -1;\n\t}\n\n\t_assocIndexOf = assocIndexOf;\n\treturn _assocIndexOf;\n}\n\nvar _listCacheDelete;\nvar hasRequired_listCacheDelete;\n\nfunction require_listCacheDelete () {\n\tif (hasRequired_listCacheDelete) return _listCacheDelete;\n\thasRequired_listCacheDelete = 1;\n\tvar assocIndexOf = require_assocIndexOf();\n\n\t/** Used for built-in method references. */\n\tvar arrayProto = Array.prototype;\n\n\t/** Built-in value references. */\n\tvar splice = arrayProto.splice;\n\n\t/**\n\t * Removes `key` and its value from the list cache.\n\t *\n\t * @private\n\t * @name delete\n\t * @memberOf ListCache\n\t * @param {string} key The key of the value to remove.\n\t * @returns {boolean} Returns `true` if the entry was removed, else `false`.\n\t */\n\tfunction listCacheDelete(key) {\n\t var data = this.__data__,\n\t index = assocIndexOf(data, key);\n\n\t if (index < 0) {\n\t return false;\n\t }\n\t var lastIndex = data.length - 1;\n\t if (index == lastIndex) {\n\t data.pop();\n\t } else {\n\t splice.call(data, index, 1);\n\t }\n\t --this.size;\n\t return true;\n\t}\n\n\t_listCacheDelete = listCacheDelete;\n\treturn _listCacheDelete;\n}\n\nvar _listCacheGet;\nvar hasRequired_listCacheGet;\n\nfunction require_listCacheGet () {\n\tif (hasRequired_listCacheGet) return _listCacheGet;\n\thasRequired_listCacheGet = 1;\n\tvar assocIndexOf = require_assocIndexOf();\n\n\t/**\n\t * Gets the list cache value for `key`.\n\t *\n\t * @private\n\t * @name get\n\t * @memberOf ListCache\n\t * @param {string} key The key of the value to get.\n\t * @returns {*} Returns the entry value.\n\t */\n\tfunction listCacheGet(key) {\n\t var data = this.__data__,\n\t index = assocIndexOf(data, key);\n\n\t return index < 0 ? undefined : data[index][1];\n\t}\n\n\t_listCacheGet = listCacheGet;\n\treturn _listCacheGet;\n}\n\nvar _listCacheHas;\nvar hasRequired_listCacheHas;\n\nfunction require_listCacheHas () {\n\tif (hasRequired_listCacheHas) return _listCacheHas;\n\thasRequired_listCacheHas = 1;\n\tvar assocIndexOf = require_assocIndexOf();\n\n\t/**\n\t * Checks if a list cache value for `key` exists.\n\t *\n\t * @private\n\t * @name has\n\t * @memberOf ListCache\n\t * @param {string} key The key of the entry to check.\n\t * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`.\n\t */\n\tfunction listCacheHas(key) {\n\t return assocIndexOf(this.__data__, key) > -1;\n\t}\n\n\t_listCacheHas = listCacheHas;\n\treturn _listCacheHas;\n}\n\nvar _listCacheSet;\nvar hasRequired_listCacheSet;\n\nfunction require_listCacheSet () {\n\tif (hasRequired_listCacheSet) return _listCacheSet;\n\thasRequired_listCacheSet = 1;\n\tvar assocIndexOf = require_assocIndexOf();\n\n\t/**\n\t * Sets the list cache `key` to `value`.\n\t *\n\t * @private\n\t * @name set\n\t * @memberOf ListCache\n\t * @param {string} key The key of the value to set.\n\t * @param {*} value The value to set.\n\t * @returns {Object} Returns the list cache instance.\n\t */\n\tfunction listCacheSet(key, value) {\n\t var data = this.__data__,\n\t index = assocIndexOf(data, key);\n\n\t if (index < 0) {\n\t ++this.size;\n\t data.push([key, value]);\n\t } else {\n\t data[index][1] = value;\n\t }\n\t return this;\n\t}\n\n\t_listCacheSet = listCacheSet;\n\treturn _listCacheSet;\n}\n\nvar _ListCache;\nvar hasRequired_ListCache;\n\nfunction require_ListCache () {\n\tif (hasRequired_ListCache) return _ListCache;\n\thasRequired_ListCache = 1;\n\tvar listCacheClear = require_listCacheClear(),\n\t listCacheDelete = require_listCacheDelete(),\n\t listCacheGet = require_listCacheGet(),\n\t listCacheHas = require_listCacheHas(),\n\t listCacheSet = require_listCacheSet();\n\n\t/**\n\t * Creates an list cache object.\n\t *\n\t * @private\n\t * @constructor\n\t * @param {Array} [entries] The key-value pairs to cache.\n\t */\n\tfunction ListCache(entries) {\n\t var index = -1,\n\t length = entries == null ? 0 : entries.length;\n\n\t this.clear();\n\t while (++index < length) {\n\t var entry = entries[index];\n\t this.set(entry[0], entry[1]);\n\t }\n\t}\n\n\t// Add methods to `ListCache`.\n\tListCache.prototype.clear = listCacheClear;\n\tListCache.prototype['delete'] = listCacheDelete;\n\tListCache.prototype.get = listCacheGet;\n\tListCache.prototype.has = listCacheHas;\n\tListCache.prototype.set = listCacheSet;\n\n\t_ListCache = ListCache;\n\treturn _ListCache;\n}\n\nvar _Map;\nvar hasRequired_Map;\n\nfunction require_Map () {\n\tif (hasRequired_Map) return _Map;\n\thasRequired_Map = 1;\n\tvar getNative = require_getNative(),\n\t root = require_root();\n\n\t/* Built-in method references that are verified to be native. */\n\tvar Map = getNative(root, 'Map');\n\n\t_Map = Map;\n\treturn _Map;\n}\n\nvar _mapCacheClear;\nvar hasRequired_mapCacheClear;\n\nfunction require_mapCacheClear () {\n\tif (hasRequired_mapCacheClear) return _mapCacheClear;\n\thasRequired_mapCacheClear = 1;\n\tvar Hash = require_Hash(),\n\t ListCache = require_ListCache(),\n\t Map = require_Map();\n\n\t/**\n\t * Removes all key-value entries from the map.\n\t *\n\t * @private\n\t * @name clear\n\t * @memberOf MapCache\n\t */\n\tfunction mapCacheClear() {\n\t this.size = 0;\n\t this.__data__ = {\n\t 'hash': new Hash,\n\t 'map': new (Map || ListCache),\n\t 'string': new Hash\n\t };\n\t}\n\n\t_mapCacheClear = mapCacheClear;\n\treturn _mapCacheClear;\n}\n\n/**\n * Checks if `value` is suitable for use as unique object key.\n *\n * @private\n * @param {*} value The value to check.\n * @returns {boolean} Returns `true` if `value` is suitable, else `false`.\n */\n\nvar _isKeyable;\nvar hasRequired_isKeyable;\n\nfunction require_isKeyable () {\n\tif (hasRequired_isKeyable) return _isKeyable;\n\thasRequired_isKeyable = 1;\n\tfunction isKeyable(value) {\n\t var type = typeof value;\n\t return (type == 'string' || type == 'number' || type == 'symbol' || type == 'boolean')\n\t ? (value !== '__proto__')\n\t : (value === null);\n\t}\n\n\t_isKeyable = isKeyable;\n\treturn _isKeyable;\n}\n\nvar _getMapData;\nvar hasRequired_getMapData;\n\nfunction require_getMapData () {\n\tif (hasRequired_getMapData) return _getMapData;\n\thasRequired_getMapData = 1;\n\tvar isKeyable = require_isKeyable();\n\n\t/**\n\t * Gets the data for `map`.\n\t *\n\t * @private\n\t * @param {Object} map The map to query.\n\t * @param {string} key The reference key.\n\t * @returns {*} Returns the map data.\n\t */\n\tfunction getMapData(map, key) {\n\t var data = map.__data__;\n\t return isKeyable(key)\n\t ? data[typeof key == 'string' ? 'string' : 'hash']\n\t : data.map;\n\t}\n\n\t_getMapData = getMapData;\n\treturn _getMapData;\n}\n\nvar _mapCacheDelete;\nvar hasRequired_mapCacheDelete;\n\nfunction require_mapCacheDelete () {\n\tif (hasRequired_mapCacheDelete) return _mapCacheDelete;\n\thasRequired_mapCacheDelete = 1;\n\tvar getMapData = require_getMapData();\n\n\t/**\n\t * Removes `key` and its value from the map.\n\t *\n\t * @private\n\t * @name delete\n\t * @memberOf MapCache\n\t * @param {string} key The key of the value to remove.\n\t * @returns {boolean} Returns `true` if the entry was removed, else `false`.\n\t */\n\tfunction mapCacheDelete(key) {\n\t var result = getMapData(this, key)['delete'](key);\n\t this.size -= result ? 1 : 0;\n\t return result;\n\t}\n\n\t_mapCacheDelete = mapCacheDelete;\n\treturn _mapCacheDelete;\n}\n\nvar _mapCacheGet;\nvar hasRequired_mapCacheGet;\n\nfunction require_mapCacheGet () {\n\tif (hasRequired_mapCacheGet) return _mapCacheGet;\n\thasRequired_mapCacheGet = 1;\n\tvar getMapData = require_getMapData();\n\n\t/**\n\t * Gets the map value for `key`.\n\t *\n\t * @private\n\t * @name get\n\t * @memberOf MapCache\n\t * @param {string} key The key of the value to get.\n\t * @returns {*} Returns the entry value.\n\t */\n\tfunction mapCacheGet(key) {\n\t return getMapData(this, key).get(key);\n\t}\n\n\t_mapCacheGet = mapCacheGet;\n\treturn _mapCacheGet;\n}\n\nvar _mapCacheHas;\nvar hasRequired_mapCacheHas;\n\nfunction require_mapCacheHas () {\n\tif (hasRequired_mapCacheHas) return _mapCacheHas;\n\thasRequired_mapCacheHas = 1;\n\tvar getMapData = require_getMapData();\n\n\t/**\n\t * Checks if a map value for `key` exists.\n\t *\n\t * @private\n\t * @name has\n\t * @memberOf MapCache\n\t * @param {string} key The key of the entry to check.\n\t * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`.\n\t */\n\tfunction mapCacheHas(key) {\n\t return getMapData(this, key).has(key);\n\t}\n\n\t_mapCacheHas = mapCacheHas;\n\treturn _mapCacheHas;\n}\n\nvar _mapCacheSet;\nvar hasRequired_mapCacheSet;\n\nfunction require_mapCacheSet () {\n\tif (hasRequired_mapCacheSet) return _mapCacheSet;\n\thasRequired_mapCacheSet = 1;\n\tvar getMapData = require_getMapData();\n\n\t/**\n\t * Sets the map `key` to `value`.\n\t *\n\t * @private\n\t * @name set\n\t * @memberOf MapCache\n\t * @param {string} key The key of the value to set.\n\t * @param {*} value The value to set.\n\t * @returns {Object} Returns the map cache instance.\n\t */\n\tfunction mapCacheSet(key, value) {\n\t var data = getMapData(this, key),\n\t size = data.size;\n\n\t data.set(key, value);\n\t this.size += data.size == size ? 0 : 1;\n\t return this;\n\t}\n\n\t_mapCacheSet = mapCacheSet;\n\treturn _mapCacheSet;\n}\n\nvar _MapCache;\nvar hasRequired_MapCache;\n\nfunction require_MapCache () {\n\tif (hasRequired_MapCache) return _MapCache;\n\thasRequired_MapCache = 1;\n\tvar mapCacheClear = require_mapCacheClear(),\n\t mapCacheDelete = require_mapCacheDelete(),\n\t mapCacheGet = require_mapCacheGet(),\n\t mapCacheHas = require_mapCacheHas(),\n\t mapCacheSet = require_mapCacheSet();\n\n\t/**\n\t * Creates a map cache object to store key-value pairs.\n\t *\n\t * @private\n\t * @constructor\n\t * @param {Array} [entries] The key-value pairs to cache.\n\t */\n\tfunction MapCache(entries) {\n\t var index = -1,\n\t length = entries == null ? 0 : entries.length;\n\n\t this.clear();\n\t while (++index < length) {\n\t var entry = entries[index];\n\t this.set(entry[0], entry[1]);\n\t }\n\t}\n\n\t// Add methods to `MapCache`.\n\tMapCache.prototype.clear = mapCacheClear;\n\tMapCache.prototype['delete'] = mapCacheDelete;\n\tMapCache.prototype.get = mapCacheGet;\n\tMapCache.prototype.has = mapCacheHas;\n\tMapCache.prototype.set = mapCacheSet;\n\n\t_MapCache = MapCache;\n\treturn _MapCache;\n}\n\nvar memoize_1;\nvar hasRequiredMemoize;\n\nfunction requireMemoize () {\n\tif (hasRequiredMemoize) return memoize_1;\n\thasRequiredMemoize = 1;\n\tvar MapCache = require_MapCache();\n\n\t/** Error message constants. */\n\tvar FUNC_ERROR_TEXT = 'Expected a function';\n\n\t/**\n\t * Creates a function that memoizes the result of `func`. If `resolver` is\n\t * provided, it determines the cache key for storing the result based on the\n\t * arguments provided to the memoized function. By default, the first argument\n\t * provided to the memoized function is used as the map cache key. The `func`\n\t * is invoked with the `this` binding of the memoized function.\n\t *\n\t * **Note:** The cache is exposed as the `cache` property on the memoized\n\t * function. Its creation may be customized by replacing the `_.memoize.Cache`\n\t * constructor with one whose instances implement the\n\t * [`Map`](http://ecma-international.org/ecma-262/7.0/#sec-properties-of-the-map-prototype-object)\n\t * method interface of `clear`, `delete`, `get`, `has`, and `set`.\n\t *\n\t * @static\n\t * @memberOf _\n\t * @since 0.1.0\n\t * @category Function\n\t * @param {Function} func The function to have its output memoized.\n\t * @param {Function} [resolver] The function to resolve the cache key.\n\t * @returns {Function} Returns the new memoized function.\n\t * @example\n\t *\n\t * var object = { 'a': 1, 'b': 2 };\n\t * var other = { 'c': 3, 'd': 4 };\n\t *\n\t * var values = _.memoize(_.values);\n\t * values(object);\n\t * // => [1, 2]\n\t *\n\t * values(other);\n\t * // => [3, 4]\n\t *\n\t * object.a = 2;\n\t * values(object);\n\t * // => [1, 2]\n\t *\n\t * // Modify the result cache.\n\t * values.cache.set(object, ['a', 'b']);\n\t * values(object);\n\t * // => ['a', 'b']\n\t *\n\t * // Replace `_.memoize.Cache`.\n\t * _.memoize.Cache = WeakMap;\n\t */\n\tfunction memoize(func, resolver) {\n\t if (typeof func != 'function' || (resolver != null && typeof resolver != 'function')) {\n\t throw new TypeError(FUNC_ERROR_TEXT);\n\t }\n\t var memoized = function() {\n\t var args = arguments,\n\t key = resolver ? resolver.apply(this, args) : args[0],\n\t cache = memoized.cache;\n\n\t if (cache.has(key)) {\n\t return cache.get(key);\n\t }\n\t var result = func.apply(this, args);\n\t memoized.cache = cache.set(key, result) || cache;\n\t return result;\n\t };\n\t memoized.cache = new (memoize.Cache || MapCache);\n\t return memoized;\n\t}\n\n\t// Expose `MapCache`.\n\tmemoize.Cache = MapCache;\n\n\tmemoize_1 = memoize;\n\treturn memoize_1;\n}\n\nvar _memoizeCapped;\nvar hasRequired_memoizeCapped;\n\nfunction require_memoizeCapped () {\n\tif (hasRequired_memoizeCapped) return _memoizeCapped;\n\thasRequired_memoizeCapped = 1;\n\tvar memoize = requireMemoize();\n\n\t/** Used as the maximum memoize cache size. */\n\tvar MAX_MEMOIZE_SIZE = 500;\n\n\t/**\n\t * A specialized version of `_.memoize` which clears the memoized function's\n\t * cache when it exceeds `MAX_MEMOIZE_SIZE`.\n\t *\n\t * @private\n\t * @param {Function} func The function to have its output memoized.\n\t * @returns {Function} Returns the new memoized function.\n\t */\n\tfunction memoizeCapped(func) {\n\t var result = memoize(func, function(key) {\n\t if (cache.size === MAX_MEMOIZE_SIZE) {\n\t cache.clear();\n\t }\n\t return key;\n\t });\n\n\t var cache = result.cache;\n\t return result;\n\t}\n\n\t_memoizeCapped = memoizeCapped;\n\treturn _memoizeCapped;\n}\n\nvar _stringToPath;\nvar hasRequired_stringToPath;\n\nfunction require_stringToPath () {\n\tif (hasRequired_stringToPath) return _stringToPath;\n\thasRequired_stringToPath = 1;\n\tvar memoizeCapped = require_memoizeCapped();\n\n\t/** Used to match property names within property paths. */\n\tvar rePropName = /[^.[\\]]+|\\[(?:(-?\\d+(?:\\.\\d+)?)|([\"'])((?:(?!\\2)[^\\\\]|\\\\.)*?)\\2)\\]|(?=(?:\\.|\\[\\])(?:\\.|\\[\\]|$))/g;\n\n\t/** Used to match backslashes in property paths. */\n\tvar reEscapeChar = /\\\\(\\\\)?/g;\n\n\t/**\n\t * Converts `string` to a property path array.\n\t *\n\t * @private\n\t * @param {string} string The string to convert.\n\t * @returns {Array} Returns the property path array.\n\t */\n\tvar stringToPath = memoizeCapped(function(string) {\n\t var result = [];\n\t if (string.charCodeAt(0) === 46 /* . */) {\n\t result.push('');\n\t }\n\t string.replace(rePropName, function(match, number, quote, subString) {\n\t result.push(quote ? subString.replace(reEscapeChar, '$1') : (number || match));\n\t });\n\t return result;\n\t});\n\n\t_stringToPath = stringToPath;\n\treturn _stringToPath;\n}\n\n/**\n * A specialized version of `_.map` for arrays without support for iteratee\n * shorthands.\n *\n * @private\n * @param {Array} [array] The array to iterate over.\n * @param {Function} iteratee The function invoked per iteration.\n * @returns {Array} Returns the new mapped array.\n */\n\nvar _arrayMap;\nvar hasRequired_arrayMap;\n\nfunction require_arrayMap () {\n\tif (hasRequired_arrayMap) return _arrayMap;\n\thasRequired_arrayMap = 1;\n\tfunction arrayMap(array, iteratee) {\n\t var index = -1,\n\t length = array == null ? 0 : array.length,\n\t result = Array(length);\n\n\t while (++index < length) {\n\t result[index] = iteratee(array[index], index, array);\n\t }\n\t return result;\n\t}\n\n\t_arrayMap = arrayMap;\n\treturn _arrayMap;\n}\n\nvar _baseToString;\nvar hasRequired_baseToString;\n\nfunction require_baseToString () {\n\tif (hasRequired_baseToString) return _baseToString;\n\thasRequired_baseToString = 1;\n\tvar Symbol = require_Symbol(),\n\t arrayMap = require_arrayMap(),\n\t isArray = requireIsArray(),\n\t isSymbol = requireIsSymbol();\n\n\t/** Used to convert symbols to primitives and strings. */\n\tvar symbolProto = Symbol ? Symbol.prototype : undefined,\n\t symbolToString = symbolProto ? symbolProto.toString : undefined;\n\n\t/**\n\t * The base implementation of `_.toString` which doesn't convert nullish\n\t * values to empty strings.\n\t *\n\t * @private\n\t * @param {*} value The value to process.\n\t * @returns {string} Returns the string.\n\t */\n\tfunction baseToString(value) {\n\t // Exit early for strings to avoid a performance hit in some environments.\n\t if (typeof value == 'string') {\n\t return value;\n\t }\n\t if (isArray(value)) {\n\t // Recursively convert values (susceptible to call stack limits).\n\t return arrayMap(value, baseToString) + '';\n\t }\n\t if (isSymbol(value)) {\n\t return symbolToString ? symbolToString.call(value) : '';\n\t }\n\t var result = (value + '');\n\t return (result == '0' && (1 / value) == -Infinity) ? '-0' : result;\n\t}\n\n\t_baseToString = baseToString;\n\treturn _baseToString;\n}\n\nvar toString_1;\nvar hasRequiredToString;\n\nfunction requireToString () {\n\tif (hasRequiredToString) return toString_1;\n\thasRequiredToString = 1;\n\tvar baseToString = require_baseToString();\n\n\t/**\n\t * Converts `value` to a string. An empty string is returned for `null`\n\t * and `undefined` values. The sign of `-0` is preserved.\n\t *\n\t * @static\n\t * @memberOf _\n\t * @since 4.0.0\n\t * @category Lang\n\t * @param {*} value The value to convert.\n\t * @returns {string} Returns the converted string.\n\t * @example\n\t *\n\t * _.toString(null);\n\t * // => ''\n\t *\n\t * _.toString(-0);\n\t * // => '-0'\n\t *\n\t * _.toString([1, 2, 3]);\n\t * // => '1,2,3'\n\t */\n\tfunction toString(value) {\n\t return value == null ? '' : baseToString(value);\n\t}\n\n\ttoString_1 = toString;\n\treturn toString_1;\n}\n\nvar _castPath;\nvar hasRequired_castPath;\n\nfunction require_castPath () {\n\tif (hasRequired_castPath) return _castPath;\n\thasRequired_castPath = 1;\n\tvar isArray = requireIsArray(),\n\t isKey = require_isKey(),\n\t stringToPath = require_stringToPath(),\n\t toString = requireToString();\n\n\t/**\n\t * Casts `value` to a path array if it's not one.\n\t *\n\t * @private\n\t * @param {*} value The value to inspect.\n\t * @param {Object} [object] The object to query keys on.\n\t * @returns {Array} Returns the cast property path array.\n\t */\n\tfunction castPath(value, object) {\n\t if (isArray(value)) {\n\t return value;\n\t }\n\t return isKey(value, object) ? [value] : stringToPath(toString(value));\n\t}\n\n\t_castPath = castPath;\n\treturn _castPath;\n}\n\nvar _toKey;\nvar hasRequired_toKey;\n\nfunction require_toKey () {\n\tif (hasRequired_toKey) return _toKey;\n\thasRequired_toKey = 1;\n\tvar isSymbol = requireIsSymbol();\n\n\t/**\n\t * Converts `value` to a string key if it's not a string or symbol.\n\t *\n\t * @private\n\t * @param {*} value The value to inspect.\n\t * @returns {string|symbol} Returns the key.\n\t */\n\tfunction toKey(value) {\n\t if (typeof value == 'string' || isSymbol(value)) {\n\t return value;\n\t }\n\t var result = (value + '');\n\t return (result == '0' && (1 / value) == -Infinity) ? '-0' : result;\n\t}\n\n\t_toKey = toKey;\n\treturn _toKey;\n}\n\nvar _baseGet;\nvar hasRequired_baseGet;\n\nfunction require_baseGet () {\n\tif (hasRequired_baseGet) return _baseGet;\n\thasRequired_baseGet = 1;\n\tvar castPath = require_castPath(),\n\t toKey = require_toKey();\n\n\t/**\n\t * The base implementation of `_.get` without support for default values.\n\t *\n\t * @private\n\t * @param {Object} object The object to query.\n\t * @param {Array|string} path The path of the property to get.\n\t * @returns {*} Returns the resolved value.\n\t */\n\tfunction baseGet(object, path) {\n\t path = castPath(path, object);\n\n\t var index = 0,\n\t length = path.length;\n\n\t while (object != null && index < length) {\n\t object = object[toKey(path[index++])];\n\t }\n\t return (index && index == length) ? object : undefined;\n\t}\n\n\t_baseGet = baseGet;\n\treturn _baseGet;\n}\n\nvar get_1;\nvar hasRequiredGet;\n\nfunction requireGet () {\n\tif (hasRequiredGet) return get_1;\n\thasRequiredGet = 1;\n\tvar baseGet = require_baseGet();\n\n\t/**\n\t * Gets the value at `path` of `object`. If the resolved value is\n\t * `undefined`, the `defaultValue` is returned in its place.\n\t *\n\t * @static\n\t * @memberOf _\n\t * @since 3.7.0\n\t * @category Object\n\t * @param {Object} object The object to query.\n\t * @param {Array|string} path The path of the property to get.\n\t * @param {*} [defaultValue] The value returned for `undefined` resolved values.\n\t * @returns {*} Returns the resolved value.\n\t * @example\n\t *\n\t * var object = { 'a': [{ 'b': { 'c': 3 } }] };\n\t *\n\t * _.get(object, 'a[0].b.c');\n\t * // => 3\n\t *\n\t * _.get(object, ['a', '0', 'b', 'c']);\n\t * // => 3\n\t *\n\t * _.get(object, 'a.b.c', 'default');\n\t * // => 'default'\n\t */\n\tfunction get(object, path, defaultValue) {\n\t var result = object == null ? undefined : baseGet(object, path);\n\t return result === undefined ? defaultValue : result;\n\t}\n\n\tget_1 = get;\n\treturn get_1;\n}\n\nvar getExports = requireGet();\nvar get = /*@__PURE__*/getDefaultExportFromCjs(getExports);\n\nvar _defineProperty;\nvar hasRequired_defineProperty;\n\nfunction require_defineProperty () {\n\tif (hasRequired_defineProperty) return _defineProperty;\n\thasRequired_defineProperty = 1;\n\tvar getNative = require_getNative();\n\n\tvar defineProperty = (function() {\n\t try {\n\t var func = getNative(Object, 'defineProperty');\n\t func({}, '', {});\n\t return func;\n\t } catch (e) {}\n\t}());\n\n\t_defineProperty = defineProperty;\n\treturn _defineProperty;\n}\n\nvar _baseAssignValue;\nvar hasRequired_baseAssignValue;\n\nfunction require_baseAssignValue () {\n\tif (hasRequired_baseAssignValue) return _baseAssignValue;\n\thasRequired_baseAssignValue = 1;\n\tvar defineProperty = require_defineProperty();\n\n\t/**\n\t * The base implementation of `assignValue` and `assignMergeValue` without\n\t * value checks.\n\t *\n\t * @private\n\t * @param {Object} object The object to modify.\n\t * @param {string} key The key of the property to assign.\n\t * @param {*} value The value to assign.\n\t */\n\tfunction baseAssignValue(object, key, value) {\n\t if (key == '__proto__' && defineProperty) {\n\t defineProperty(object, key, {\n\t 'configurable': true,\n\t 'enumerable': true,\n\t 'value': value,\n\t 'writable': true\n\t });\n\t } else {\n\t object[key] = value;\n\t }\n\t}\n\n\t_baseAssignValue = baseAssignValue;\n\treturn _baseAssignValue;\n}\n\nvar _assignValue;\nvar hasRequired_assignValue;\n\nfunction require_assignValue () {\n\tif (hasRequired_assignValue) return _assignValue;\n\thasRequired_assignValue = 1;\n\tvar baseAssignValue = require_baseAssignValue(),\n\t eq = requireEq();\n\n\t/** Used for built-in method references. */\n\tvar objectProto = Object.prototype;\n\n\t/** Used to check objects for own properties. */\n\tvar hasOwnProperty = objectProto.hasOwnProperty;\n\n\t/**\n\t * Assigns `value` to `key` of `object` if the existing value is not equivalent\n\t * using [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero)\n\t * for equality comparisons.\n\t *\n\t * @private\n\t * @param {Object} object The object to modify.\n\t * @param {string} key The key of the property to assign.\n\t * @param {*} value The value to assign.\n\t */\n\tfunction assignValue(object, key, value) {\n\t var objValue = object[key];\n\t if (!(hasOwnProperty.call(object, key) && eq(objValue, value)) ||\n\t (value === undefined && !(key in object))) {\n\t baseAssignValue(object, key, value);\n\t }\n\t}\n\n\t_assignValue = assignValue;\n\treturn _assignValue;\n}\n\n/** Used as references for various `Number` constants. */\n\nvar _isIndex;\nvar hasRequired_isIndex;\n\nfunction require_isIndex () {\n\tif (hasRequired_isIndex) return _isIndex;\n\thasRequired_isIndex = 1;\n\tvar MAX_SAFE_INTEGER = 9007199254740991;\n\n\t/** Used to detect unsigned integer values. */\n\tvar reIsUint = /^(?:0|[1-9]\\d*)$/;\n\n\t/**\n\t * Checks if `value` is a valid array-like index.\n\t *\n\t * @private\n\t * @param {*} value The value to check.\n\t * @param {number} [length=MAX_SAFE_INTEGER] The upper bounds of a valid index.\n\t * @returns {boolean} Returns `true` if `value` is a valid index, else `false`.\n\t */\n\tfunction isIndex(value, length) {\n\t var type = typeof value;\n\t length = length == null ? MAX_SAFE_INTEGER : length;\n\n\t return !!length &&\n\t (type == 'number' ||\n\t (type != 'symbol' && reIsUint.test(value))) &&\n\t (value > -1 && value % 1 == 0 && value < length);\n\t}\n\n\t_isIndex = isIndex;\n\treturn _isIndex;\n}\n\nvar _baseSet;\nvar hasRequired_baseSet;\n\nfunction require_baseSet () {\n\tif (hasRequired_baseSet) return _baseSet;\n\thasRequired_baseSet = 1;\n\tvar assignValue = require_assignValue(),\n\t castPath = require_castPath(),\n\t isIndex = require_isIndex(),\n\t isObject = requireIsObject(),\n\t toKey = require_toKey();\n\n\t/**\n\t * The base implementation of `_.set`.\n\t *\n\t * @private\n\t * @param {Object} object The object to modify.\n\t * @param {Array|string} path The path of the property to set.\n\t * @param {*} value The value to set.\n\t * @param {Function} [customizer] The function to customize path creation.\n\t * @returns {Object} Returns `object`.\n\t */\n\tfunction baseSet(object, path, value, customizer) {\n\t if (!isObject(object)) {\n\t return object;\n\t }\n\t path = castPath(path, object);\n\n\t var index = -1,\n\t length = path.length,\n\t lastIndex = length - 1,\n\t nested = object;\n\n\t while (nested != null && ++index < length) {\n\t var key = toKey(path[index]),\n\t newValue = value;\n\n\t if (key === '__proto__' || key === 'constructor' || key === 'prototype') {\n\t return object;\n\t }\n\n\t if (index != lastIndex) {\n\t var objValue = nested[key];\n\t newValue = customizer ? customizer(objValue, key, nested) : undefined;\n\t if (newValue === undefined) {\n\t newValue = isObject(objValue)\n\t ? objValue\n\t : (isIndex(path[index + 1]) ? [] : {});\n\t }\n\t }\n\t assignValue(nested, key, newValue);\n\t nested = nested[key];\n\t }\n\t return object;\n\t}\n\n\t_baseSet = baseSet;\n\treturn _baseSet;\n}\n\nvar set_1;\nvar hasRequiredSet;\n\nfunction requireSet () {\n\tif (hasRequiredSet) return set_1;\n\thasRequiredSet = 1;\n\tvar baseSet = require_baseSet();\n\n\t/**\n\t * Sets the value at `path` of `object`. If a portion of `path` doesn't exist,\n\t * it's created. Arrays are created for missing index properties while objects\n\t * are created for all other missing properties. Use `_.setWith` to customize\n\t * `path` creation.\n\t *\n\t * **Note:** This method mutates `object`.\n\t *\n\t * @static\n\t * @memberOf _\n\t * @since 3.7.0\n\t * @category Object\n\t * @param {Object} object The object to modify.\n\t * @param {Array|string} path The path of the property to set.\n\t * @param {*} value The value to set.\n\t * @returns {Object} Returns `object`.\n\t * @example\n\t *\n\t * var object = { 'a': [{ 'b': { 'c': 3 } }] };\n\t *\n\t * _.set(object, 'a[0].b.c', 4);\n\t * console.log(object.a[0].b.c);\n\t * // => 4\n\t *\n\t * _.set(object, ['x', '0', 'y', 'z'], 5);\n\t * console.log(object.x[0].y.z);\n\t * // => 5\n\t */\n\tfunction set(object, path, value) {\n\t return object == null ? object : baseSet(object, path, value);\n\t}\n\n\tset_1 = set;\n\treturn set_1;\n}\n\nvar setExports = requireSet();\nvar set = /*@__PURE__*/getDefaultExportFromCjs(setExports);\n\n/**\n * Copies the values of `source` to `array`.\n *\n * @private\n * @param {Array} source The array to copy values from.\n * @param {Array} [array=[]] The array to copy values to.\n * @returns {Array} Returns `array`.\n */\n\nvar _copyArray;\nvar hasRequired_copyArray;\n\nfunction require_copyArray () {\n\tif (hasRequired_copyArray) return _copyArray;\n\thasRequired_copyArray = 1;\n\tfunction copyArray(source, array) {\n\t var index = -1,\n\t length = source.length;\n\n\t array || (array = Array(length));\n\t while (++index < length) {\n\t array[index] = source[index];\n\t }\n\t return array;\n\t}\n\n\t_copyArray = copyArray;\n\treturn _copyArray;\n}\n\nvar toPath_1;\nvar hasRequiredToPath;\n\nfunction requireToPath () {\n\tif (hasRequiredToPath) return toPath_1;\n\thasRequiredToPath = 1;\n\tvar arrayMap = require_arrayMap(),\n\t copyArray = require_copyArray(),\n\t isArray = requireIsArray(),\n\t isSymbol = requireIsSymbol(),\n\t stringToPath = require_stringToPath(),\n\t toKey = require_toKey(),\n\t toString = requireToString();\n\n\t/**\n\t * Converts `value` to a property path array.\n\t *\n\t * @static\n\t * @memberOf _\n\t * @since 4.0.0\n\t * @category Util\n\t * @param {*} value The value to convert.\n\t * @returns {Array} Returns the new property path array.\n\t * @example\n\t *\n\t * _.toPath('a.b.c');\n\t * // => ['a', 'b', 'c']\n\t *\n\t * _.toPath('a[0].b.c');\n\t * // => ['a', '0', 'b', 'c']\n\t */\n\tfunction toPath(value) {\n\t if (isArray(value)) {\n\t return arrayMap(value, toKey);\n\t }\n\t return isSymbol(value) ? [value] : copyArray(stringToPath(toString(value)));\n\t}\n\n\ttoPath_1 = toPath;\n\treturn toPath_1;\n}\n\nvar toPathExports = requireToPath();\nvar toPath = /*@__PURE__*/getDefaultExportFromCjs(toPathExports);\n\nvar define$2 = {\n // access data field\n data: function data(params) {\n var defaults = {\n field: 'data',\n bindingEvent: 'data',\n allowBinding: false,\n allowSetting: false,\n allowGetting: false,\n settingEvent: 'data',\n settingTriggersEvent: false,\n triggerFnName: 'trigger',\n immutableKeys: {},\n // key => true if immutable\n updateStyle: false,\n beforeGet: function beforeGet(self) {},\n beforeSet: function beforeSet(self, obj) {},\n onSet: function onSet(self) {},\n canSet: function canSet(self) {\n return true;\n }\n };\n params = extend({}, defaults, params);\n return function dataImpl(name, value) {\n var p = params;\n var self = this;\n var selfIsArrayLike = self.length !== undefined;\n var all = selfIsArrayLike ? self : [self]; // put in array if not array-like\n var single = selfIsArrayLike ? self[0] : self;\n\n // .data('foo', ...)\n if (string(name)) {\n // set or get property\n var isPathLike = name.indexOf('.') !== -1; // there might be a normal field with a dot \n var path = isPathLike && toPath(name);\n\n // .data('foo')\n if (p.allowGetting && value === undefined) {\n // get\n\n var ret;\n if (single) {\n p.beforeGet(single);\n\n // check if it's path and a field with the same name doesn't exist\n if (path && single._private[p.field][name] === undefined) {\n ret = get(single._private[p.field], path);\n } else {\n ret = single._private[p.field][name];\n }\n }\n return ret;\n\n // .data('foo', 'bar')\n } else if (p.allowSetting && value !== undefined) {\n // set\n var valid = !p.immutableKeys[name];\n if (valid) {\n var change = _defineProperty$1({}, name, value);\n p.beforeSet(self, change);\n for (var i = 0, l = all.length; i < l; i++) {\n var ele = all[i];\n if (p.canSet(ele)) {\n if (path && single._private[p.field][name] === undefined) {\n set(ele._private[p.field], path, value);\n } else {\n ele._private[p.field][name] = value;\n }\n }\n }\n\n // update mappers if asked\n if (p.updateStyle) {\n self.updateStyle();\n }\n\n // call onSet callback\n p.onSet(self);\n if (p.settingTriggersEvent) {\n self[p.triggerFnName](p.settingEvent);\n }\n }\n }\n\n // .data({ 'foo': 'bar' })\n } else if (p.allowSetting && plainObject(name)) {\n // extend\n var obj = name;\n var k, v;\n var keys = Object.keys(obj);\n p.beforeSet(self, obj);\n for (var _i = 0; _i < keys.length; _i++) {\n k = keys[_i];\n v = obj[k];\n var _valid = !p.immutableKeys[k];\n if (_valid) {\n for (var j = 0; j < all.length; j++) {\n var _ele = all[j];\n if (p.canSet(_ele)) {\n _ele._private[p.field][k] = v;\n }\n }\n }\n }\n\n // update mappers if asked\n if (p.updateStyle) {\n self.updateStyle();\n }\n\n // call onSet callback\n p.onSet(self);\n if (p.settingTriggersEvent) {\n self[p.triggerFnName](p.settingEvent);\n }\n\n // .data(function(){ ... })\n } else if (p.allowBinding && fn$6(name)) {\n // bind to event\n var fn = name;\n self.on(p.bindingEvent, fn);\n\n // .data()\n } else if (p.allowGetting && name === undefined) {\n // get whole object\n var _ret;\n if (single) {\n p.beforeGet(single);\n _ret = single._private[p.field];\n }\n return _ret;\n }\n return self; // maintain chainability\n }; // function\n },\n // data\n\n // remove data field\n removeData: function removeData(params) {\n var defaults = {\n field: 'data',\n event: 'data',\n triggerFnName: 'trigger',\n triggerEvent: false,\n immutableKeys: {} // key => true if immutable\n };\n params = extend({}, defaults, params);\n return function removeDataImpl(names) {\n var p = params;\n var self = this;\n var selfIsArrayLike = self.length !== undefined;\n var all = selfIsArrayLike ? self : [self]; // put in array if not array-like\n\n // .removeData('foo bar')\n if (string(names)) {\n // then get the list of keys, and delete them\n var keys = names.split(/\\s+/);\n var l = keys.length;\n for (var i = 0; i < l; i++) {\n // delete each non-empty key\n var key = keys[i];\n if (emptyString(key)) {\n continue;\n }\n var valid = !p.immutableKeys[key]; // not valid if immutable\n if (valid) {\n for (var i_a = 0, l_a = all.length; i_a < l_a; i_a++) {\n all[i_a]._private[p.field][key] = undefined;\n }\n }\n }\n if (p.triggerEvent) {\n self[p.triggerFnName](p.event);\n }\n\n // .removeData()\n } else if (names === undefined) {\n // then delete all keys\n\n for (var _i_a = 0, _l_a = all.length; _i_a < _l_a; _i_a++) {\n var _privateFields = all[_i_a]._private[p.field];\n var _keys = Object.keys(_privateFields);\n for (var _i2 = 0; _i2 < _keys.length; _i2++) {\n var _key = _keys[_i2];\n var validKeyToDelete = !p.immutableKeys[_key];\n if (validKeyToDelete) {\n _privateFields[_key] = undefined;\n }\n }\n }\n if (p.triggerEvent) {\n self[p.triggerFnName](p.event);\n }\n }\n return self; // maintain chaining\n }; // function\n } // removeData\n}; // define\n\nvar define$1 = {\n eventAliasesOn: function eventAliasesOn(proto) {\n var p = proto;\n p.addListener = p.listen = p.bind = p.on;\n p.unlisten = p.unbind = p.off = p.removeListener;\n p.trigger = p.emit;\n\n // this is just a wrapper alias of .on()\n p.pon = p.promiseOn = function (events, selector) {\n var self = this;\n var args = Array.prototype.slice.call(arguments, 0);\n return new Promise$1(function (resolve, reject) {\n var callback = function callback(e) {\n self.off.apply(self, offArgs);\n resolve(e);\n };\n var onArgs = args.concat([callback]);\n var offArgs = onArgs.concat([]);\n self.on.apply(self, onArgs);\n });\n };\n }\n}; // define\n\n// use this module to cherry pick functions into your prototype\n// (useful for functions shared between the core and collections, for example)\n\nvar define = {};\n[define$3, define$2, define$1].forEach(function (m) {\n extend(define, m);\n});\n\nvar elesfn$i = {\n animate: define.animate(),\n animation: define.animation(),\n animated: define.animated(),\n clearQueue: define.clearQueue(),\n delay: define.delay(),\n delayAnimation: define.delayAnimation(),\n stop: define.stop()\n};\n\nvar elesfn$h = {\n classes: function classes(_classes) {\n var self = this;\n if (_classes === undefined) {\n var ret = [];\n self[0]._private.classes.forEach(function (cls) {\n return ret.push(cls);\n });\n return ret;\n } else if (!array(_classes)) {\n // extract classes from string\n _classes = (_classes || '').match(/\\S+/g) || [];\n }\n var changed = [];\n var classesSet = new Set$1(_classes);\n\n // check and update each ele\n for (var j = 0; j < self.length; j++) {\n var ele = self[j];\n var _p = ele._private;\n var eleClasses = _p.classes;\n var changedEle = false;\n\n // check if ele has all of the passed classes\n for (var i = 0; i < _classes.length; i++) {\n var cls = _classes[i];\n var eleHasClass = eleClasses.has(cls);\n if (!eleHasClass) {\n changedEle = true;\n break;\n }\n }\n\n // check if ele has classes outside of those passed\n if (!changedEle) {\n changedEle = eleClasses.size !== _classes.length;\n }\n if (changedEle) {\n _p.classes = classesSet;\n changed.push(ele);\n }\n }\n\n // trigger update style on those eles that had class changes\n if (changed.length > 0) {\n this.spawn(changed).updateStyle().emit('class');\n }\n return self;\n },\n addClass: function addClass(classes) {\n return this.toggleClass(classes, true);\n },\n hasClass: function hasClass(className) {\n var ele = this[0];\n return ele != null && ele._private.classes.has(className);\n },\n toggleClass: function toggleClass(classes, toggle) {\n if (!array(classes)) {\n // extract classes from string\n classes = classes.match(/\\S+/g) || [];\n }\n var self = this;\n var toggleUndefd = toggle === undefined;\n var changed = []; // eles who had classes changed\n\n for (var i = 0, il = self.length; i < il; i++) {\n var ele = self[i];\n var eleClasses = ele._private.classes;\n var changedEle = false;\n for (var j = 0; j < classes.length; j++) {\n var cls = classes[j];\n var hasClass = eleClasses.has(cls);\n var changedNow = false;\n if (toggle || toggleUndefd && !hasClass) {\n eleClasses.add(cls);\n changedNow = true;\n } else if (!toggle || toggleUndefd && hasClass) {\n eleClasses[\"delete\"](cls);\n changedNow = true;\n }\n if (!changedEle && changedNow) {\n changed.push(ele);\n changedEle = true;\n }\n } // for j classes\n } // for i eles\n\n // trigger update style on those eles that had class changes\n if (changed.length > 0) {\n this.spawn(changed).updateStyle().emit('class');\n }\n return self;\n },\n removeClass: function removeClass(classes) {\n return this.toggleClass(classes, false);\n },\n flashClass: function flashClass(classes, duration) {\n var self = this;\n if (duration == null) {\n duration = 250;\n } else if (duration === 0) {\n return self; // nothing to do really\n }\n self.addClass(classes);\n setTimeout(function () {\n self.removeClass(classes);\n }, duration);\n return self;\n }\n};\nelesfn$h.className = elesfn$h.classNames = elesfn$h.classes;\n\n// tokens in the query language\nvar tokens = {\n metaChar: '[\\\\!\\\\\"\\\\#\\\\$\\\\%\\\\&\\\\\\'\\\\(\\\\)\\\\*\\\\+\\\\,\\\\.\\\\/\\\\:\\\\;\\\\<\\\\=\\\\>\\\\?\\\\@\\\\[\\\\]\\\\^\\\\`\\\\{\\\\|\\\\}\\\\~]',\n // chars we need to escape in let names, etc\n comparatorOp: '=|\\\\!=|>|>=|<|<=|\\\\$=|\\\\^=|\\\\*=',\n // binary comparison op (used in data selectors)\n boolOp: '\\\\?|\\\\!|\\\\^',\n // boolean (unary) operators (used in data selectors)\n string: '\"(?:\\\\\\\\\"|[^\"])*\"' + '|' + \"'(?:\\\\\\\\'|[^'])*'\",\n // string literals (used in data selectors) -- doublequotes | singlequotes\n number: number,\n // number literal (used in data selectors) --- e.g. 0.1234, 1234, 12e123\n meta: 'degree|indegree|outdegree',\n // allowed metadata fields (i.e. allowed functions to use from Collection)\n separator: '\\\\s*,\\\\s*',\n // queries are separated by commas, e.g. edge[foo = 'bar'], node.someClass\n descendant: '\\\\s+',\n child: '\\\\s+>\\\\s+',\n subject: '\\\\$',\n group: 'node|edge|\\\\*',\n directedEdge: '\\\\s+->\\\\s+',\n undirectedEdge: '\\\\s+<->\\\\s+'\n};\ntokens.variable = '(?:[\\\\w-.]|(?:\\\\\\\\' + tokens.metaChar + '))+'; // a variable name can have letters, numbers, dashes, and periods\ntokens.className = '(?:[\\\\w-]|(?:\\\\\\\\' + tokens.metaChar + '))+'; // a class name has the same rules as a variable except it can't have a '.' in the name\ntokens.value = tokens.string + '|' + tokens.number; // a value literal, either a string or number\ntokens.id = tokens.variable; // an element id (follows variable conventions)\n\n(function () {\n var ops, op, i;\n\n // add @ variants to comparatorOp\n ops = tokens.comparatorOp.split('|');\n for (i = 0; i < ops.length; i++) {\n op = ops[i];\n tokens.comparatorOp += '|@' + op;\n }\n\n // add ! variants to comparatorOp\n ops = tokens.comparatorOp.split('|');\n for (i = 0; i < ops.length; i++) {\n op = ops[i];\n if (op.indexOf('!') >= 0) {\n continue;\n } // skip ops that explicitly contain !\n if (op === '=') {\n continue;\n } // skip = b/c != is explicitly defined\n\n tokens.comparatorOp += '|\\\\!' + op;\n }\n})();\n\n/**\n * Make a new query object\n *\n * @prop type {Type} The type enum (int) of the query\n * @prop checks List of checks to make against an ele to test for a match\n */\nvar newQuery = function newQuery() {\n return {\n checks: []\n };\n};\n\n/**\n * A check type enum-like object. Uses integer values for fast match() lookup.\n * The ordering does not matter as long as the ints are unique.\n */\nvar Type = {\n /** E.g. node */\n GROUP: 0,\n /** A collection of elements */\n COLLECTION: 1,\n /** A filter(ele) function */\n FILTER: 2,\n /** E.g. [foo > 1] */\n DATA_COMPARE: 3,\n /** E.g. [foo] */\n DATA_EXIST: 4,\n /** E.g. [?foo] */\n DATA_BOOL: 5,\n /** E.g. [[degree > 2]] */\n META_COMPARE: 6,\n /** E.g. :selected */\n STATE: 7,\n /** E.g. #foo */\n ID: 8,\n /** E.g. .foo */\n CLASS: 9,\n /** E.g. #foo <-> #bar */\n UNDIRECTED_EDGE: 10,\n /** E.g. #foo -> #bar */\n DIRECTED_EDGE: 11,\n /** E.g. $#foo -> #bar */\n NODE_SOURCE: 12,\n /** E.g. #foo -> $#bar */\n NODE_TARGET: 13,\n /** E.g. $#foo <-> #bar */\n NODE_NEIGHBOR: 14,\n /** E.g. #foo > #bar */\n CHILD: 15,\n /** E.g. #foo #bar */\n DESCENDANT: 16,\n /** E.g. $#foo > #bar */\n PARENT: 17,\n /** E.g. $#foo #bar */\n ANCESTOR: 18,\n /** E.g. #foo > $bar > #baz */\n COMPOUND_SPLIT: 19,\n /** Always matches, useful placeholder for subject in `COMPOUND_SPLIT` */\n TRUE: 20\n};\n\nvar stateSelectors = [{\n selector: ':selected',\n matches: function matches(ele) {\n return ele.selected();\n }\n}, {\n selector: ':unselected',\n matches: function matches(ele) {\n return !ele.selected();\n }\n}, {\n selector: ':selectable',\n matches: function matches(ele) {\n return ele.selectable();\n }\n}, {\n selector: ':unselectable',\n matches: function matches(ele) {\n return !ele.selectable();\n }\n}, {\n selector: ':locked',\n matches: function matches(ele) {\n return ele.locked();\n }\n}, {\n selector: ':unlocked',\n matches: function matches(ele) {\n return !ele.locked();\n }\n}, {\n selector: ':visible',\n matches: function matches(ele) {\n return ele.visible();\n }\n}, {\n selector: ':hidden',\n matches: function matches(ele) {\n return !ele.visible();\n }\n}, {\n selector: ':transparent',\n matches: function matches(ele) {\n return ele.transparent();\n }\n}, {\n selector: ':grabbed',\n matches: function matches(ele) {\n return ele.grabbed();\n }\n}, {\n selector: ':free',\n matches: function matches(ele) {\n return !ele.grabbed();\n }\n}, {\n selector: ':removed',\n matches: function matches(ele) {\n return ele.removed();\n }\n}, {\n selector: ':inside',\n matches: function matches(ele) {\n return !ele.removed();\n }\n}, {\n selector: ':grabbable',\n matches: function matches(ele) {\n return ele.grabbable();\n }\n}, {\n selector: ':ungrabbable',\n matches: function matches(ele) {\n return !ele.grabbable();\n }\n}, {\n selector: ':animated',\n matches: function matches(ele) {\n return ele.animated();\n }\n}, {\n selector: ':unanimated',\n matches: function matches(ele) {\n return !ele.animated();\n }\n}, {\n selector: ':parent',\n matches: function matches(ele) {\n return ele.isParent();\n }\n}, {\n selector: ':childless',\n matches: function matches(ele) {\n return ele.isChildless();\n }\n}, {\n selector: ':child',\n matches: function matches(ele) {\n return ele.isChild();\n }\n}, {\n selector: ':orphan',\n matches: function matches(ele) {\n return ele.isOrphan();\n }\n}, {\n selector: ':nonorphan',\n matches: function matches(ele) {\n return ele.isChild();\n }\n}, {\n selector: ':compound',\n matches: function matches(ele) {\n if (ele.isNode()) {\n return ele.isParent();\n } else {\n return ele.source().isParent() || ele.target().isParent();\n }\n }\n}, {\n selector: ':loop',\n matches: function matches(ele) {\n return ele.isLoop();\n }\n}, {\n selector: ':simple',\n matches: function matches(ele) {\n return ele.isSimple();\n }\n}, {\n selector: ':active',\n matches: function matches(ele) {\n return ele.active();\n }\n}, {\n selector: ':inactive',\n matches: function matches(ele) {\n return !ele.active();\n }\n}, {\n selector: ':backgrounding',\n matches: function matches(ele) {\n return ele.backgrounding();\n }\n}, {\n selector: ':nonbackgrounding',\n matches: function matches(ele) {\n return !ele.backgrounding();\n }\n}].sort(function (a, b) {\n // n.b. selectors that are starting substrings of others must have the longer ones first\n return descending(a.selector, b.selector);\n});\nvar lookup = function () {\n var selToFn = {};\n var s;\n for (var i = 0; i < stateSelectors.length; i++) {\n s = stateSelectors[i];\n selToFn[s.selector] = s.matches;\n }\n return selToFn;\n}();\nvar stateSelectorMatches = function stateSelectorMatches(sel, ele) {\n return lookup[sel](ele);\n};\nvar stateSelectorRegex = '(' + stateSelectors.map(function (s) {\n return s.selector;\n}).join('|') + ')';\n\n// when a token like a variable has escaped meta characters, we need to clean the backslashes out\n// so that values get compared properly in Selector.filter()\nvar cleanMetaChars = function cleanMetaChars(str) {\n return str.replace(new RegExp('\\\\\\\\(' + tokens.metaChar + ')', 'g'), function (match, $1) {\n return $1;\n });\n};\nvar replaceLastQuery = function replaceLastQuery(selector, examiningQuery, replacementQuery) {\n selector[selector.length - 1] = replacementQuery;\n};\n\n// NOTE: add new expression syntax here to have it recognised by the parser;\n// - a query contains all adjacent (i.e. no separator in between) expressions;\n// - the current query is stored in selector[i]\n// - you need to check the query objects in match() for it actually filter properly, but that's pretty straight forward\nvar exprs = [{\n name: 'group',\n // just used for identifying when debugging\n query: true,\n regex: '(' + tokens.group + ')',\n populate: function populate(selector, query, _ref) {\n var _ref2 = _slicedToArray(_ref, 1),\n group = _ref2[0];\n query.checks.push({\n type: Type.GROUP,\n value: group === '*' ? group : group + 's'\n });\n }\n}, {\n name: 'state',\n query: true,\n regex: stateSelectorRegex,\n populate: function populate(selector, query, _ref3) {\n var _ref4 = _slicedToArray(_ref3, 1),\n state = _ref4[0];\n query.checks.push({\n type: Type.STATE,\n value: state\n });\n }\n}, {\n name: 'id',\n query: true,\n regex: '\\\\#(' + tokens.id + ')',\n populate: function populate(selector, query, _ref5) {\n var _ref6 = _slicedToArray(_ref5, 1),\n id = _ref6[0];\n query.checks.push({\n type: Type.ID,\n value: cleanMetaChars(id)\n });\n }\n}, {\n name: 'className',\n query: true,\n regex: '\\\\.(' + tokens.className + ')',\n populate: function populate(selector, query, _ref7) {\n var _ref8 = _slicedToArray(_ref7, 1),\n className = _ref8[0];\n query.checks.push({\n type: Type.CLASS,\n value: cleanMetaChars(className)\n });\n }\n}, {\n name: 'dataExists',\n query: true,\n regex: '\\\\[\\\\s*(' + tokens.variable + ')\\\\s*\\\\]',\n populate: function populate(selector, query, _ref9) {\n var _ref10 = _slicedToArray(_ref9, 1),\n variable = _ref10[0];\n query.checks.push({\n type: Type.DATA_EXIST,\n field: cleanMetaChars(variable)\n });\n }\n}, {\n name: 'dataCompare',\n query: true,\n regex: '\\\\[\\\\s*(' + tokens.variable + ')\\\\s*(' + tokens.comparatorOp + ')\\\\s*(' + tokens.value + ')\\\\s*\\\\]',\n populate: function populate(selector, query, _ref11) {\n var _ref12 = _slicedToArray(_ref11, 3),\n variable = _ref12[0],\n comparatorOp = _ref12[1],\n value = _ref12[2];\n var valueIsString = new RegExp('^' + tokens.string + '$').exec(value) != null;\n if (valueIsString) {\n value = value.substring(1, value.length - 1);\n } else {\n value = parseFloat(value);\n }\n query.checks.push({\n type: Type.DATA_COMPARE,\n field: cleanMetaChars(variable),\n operator: comparatorOp,\n value: value\n });\n }\n}, {\n name: 'dataBool',\n query: true,\n regex: '\\\\[\\\\s*(' + tokens.boolOp + ')\\\\s*(' + tokens.variable + ')\\\\s*\\\\]',\n populate: function populate(selector, query, _ref13) {\n var _ref14 = _slicedToArray(_ref13, 2),\n boolOp = _ref14[0],\n variable = _ref14[1];\n query.checks.push({\n type: Type.DATA_BOOL,\n field: cleanMetaChars(variable),\n operator: boolOp\n });\n }\n}, {\n name: 'metaCompare',\n query: true,\n regex: '\\\\[\\\\[\\\\s*(' + tokens.meta + ')\\\\s*(' + tokens.comparatorOp + ')\\\\s*(' + tokens.number + ')\\\\s*\\\\]\\\\]',\n populate: function populate(selector, query, _ref15) {\n var _ref16 = _slicedToArray(_ref15, 3),\n meta = _ref16[0],\n comparatorOp = _ref16[1],\n number = _ref16[2];\n query.checks.push({\n type: Type.META_COMPARE,\n field: cleanMetaChars(meta),\n operator: comparatorOp,\n value: parseFloat(number)\n });\n }\n}, {\n name: 'nextQuery',\n separator: true,\n regex: tokens.separator,\n populate: function populate(selector, query) {\n var currentSubject = selector.currentSubject;\n var edgeCount = selector.edgeCount;\n var compoundCount = selector.compoundCount;\n var lastQ = selector[selector.length - 1];\n if (currentSubject != null) {\n lastQ.subject = currentSubject;\n selector.currentSubject = null;\n }\n lastQ.edgeCount = edgeCount;\n lastQ.compoundCount = compoundCount;\n selector.edgeCount = 0;\n selector.compoundCount = 0;\n\n // go on to next query\n var nextQuery = selector[selector.length++] = newQuery();\n return nextQuery; // this is the new query to be filled by the following exprs\n }\n}, {\n name: 'directedEdge',\n separator: true,\n regex: tokens.directedEdge,\n populate: function populate(selector, query) {\n if (selector.currentSubject == null) {\n // undirected edge\n var edgeQuery = newQuery();\n var source = query;\n var target = newQuery();\n edgeQuery.checks.push({\n type: Type.DIRECTED_EDGE,\n source: source,\n target: target\n });\n\n // the query in the selector should be the edge rather than the source\n replaceLastQuery(selector, query, edgeQuery);\n selector.edgeCount++;\n\n // we're now populating the target query with expressions that follow\n return target;\n } else {\n // source/target\n var srcTgtQ = newQuery();\n var _source = query;\n var _target = newQuery();\n srcTgtQ.checks.push({\n type: Type.NODE_SOURCE,\n source: _source,\n target: _target\n });\n\n // the query in the selector should be the neighbourhood rather than the node\n replaceLastQuery(selector, query, srcTgtQ);\n selector.edgeCount++;\n return _target; // now populating the target with the following expressions\n }\n }\n}, {\n name: 'undirectedEdge',\n separator: true,\n regex: tokens.undirectedEdge,\n populate: function populate(selector, query) {\n if (selector.currentSubject == null) {\n // undirected edge\n var edgeQuery = newQuery();\n var source = query;\n var target = newQuery();\n edgeQuery.checks.push({\n type: Type.UNDIRECTED_EDGE,\n nodes: [source, target]\n });\n\n // the query in the selector should be the edge rather than the source\n replaceLastQuery(selector, query, edgeQuery);\n selector.edgeCount++;\n\n // we're now populating the target query with expressions that follow\n return target;\n } else {\n // neighbourhood\n var nhoodQ = newQuery();\n var node = query;\n var neighbor = newQuery();\n nhoodQ.checks.push({\n type: Type.NODE_NEIGHBOR,\n node: node,\n neighbor: neighbor\n });\n\n // the query in the selector should be the neighbourhood rather than the node\n replaceLastQuery(selector, query, nhoodQ);\n return neighbor; // now populating the neighbor with following expressions\n }\n }\n}, {\n name: 'child',\n separator: true,\n regex: tokens.child,\n populate: function populate(selector, query) {\n if (selector.currentSubject == null) {\n // default: child query\n var parentChildQuery = newQuery();\n var child = newQuery();\n var parent = selector[selector.length - 1];\n parentChildQuery.checks.push({\n type: Type.CHILD,\n parent: parent,\n child: child\n });\n\n // the query in the selector should be the '>' itself\n replaceLastQuery(selector, query, parentChildQuery);\n selector.compoundCount++;\n\n // we're now populating the child query with expressions that follow\n return child;\n } else if (selector.currentSubject === query) {\n // compound split query\n var compound = newQuery();\n var left = selector[selector.length - 1];\n var right = newQuery();\n var subject = newQuery();\n var _child = newQuery();\n var _parent = newQuery();\n\n // set up the root compound q\n compound.checks.push({\n type: Type.COMPOUND_SPLIT,\n left: left,\n right: right,\n subject: subject\n });\n\n // populate the subject and replace the q at the old spot (within left) with TRUE\n subject.checks = query.checks; // take the checks from the left\n query.checks = [{\n type: Type.TRUE\n }]; // checks under left refs the subject implicitly\n\n // set up the right q\n _parent.checks.push({\n type: Type.TRUE\n }); // parent implicitly refs the subject\n right.checks.push({\n type: Type.PARENT,\n // type is swapped on right side queries\n parent: _parent,\n child: _child // empty for now\n });\n replaceLastQuery(selector, left, compound);\n\n // update the ref since we moved things around for `query`\n selector.currentSubject = subject;\n selector.compoundCount++;\n return _child; // now populating the right side's child\n } else {\n // parent query\n // info for parent query\n var _parent2 = newQuery();\n var _child2 = newQuery();\n var pcQChecks = [{\n type: Type.PARENT,\n parent: _parent2,\n child: _child2\n }];\n\n // the parent-child query takes the place of the query previously being populated\n _parent2.checks = query.checks; // the previous query contains the checks for the parent\n query.checks = pcQChecks; // pc query takes over\n\n selector.compoundCount++;\n return _child2; // we're now populating the child\n }\n }\n}, {\n name: 'descendant',\n separator: true,\n regex: tokens.descendant,\n populate: function populate(selector, query) {\n if (selector.currentSubject == null) {\n // default: descendant query\n var ancChQuery = newQuery();\n var descendant = newQuery();\n var ancestor = selector[selector.length - 1];\n ancChQuery.checks.push({\n type: Type.DESCENDANT,\n ancestor: ancestor,\n descendant: descendant\n });\n\n // the query in the selector should be the '>' itself\n replaceLastQuery(selector, query, ancChQuery);\n selector.compoundCount++;\n\n // we're now populating the descendant query with expressions that follow\n return descendant;\n } else if (selector.currentSubject === query) {\n // compound split query\n var compound = newQuery();\n var left = selector[selector.length - 1];\n var right = newQuery();\n var subject = newQuery();\n var _descendant = newQuery();\n var _ancestor = newQuery();\n\n // set up the root compound q\n compound.checks.push({\n type: Type.COMPOUND_SPLIT,\n left: left,\n right: right,\n subject: subject\n });\n\n // populate the subject and replace the q at the old spot (within left) with TRUE\n subject.checks = query.checks; // take the checks from the left\n query.checks = [{\n type: Type.TRUE\n }]; // checks under left refs the subject implicitly\n\n // set up the right q\n _ancestor.checks.push({\n type: Type.TRUE\n }); // ancestor implicitly refs the subject\n right.checks.push({\n type: Type.ANCESTOR,\n // type is swapped on right side queries\n ancestor: _ancestor,\n descendant: _descendant // empty for now\n });\n replaceLastQuery(selector, left, compound);\n\n // update the ref since we moved things around for `query`\n selector.currentSubject = subject;\n selector.compoundCount++;\n return _descendant; // now populating the right side's descendant\n } else {\n // ancestor query\n // info for parent query\n var _ancestor2 = newQuery();\n var _descendant2 = newQuery();\n var adQChecks = [{\n type: Type.ANCESTOR,\n ancestor: _ancestor2,\n descendant: _descendant2\n }];\n\n // the parent-child query takes the place of the query previously being populated\n _ancestor2.checks = query.checks; // the previous query contains the checks for the parent\n query.checks = adQChecks; // pc query takes over\n\n selector.compoundCount++;\n return _descendant2; // we're now populating the child\n }\n }\n}, {\n name: 'subject',\n modifier: true,\n regex: tokens.subject,\n populate: function populate(selector, query) {\n if (selector.currentSubject != null && selector.currentSubject !== query) {\n warn('Redefinition of subject in selector `' + selector.toString() + '`');\n return false;\n }\n selector.currentSubject = query;\n var topQ = selector[selector.length - 1];\n var topChk = topQ.checks[0];\n var topType = topChk == null ? null : topChk.type;\n if (topType === Type.DIRECTED_EDGE) {\n // directed edge with subject on the target\n\n // change to target node check\n topChk.type = Type.NODE_TARGET;\n } else if (topType === Type.UNDIRECTED_EDGE) {\n // undirected edge with subject on the second node\n\n // change to neighbor check\n topChk.type = Type.NODE_NEIGHBOR;\n topChk.node = topChk.nodes[1]; // second node is subject\n topChk.neighbor = topChk.nodes[0];\n\n // clean up unused fields for new type\n topChk.nodes = null;\n }\n }\n}];\nexprs.forEach(function (e) {\n return e.regexObj = new RegExp('^' + e.regex);\n});\n\n/**\n * Of all the expressions, find the first match in the remaining text.\n * @param {string} remaining The remaining text to parse\n * @returns The matched expression and the newly remaining text `{ expr, match, name, remaining }`\n */\nvar consumeExpr = function consumeExpr(remaining) {\n var expr;\n var match;\n var name;\n for (var j = 0; j < exprs.length; j++) {\n var e = exprs[j];\n var n = e.name;\n var m = remaining.match(e.regexObj);\n if (m != null) {\n match = m;\n expr = e;\n name = n;\n var consumed = m[0];\n remaining = remaining.substring(consumed.length);\n break; // we've consumed one expr, so we can return now\n }\n }\n return {\n expr: expr,\n match: match,\n name: name,\n remaining: remaining\n };\n};\n\n/**\n * Consume all the leading whitespace\n * @param {string} remaining The text to consume\n * @returns The text with the leading whitespace removed\n */\nvar consumeWhitespace = function consumeWhitespace(remaining) {\n var match = remaining.match(/^\\s+/);\n if (match) {\n var consumed = match[0];\n remaining = remaining.substring(consumed.length);\n }\n return remaining;\n};\n\n/**\n * Parse the string and store the parsed representation in the Selector.\n * @param {string} selector The selector string\n * @returns `true` if the selector was successfully parsed, `false` otherwise\n */\nvar parse = function parse(selector) {\n var self = this;\n var remaining = self.inputText = selector;\n var currentQuery = self[0] = newQuery();\n self.length = 1;\n remaining = consumeWhitespace(remaining); // get rid of leading whitespace\n\n for (;;) {\n var exprInfo = consumeExpr(remaining);\n if (exprInfo.expr == null) {\n warn('The selector `' + selector + '`is invalid');\n return false;\n } else {\n var args = exprInfo.match.slice(1);\n\n // let the token populate the selector object in currentQuery\n var ret = exprInfo.expr.populate(self, currentQuery, args);\n if (ret === false) {\n return false; // exit if population failed\n } else if (ret != null) {\n currentQuery = ret; // change the current query to be filled if the expr specifies\n }\n }\n remaining = exprInfo.remaining;\n\n // we're done when there's nothing left to parse\n if (remaining.match(/^\\s*$/)) {\n break;\n }\n }\n var lastQ = self[self.length - 1];\n if (self.currentSubject != null) {\n lastQ.subject = self.currentSubject;\n }\n lastQ.edgeCount = self.edgeCount;\n lastQ.compoundCount = self.compoundCount;\n for (var i = 0; i < self.length; i++) {\n var q = self[i];\n\n // in future, this could potentially be allowed if there were operator precedence and detection of invalid combinations\n if (q.compoundCount > 0 && q.edgeCount > 0) {\n warn('The selector `' + selector + '` is invalid because it uses both a compound selector and an edge selector');\n return false;\n }\n if (q.edgeCount > 1) {\n warn('The selector `' + selector + '` is invalid because it uses multiple edge selectors');\n return false;\n } else if (q.edgeCount === 1) {\n warn('The selector `' + selector + '` is deprecated. Edge selectors do not take effect on changes to source and target nodes after an edge is added, for performance reasons. Use a class or data selector on edges instead, updating the class or data of an edge when your app detects a change in source or target nodes.');\n }\n }\n return true; // success\n};\n\n/**\n * Get the selector represented as a string. This value uses default formatting,\n * so things like spacing may differ from the input text passed to the constructor.\n * @returns {string} The selector string\n */\nvar toString = function toString() {\n if (this.toStringCache != null) {\n return this.toStringCache;\n }\n var clean = function clean(obj) {\n if (obj == null) {\n return '';\n } else {\n return obj;\n }\n };\n var cleanVal = function cleanVal(val) {\n if (string(val)) {\n return '\"' + val + '\"';\n } else {\n return clean(val);\n }\n };\n var space = function space(val) {\n return ' ' + val + ' ';\n };\n var checkToString = function checkToString(check, subject) {\n var type = check.type,\n value = check.value;\n switch (type) {\n case Type.GROUP:\n {\n var group = clean(value);\n return group.substring(0, group.length - 1);\n }\n case Type.DATA_COMPARE:\n {\n var field = check.field,\n operator = check.operator;\n return '[' + field + space(clean(operator)) + cleanVal(value) + ']';\n }\n case Type.DATA_BOOL:\n {\n var _operator = check.operator,\n _field = check.field;\n return '[' + clean(_operator) + _field + ']';\n }\n case Type.DATA_EXIST:\n {\n var _field2 = check.field;\n return '[' + _field2 + ']';\n }\n case Type.META_COMPARE:\n {\n var _operator2 = check.operator,\n _field3 = check.field;\n return '[[' + _field3 + space(clean(_operator2)) + cleanVal(value) + ']]';\n }\n case Type.STATE:\n {\n return value;\n }\n case Type.ID:\n {\n return '#' + value;\n }\n case Type.CLASS:\n {\n return '.' + value;\n }\n case Type.PARENT:\n case Type.CHILD:\n {\n return queryToString(check.parent, subject) + space('>') + queryToString(check.child, subject);\n }\n case Type.ANCESTOR:\n case Type.DESCENDANT:\n {\n return queryToString(check.ancestor, subject) + ' ' + queryToString(check.descendant, subject);\n }\n case Type.COMPOUND_SPLIT:\n {\n var lhs = queryToString(check.left, subject);\n var sub = queryToString(check.subject, subject);\n var rhs = queryToString(check.right, subject);\n return lhs + (lhs.length > 0 ? ' ' : '') + sub + rhs;\n }\n case Type.TRUE:\n {\n return '';\n }\n }\n };\n var queryToString = function queryToString(query, subject) {\n return query.checks.reduce(function (str, chk, i) {\n return str + (subject === query && i === 0 ? '$' : '') + checkToString(chk, subject);\n }, '');\n };\n var str = '';\n for (var i = 0; i < this.length; i++) {\n var query = this[i];\n str += queryToString(query, query.subject);\n if (this.length > 1 && i < this.length - 1) {\n str += ', ';\n }\n }\n this.toStringCache = str;\n return str;\n};\nvar parse$1 = {\n parse: parse,\n toString: toString\n};\n\nvar valCmp = function valCmp(fieldVal, operator, value) {\n var matches;\n var isFieldStr = string(fieldVal);\n var isFieldNum = number$1(fieldVal);\n var isValStr = string(value);\n var fieldStr, valStr;\n var caseInsensitive = false;\n var notExpr = false;\n var isIneqCmp = false;\n if (operator.indexOf('!') >= 0) {\n operator = operator.replace('!', '');\n notExpr = true;\n }\n if (operator.indexOf('@') >= 0) {\n operator = operator.replace('@', '');\n caseInsensitive = true;\n }\n if (isFieldStr || isValStr || caseInsensitive) {\n fieldStr = !isFieldStr && !isFieldNum ? '' : '' + fieldVal;\n valStr = '' + value;\n }\n\n // if we're doing a case insensitive comparison, then we're using a STRING comparison\n // even if we're comparing numbers\n if (caseInsensitive) {\n fieldVal = fieldStr = fieldStr.toLowerCase();\n value = valStr = valStr.toLowerCase();\n }\n switch (operator) {\n case '*=':\n matches = fieldStr.indexOf(valStr) >= 0;\n break;\n case '$=':\n matches = fieldStr.indexOf(valStr, fieldStr.length - valStr.length) >= 0;\n break;\n case '^=':\n matches = fieldStr.indexOf(valStr) === 0;\n break;\n case '=':\n matches = fieldVal === value;\n break;\n case '>':\n isIneqCmp = true;\n matches = fieldVal > value;\n break;\n case '>=':\n isIneqCmp = true;\n matches = fieldVal >= value;\n break;\n case '<':\n isIneqCmp = true;\n matches = fieldVal < value;\n break;\n case '<=':\n isIneqCmp = true;\n matches = fieldVal <= value;\n break;\n default:\n matches = false;\n break;\n }\n\n // apply the not op, but null vals for inequalities should always stay non-matching\n if (notExpr && (fieldVal != null || !isIneqCmp)) {\n matches = !matches;\n }\n return matches;\n};\nvar boolCmp = function boolCmp(fieldVal, operator) {\n switch (operator) {\n case '?':\n return fieldVal ? true : false;\n case '!':\n return fieldVal ? false : true;\n case '^':\n return fieldVal === undefined;\n }\n};\nvar existCmp = function existCmp(fieldVal) {\n return fieldVal !== undefined;\n};\nvar data$1 = function data(ele, field) {\n return ele.data(field);\n};\nvar meta = function meta(ele, field) {\n return ele[field]();\n};\n\n/** A lookup of `match(check, ele)` functions by `Type` int */\nvar match = [];\n\n/**\n * Returns whether the query matches for the element\n * @param query The `{ type, value, ... }` query object\n * @param ele The element to compare against\n*/\nvar matches$1 = function matches(query, ele) {\n return query.checks.every(function (chk) {\n return match[chk.type](chk, ele);\n });\n};\nmatch[Type.GROUP] = function (check, ele) {\n var group = check.value;\n return group === '*' || group === ele.group();\n};\nmatch[Type.STATE] = function (check, ele) {\n var stateSelector = check.value;\n return stateSelectorMatches(stateSelector, ele);\n};\nmatch[Type.ID] = function (check, ele) {\n var id = check.value;\n return ele.id() === id;\n};\nmatch[Type.CLASS] = function (check, ele) {\n var cls = check.value;\n return ele.hasClass(cls);\n};\nmatch[Type.META_COMPARE] = function (check, ele) {\n var field = check.field,\n operator = check.operator,\n value = check.value;\n return valCmp(meta(ele, field), operator, value);\n};\nmatch[Type.DATA_COMPARE] = function (check, ele) {\n var field = check.field,\n operator = check.operator,\n value = check.value;\n return valCmp(data$1(ele, field), operator, value);\n};\nmatch[Type.DATA_BOOL] = function (check, ele) {\n var field = check.field,\n operator = check.operator;\n return boolCmp(data$1(ele, field), operator);\n};\nmatch[Type.DATA_EXIST] = function (check, ele) {\n var field = check.field;\n check.operator;\n return existCmp(data$1(ele, field));\n};\nmatch[Type.UNDIRECTED_EDGE] = function (check, ele) {\n var qA = check.nodes[0];\n var qB = check.nodes[1];\n var src = ele.source();\n var tgt = ele.target();\n return matches$1(qA, src) && matches$1(qB, tgt) || matches$1(qB, src) && matches$1(qA, tgt);\n};\nmatch[Type.NODE_NEIGHBOR] = function (check, ele) {\n return matches$1(check.node, ele) && ele.neighborhood().some(function (n) {\n return n.isNode() && matches$1(check.neighbor, n);\n });\n};\nmatch[Type.DIRECTED_EDGE] = function (check, ele) {\n return matches$1(check.source, ele.source()) && matches$1(check.target, ele.target());\n};\nmatch[Type.NODE_SOURCE] = function (check, ele) {\n return matches$1(check.source, ele) && ele.outgoers().some(function (n) {\n return n.isNode() && matches$1(check.target, n);\n });\n};\nmatch[Type.NODE_TARGET] = function (check, ele) {\n return matches$1(check.target, ele) && ele.incomers().some(function (n) {\n return n.isNode() && matches$1(check.source, n);\n });\n};\nmatch[Type.CHILD] = function (check, ele) {\n return matches$1(check.child, ele) && matches$1(check.parent, ele.parent());\n};\nmatch[Type.PARENT] = function (check, ele) {\n return matches$1(check.parent, ele) && ele.children().some(function (c) {\n return matches$1(check.child, c);\n });\n};\nmatch[Type.DESCENDANT] = function (check, ele) {\n return matches$1(check.descendant, ele) && ele.ancestors().some(function (a) {\n return matches$1(check.ancestor, a);\n });\n};\nmatch[Type.ANCESTOR] = function (check, ele) {\n return matches$1(check.ancestor, ele) && ele.descendants().some(function (d) {\n return matches$1(check.descendant, d);\n });\n};\nmatch[Type.COMPOUND_SPLIT] = function (check, ele) {\n return matches$1(check.subject, ele) && matches$1(check.left, ele) && matches$1(check.right, ele);\n};\nmatch[Type.TRUE] = function () {\n return true;\n};\nmatch[Type.COLLECTION] = function (check, ele) {\n var collection = check.value;\n return collection.has(ele);\n};\nmatch[Type.FILTER] = function (check, ele) {\n var filter = check.value;\n return filter(ele);\n};\n\n// filter an existing collection\nvar filter = function filter(collection) {\n var self = this;\n\n // for 1 id #foo queries, just get the element\n if (self.length === 1 && self[0].checks.length === 1 && self[0].checks[0].type === Type.ID) {\n return collection.getElementById(self[0].checks[0].value).collection();\n }\n var selectorFunction = function selectorFunction(element) {\n for (var j = 0; j < self.length; j++) {\n var query = self[j];\n if (matches$1(query, element)) {\n return true;\n }\n }\n return false;\n };\n if (self.text() == null) {\n selectorFunction = function selectorFunction() {\n return true;\n };\n }\n return collection.filter(selectorFunction);\n}; // filter\n\n// does selector match a single element?\nvar matches = function matches(ele) {\n var self = this;\n for (var j = 0; j < self.length; j++) {\n var query = self[j];\n if (matches$1(query, ele)) {\n return true;\n }\n }\n return false;\n}; // matches\n\nvar matching = {\n matches: matches,\n filter: filter\n};\n\nvar Selector = function Selector(selector) {\n this.inputText = selector;\n this.currentSubject = null;\n this.compoundCount = 0;\n this.edgeCount = 0;\n this.length = 0;\n if (selector == null || string(selector) && selector.match(/^\\s*$/)) ; else if (elementOrCollection(selector)) {\n this.addQuery({\n checks: [{\n type: Type.COLLECTION,\n value: selector.collection()\n }]\n });\n } else if (fn$6(selector)) {\n this.addQuery({\n checks: [{\n type: Type.FILTER,\n value: selector\n }]\n });\n } else if (string(selector)) {\n if (!this.parse(selector)) {\n this.invalid = true;\n }\n } else {\n error('A selector must be created from a string; found ');\n }\n};\nvar selfn = Selector.prototype;\n[parse$1, matching].forEach(function (p) {\n return extend(selfn, p);\n});\nselfn.text = function () {\n return this.inputText;\n};\nselfn.size = function () {\n return this.length;\n};\nselfn.eq = function (i) {\n return this[i];\n};\nselfn.sameText = function (otherSel) {\n return !this.invalid && !otherSel.invalid && this.text() === otherSel.text();\n};\nselfn.addQuery = function (q) {\n this[this.length++] = q;\n};\nselfn.selector = selfn.toString;\n\nvar elesfn$g = {\n allAre: function allAre(selector) {\n var selObj = new Selector(selector);\n return this.every(function (ele) {\n return selObj.matches(ele);\n });\n },\n is: function is(selector) {\n var selObj = new Selector(selector);\n return this.some(function (ele) {\n return selObj.matches(ele);\n });\n },\n some: function some(fn, thisArg) {\n for (var i = 0; i < this.length; i++) {\n var ret = !thisArg ? fn(this[i], i, this) : fn.apply(thisArg, [this[i], i, this]);\n if (ret) {\n return true;\n }\n }\n return false;\n },\n every: function every(fn, thisArg) {\n for (var i = 0; i < this.length; i++) {\n var ret = !thisArg ? fn(this[i], i, this) : fn.apply(thisArg, [this[i], i, this]);\n if (!ret) {\n return false;\n }\n }\n return true;\n },\n same: function same(collection) {\n // cheap collection ref check\n if (this === collection) {\n return true;\n }\n collection = this.cy().collection(collection);\n var thisLength = this.length;\n var collectionLength = collection.length;\n\n // cheap length check\n if (thisLength !== collectionLength) {\n return false;\n }\n\n // cheap element ref check\n if (thisLength === 1) {\n return this[0] === collection[0];\n }\n return this.every(function (ele) {\n return collection.hasElementWithId(ele.id());\n });\n },\n anySame: function anySame(collection) {\n collection = this.cy().collection(collection);\n return this.some(function (ele) {\n return collection.hasElementWithId(ele.id());\n });\n },\n allAreNeighbors: function allAreNeighbors(collection) {\n collection = this.cy().collection(collection);\n var nhood = this.neighborhood();\n return collection.every(function (ele) {\n return nhood.hasElementWithId(ele.id());\n });\n },\n contains: function contains(collection) {\n collection = this.cy().collection(collection);\n var self = this;\n return collection.every(function (ele) {\n return self.hasElementWithId(ele.id());\n });\n }\n};\nelesfn$g.allAreNeighbours = elesfn$g.allAreNeighbors;\nelesfn$g.has = elesfn$g.contains;\nelesfn$g.equal = elesfn$g.equals = elesfn$g.same;\n\nvar cache = function cache(fn, name) {\n return function traversalCache(arg1, arg2, arg3, arg4) {\n var selectorOrEles = arg1;\n var eles = this;\n var key;\n if (selectorOrEles == null) {\n key = '';\n } else if (elementOrCollection(selectorOrEles) && selectorOrEles.length === 1) {\n key = selectorOrEles.id();\n }\n if (eles.length === 1 && key) {\n var _p = eles[0]._private;\n var tch = _p.traversalCache = _p.traversalCache || {};\n var ch = tch[name] = tch[name] || [];\n var hash = hashString(key);\n var cacheHit = ch[hash];\n if (cacheHit) {\n return cacheHit;\n } else {\n return ch[hash] = fn.call(eles, arg1, arg2, arg3, arg4);\n }\n } else {\n return fn.call(eles, arg1, arg2, arg3, arg4);\n }\n };\n};\n\nvar elesfn$f = {\n parent: function parent(selector) {\n var parents = [];\n\n // optimisation for single ele call\n if (this.length === 1) {\n var parent = this[0]._private.parent;\n if (parent) {\n return parent;\n }\n }\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n var _parent = ele._private.parent;\n if (_parent) {\n parents.push(_parent);\n }\n }\n return this.spawn(parents, true).filter(selector);\n },\n parents: function parents(selector) {\n var parents = [];\n var eles = this.parent();\n while (eles.nonempty()) {\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n parents.push(ele);\n }\n eles = eles.parent();\n }\n return this.spawn(parents, true).filter(selector);\n },\n commonAncestors: function commonAncestors(selector) {\n var ancestors;\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n var parents = ele.parents();\n ancestors = ancestors || parents;\n ancestors = ancestors.intersect(parents); // current list must be common with current ele parents set\n }\n return ancestors.filter(selector);\n },\n orphans: function orphans(selector) {\n return this.stdFilter(function (ele) {\n return ele.isOrphan();\n }).filter(selector);\n },\n nonorphans: function nonorphans(selector) {\n return this.stdFilter(function (ele) {\n return ele.isChild();\n }).filter(selector);\n },\n children: cache(function (selector) {\n var children = [];\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n var eleChildren = ele._private.children;\n for (var j = 0; j < eleChildren.length; j++) {\n children.push(eleChildren[j]);\n }\n }\n return this.spawn(children, true).filter(selector);\n }, 'children'),\n siblings: function siblings(selector) {\n return this.parent().children().not(this).filter(selector);\n },\n isParent: function isParent() {\n var ele = this[0];\n if (ele) {\n return ele.isNode() && ele._private.children.length !== 0;\n }\n },\n isChildless: function isChildless() {\n var ele = this[0];\n if (ele) {\n return ele.isNode() && ele._private.children.length === 0;\n }\n },\n isChild: function isChild() {\n var ele = this[0];\n if (ele) {\n return ele.isNode() && ele._private.parent != null;\n }\n },\n isOrphan: function isOrphan() {\n var ele = this[0];\n if (ele) {\n return ele.isNode() && ele._private.parent == null;\n }\n },\n descendants: function descendants(selector) {\n var elements = [];\n function add(eles) {\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n elements.push(ele);\n if (ele.children().nonempty()) {\n add(ele.children());\n }\n }\n }\n add(this.children());\n return this.spawn(elements, true).filter(selector);\n }\n};\nfunction forEachCompound(eles, fn, includeSelf, recursiveStep) {\n var q = [];\n var did = new Set$1();\n var cy = eles.cy();\n var hasCompounds = cy.hasCompoundNodes();\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n if (includeSelf) {\n q.push(ele);\n } else if (hasCompounds) {\n recursiveStep(q, did, ele);\n }\n }\n while (q.length > 0) {\n var _ele = q.shift();\n fn(_ele);\n did.add(_ele.id());\n if (hasCompounds) {\n recursiveStep(q, did, _ele);\n }\n }\n return eles;\n}\nfunction addChildren(q, did, ele) {\n if (ele.isParent()) {\n var children = ele._private.children;\n for (var i = 0; i < children.length; i++) {\n var child = children[i];\n if (!did.has(child.id())) {\n q.push(child);\n }\n }\n }\n}\n\n// very efficient version of eles.add( eles.descendants() ).forEach()\n// for internal use\nelesfn$f.forEachDown = function (fn) {\n var includeSelf = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : true;\n return forEachCompound(this, fn, includeSelf, addChildren);\n};\nfunction addParent(q, did, ele) {\n if (ele.isChild()) {\n var parent = ele._private.parent;\n if (!did.has(parent.id())) {\n q.push(parent);\n }\n }\n}\nelesfn$f.forEachUp = function (fn) {\n var includeSelf = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : true;\n return forEachCompound(this, fn, includeSelf, addParent);\n};\nfunction addParentAndChildren(q, did, ele) {\n addParent(q, did, ele);\n addChildren(q, did, ele);\n}\nelesfn$f.forEachUpAndDown = function (fn) {\n var includeSelf = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : true;\n return forEachCompound(this, fn, includeSelf, addParentAndChildren);\n};\n\n// aliases\nelesfn$f.ancestors = elesfn$f.parents;\n\nvar fn$5, elesfn$e;\nfn$5 = elesfn$e = {\n data: define.data({\n field: 'data',\n bindingEvent: 'data',\n allowBinding: true,\n allowSetting: true,\n settingEvent: 'data',\n settingTriggersEvent: true,\n triggerFnName: 'trigger',\n allowGetting: true,\n immutableKeys: {\n 'id': true,\n 'source': true,\n 'target': true,\n 'parent': true\n },\n updateStyle: true\n }),\n removeData: define.removeData({\n field: 'data',\n event: 'data',\n triggerFnName: 'trigger',\n triggerEvent: true,\n immutableKeys: {\n 'id': true,\n 'source': true,\n 'target': true,\n 'parent': true\n },\n updateStyle: true\n }),\n scratch: define.data({\n field: 'scratch',\n bindingEvent: 'scratch',\n allowBinding: true,\n allowSetting: true,\n settingEvent: 'scratch',\n settingTriggersEvent: true,\n triggerFnName: 'trigger',\n allowGetting: true,\n updateStyle: true\n }),\n removeScratch: define.removeData({\n field: 'scratch',\n event: 'scratch',\n triggerFnName: 'trigger',\n triggerEvent: true,\n updateStyle: true\n }),\n rscratch: define.data({\n field: 'rscratch',\n allowBinding: false,\n allowSetting: true,\n settingTriggersEvent: false,\n allowGetting: true\n }),\n removeRscratch: define.removeData({\n field: 'rscratch',\n triggerEvent: false\n }),\n id: function id() {\n var ele = this[0];\n if (ele) {\n return ele._private.data.id;\n }\n }\n};\n\n// aliases\nfn$5.attr = fn$5.data;\nfn$5.removeAttr = fn$5.removeData;\nvar data = elesfn$e;\n\nvar elesfn$d = {};\nfunction defineDegreeFunction(callback) {\n return function (includeLoops) {\n var self = this;\n if (includeLoops === undefined) {\n includeLoops = true;\n }\n if (self.length === 0) {\n return;\n }\n if (self.isNode() && !self.removed()) {\n var degree = 0;\n var node = self[0];\n var connectedEdges = node._private.edges;\n for (var i = 0; i < connectedEdges.length; i++) {\n var edge = connectedEdges[i];\n if (!includeLoops && edge.isLoop()) {\n continue;\n }\n degree += callback(node, edge);\n }\n return degree;\n } else {\n return;\n }\n };\n}\nextend(elesfn$d, {\n degree: defineDegreeFunction(function (node, edge) {\n if (edge.source().same(edge.target())) {\n return 2;\n } else {\n return 1;\n }\n }),\n indegree: defineDegreeFunction(function (node, edge) {\n if (edge.target().same(node)) {\n return 1;\n } else {\n return 0;\n }\n }),\n outdegree: defineDegreeFunction(function (node, edge) {\n if (edge.source().same(node)) {\n return 1;\n } else {\n return 0;\n }\n })\n});\nfunction defineDegreeBoundsFunction(degreeFn, callback) {\n return function (includeLoops) {\n var ret;\n var nodes = this.nodes();\n for (var i = 0; i < nodes.length; i++) {\n var ele = nodes[i];\n var degree = ele[degreeFn](includeLoops);\n if (degree !== undefined && (ret === undefined || callback(degree, ret))) {\n ret = degree;\n }\n }\n return ret;\n };\n}\nextend(elesfn$d, {\n minDegree: defineDegreeBoundsFunction('degree', function (degree, min) {\n return degree < min;\n }),\n maxDegree: defineDegreeBoundsFunction('degree', function (degree, max) {\n return degree > max;\n }),\n minIndegree: defineDegreeBoundsFunction('indegree', function (degree, min) {\n return degree < min;\n }),\n maxIndegree: defineDegreeBoundsFunction('indegree', function (degree, max) {\n return degree > max;\n }),\n minOutdegree: defineDegreeBoundsFunction('outdegree', function (degree, min) {\n return degree < min;\n }),\n maxOutdegree: defineDegreeBoundsFunction('outdegree', function (degree, max) {\n return degree > max;\n })\n});\nextend(elesfn$d, {\n totalDegree: function totalDegree(includeLoops) {\n var total = 0;\n var nodes = this.nodes();\n for (var i = 0; i < nodes.length; i++) {\n total += nodes[i].degree(includeLoops);\n }\n return total;\n }\n});\n\nvar fn$4, elesfn$c;\nvar beforePositionSet = function beforePositionSet(eles, newPos, silent) {\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n if (!ele.locked()) {\n var oldPos = ele._private.position;\n var delta = {\n x: newPos.x != null ? newPos.x - oldPos.x : 0,\n y: newPos.y != null ? newPos.y - oldPos.y : 0\n };\n if (ele.isParent() && !(delta.x === 0 && delta.y === 0)) {\n ele.children().shift(delta, silent);\n }\n ele.dirtyBoundingBoxCache();\n }\n }\n};\nvar positionDef = {\n field: 'position',\n bindingEvent: 'position',\n allowBinding: true,\n allowSetting: true,\n settingEvent: 'position',\n settingTriggersEvent: true,\n triggerFnName: 'emitAndNotify',\n allowGetting: true,\n validKeys: ['x', 'y'],\n beforeGet: function beforeGet(ele) {\n ele.updateCompoundBounds();\n },\n beforeSet: function beforeSet(eles, newPos) {\n beforePositionSet(eles, newPos, false);\n },\n onSet: function onSet(eles) {\n eles.dirtyCompoundBoundsCache();\n },\n canSet: function canSet(ele) {\n return !ele.locked();\n }\n};\nfn$4 = elesfn$c = {\n position: define.data(positionDef),\n // position but no notification to renderer\n silentPosition: define.data(extend({}, positionDef, {\n allowBinding: false,\n allowSetting: true,\n settingTriggersEvent: false,\n allowGetting: false,\n beforeSet: function beforeSet(eles, newPos) {\n beforePositionSet(eles, newPos, true);\n },\n onSet: function onSet(eles) {\n eles.dirtyCompoundBoundsCache();\n }\n })),\n positions: function positions(pos, silent) {\n if (plainObject(pos)) {\n if (silent) {\n this.silentPosition(pos);\n } else {\n this.position(pos);\n }\n } else if (fn$6(pos)) {\n var _fn = pos;\n var cy = this.cy();\n cy.startBatch();\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n var _pos = undefined;\n if (_pos = _fn(ele, i)) {\n if (silent) {\n ele.silentPosition(_pos);\n } else {\n ele.position(_pos);\n }\n }\n }\n cy.endBatch();\n }\n return this; // chaining\n },\n silentPositions: function silentPositions(pos) {\n return this.positions(pos, true);\n },\n shift: function shift(dim, val, silent) {\n var delta;\n if (plainObject(dim)) {\n delta = {\n x: number$1(dim.x) ? dim.x : 0,\n y: number$1(dim.y) ? dim.y : 0\n };\n silent = val;\n } else if (string(dim) && number$1(val)) {\n delta = {\n x: 0,\n y: 0\n };\n delta[dim] = val;\n }\n if (delta != null) {\n var cy = this.cy();\n cy.startBatch();\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n\n // exclude any node that is a descendant of the calling collection\n if (cy.hasCompoundNodes() && ele.isChild() && ele.ancestors().anySame(this)) {\n continue;\n }\n var pos = ele.position();\n var newPos = {\n x: pos.x + delta.x,\n y: pos.y + delta.y\n };\n if (silent) {\n ele.silentPosition(newPos);\n } else {\n ele.position(newPos);\n }\n }\n cy.endBatch();\n }\n return this;\n },\n silentShift: function silentShift(dim, val) {\n if (plainObject(dim)) {\n this.shift(dim, true);\n } else if (string(dim) && number$1(val)) {\n this.shift(dim, val, true);\n }\n return this;\n },\n // get/set the rendered (i.e. on screen) positon of the element\n renderedPosition: function renderedPosition(dim, val) {\n var ele = this[0];\n var cy = this.cy();\n var zoom = cy.zoom();\n var pan = cy.pan();\n var rpos = plainObject(dim) ? dim : undefined;\n var setting = rpos !== undefined || val !== undefined && string(dim);\n if (ele && ele.isNode()) {\n // must have an element and must be a node to return position\n if (setting) {\n for (var i = 0; i < this.length; i++) {\n var _ele = this[i];\n if (val !== undefined) {\n // set one dimension\n _ele.position(dim, (val - pan[dim]) / zoom);\n } else if (rpos !== undefined) {\n // set whole position\n _ele.position(renderedToModelPosition(rpos, zoom, pan));\n }\n }\n } else {\n // getting\n var pos = ele.position();\n rpos = modelToRenderedPosition$1(pos, zoom, pan);\n if (dim === undefined) {\n // then return the whole rendered position\n return rpos;\n } else {\n // then return the specified dimension\n return rpos[dim];\n }\n }\n } else if (!setting) {\n return undefined; // for empty collection case\n }\n return this; // chaining\n },\n // get/set the position relative to the parent\n relativePosition: function relativePosition(dim, val) {\n var ele = this[0];\n var cy = this.cy();\n var ppos = plainObject(dim) ? dim : undefined;\n var setting = ppos !== undefined || val !== undefined && string(dim);\n var hasCompoundNodes = cy.hasCompoundNodes();\n if (ele && ele.isNode()) {\n // must have an element and must be a node to return position\n if (setting) {\n for (var i = 0; i < this.length; i++) {\n var _ele2 = this[i];\n var parent = hasCompoundNodes ? _ele2.parent() : null;\n var hasParent = parent && parent.length > 0;\n var relativeToParent = hasParent;\n if (hasParent) {\n parent = parent[0];\n }\n var origin = relativeToParent ? parent.position() : {\n x: 0,\n y: 0\n };\n if (val !== undefined) {\n // set one dimension\n _ele2.position(dim, val + origin[dim]);\n } else if (ppos !== undefined) {\n // set whole position\n _ele2.position({\n x: ppos.x + origin.x,\n y: ppos.y + origin.y\n });\n }\n }\n } else {\n // getting\n var pos = ele.position();\n var _parent = hasCompoundNodes ? ele.parent() : null;\n var _hasParent = _parent && _parent.length > 0;\n var _relativeToParent = _hasParent;\n if (_hasParent) {\n _parent = _parent[0];\n }\n var _origin = _relativeToParent ? _parent.position() : {\n x: 0,\n y: 0\n };\n ppos = {\n x: pos.x - _origin.x,\n y: pos.y - _origin.y\n };\n if (dim === undefined) {\n // then return the whole rendered position\n return ppos;\n } else {\n // then return the specified dimension\n return ppos[dim];\n }\n }\n } else if (!setting) {\n return undefined; // for empty collection case\n }\n return this; // chaining\n }\n};\n\n// aliases\nfn$4.modelPosition = fn$4.point = fn$4.position;\nfn$4.modelPositions = fn$4.points = fn$4.positions;\nfn$4.renderedPoint = fn$4.renderedPosition;\nfn$4.relativePoint = fn$4.relativePosition;\nvar position = elesfn$c;\n\nvar fn$3, elesfn$b;\nfn$3 = elesfn$b = {};\nelesfn$b.renderedBoundingBox = function (options) {\n var bb = this.boundingBox(options);\n var cy = this.cy();\n var zoom = cy.zoom();\n var pan = cy.pan();\n var x1 = bb.x1 * zoom + pan.x;\n var x2 = bb.x2 * zoom + pan.x;\n var y1 = bb.y1 * zoom + pan.y;\n var y2 = bb.y2 * zoom + pan.y;\n return {\n x1: x1,\n x2: x2,\n y1: y1,\n y2: y2,\n w: x2 - x1,\n h: y2 - y1\n };\n};\nelesfn$b.dirtyCompoundBoundsCache = function () {\n var silent = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : false;\n var cy = this.cy();\n if (!cy.styleEnabled() || !cy.hasCompoundNodes()) {\n return this;\n }\n this.forEachUp(function (ele) {\n if (ele.isParent()) {\n var _p = ele._private;\n _p.compoundBoundsClean = false;\n _p.bbCache = null;\n if (!silent) {\n ele.emitAndNotify('bounds');\n }\n }\n });\n return this;\n};\nelesfn$b.updateCompoundBounds = function () {\n var force = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : false;\n var cy = this.cy();\n\n // not possible to do on non-compound graphs or with the style disabled\n if (!cy.styleEnabled() || !cy.hasCompoundNodes()) {\n return this;\n }\n\n // save cycles when batching -- but bounds will be stale (or not exist yet)\n if (!force && cy.batching()) {\n return this;\n }\n function update(parent) {\n if (!parent.isParent()) {\n return;\n }\n var _p = parent._private;\n var children = parent.children();\n var includeLabels = parent.pstyle('compound-sizing-wrt-labels').value === 'include';\n var min = {\n width: {\n val: parent.pstyle('min-width').pfValue,\n left: parent.pstyle('min-width-bias-left'),\n right: parent.pstyle('min-width-bias-right')\n },\n height: {\n val: parent.pstyle('min-height').pfValue,\n top: parent.pstyle('min-height-bias-top'),\n bottom: parent.pstyle('min-height-bias-bottom')\n }\n };\n var bb = children.boundingBox({\n includeLabels: includeLabels,\n includeOverlays: false,\n // updating the compound bounds happens outside of the regular\n // cache cycle (i.e. before fired events)\n useCache: false\n });\n var pos = _p.position;\n\n // if children take up zero area then keep position and fall back on stylesheet w/h\n if (bb.w === 0 || bb.h === 0) {\n bb = {\n w: parent.pstyle('width').pfValue,\n h: parent.pstyle('height').pfValue\n };\n bb.x1 = pos.x - bb.w / 2;\n bb.x2 = pos.x + bb.w / 2;\n bb.y1 = pos.y - bb.h / 2;\n bb.y2 = pos.y + bb.h / 2;\n }\n function computeBiasValues(propDiff, propBias, propBiasComplement) {\n var biasDiff = 0;\n var biasComplementDiff = 0;\n var biasTotal = propBias + propBiasComplement;\n if (propDiff > 0 && biasTotal > 0) {\n biasDiff = propBias / biasTotal * propDiff;\n biasComplementDiff = propBiasComplement / biasTotal * propDiff;\n }\n return {\n biasDiff: biasDiff,\n biasComplementDiff: biasComplementDiff\n };\n }\n function computePaddingValues(width, height, paddingObject, relativeTo) {\n // Assuming percentage is number from 0 to 1\n if (paddingObject.units === '%') {\n switch (relativeTo) {\n case 'width':\n return width > 0 ? paddingObject.pfValue * width : 0;\n case 'height':\n return height > 0 ? paddingObject.pfValue * height : 0;\n case 'average':\n return width > 0 && height > 0 ? paddingObject.pfValue * (width + height) / 2 : 0;\n case 'min':\n return width > 0 && height > 0 ? width > height ? paddingObject.pfValue * height : paddingObject.pfValue * width : 0;\n case 'max':\n return width > 0 && height > 0 ? width > height ? paddingObject.pfValue * width : paddingObject.pfValue * height : 0;\n default:\n return 0;\n }\n } else if (paddingObject.units === 'px') {\n return paddingObject.pfValue;\n } else {\n return 0;\n }\n }\n var leftVal = min.width.left.value;\n if (min.width.left.units === 'px' && min.width.val > 0) {\n leftVal = leftVal * 100 / min.width.val;\n }\n var rightVal = min.width.right.value;\n if (min.width.right.units === 'px' && min.width.val > 0) {\n rightVal = rightVal * 100 / min.width.val;\n }\n var topVal = min.height.top.value;\n if (min.height.top.units === 'px' && min.height.val > 0) {\n topVal = topVal * 100 / min.height.val;\n }\n var bottomVal = min.height.bottom.value;\n if (min.height.bottom.units === 'px' && min.height.val > 0) {\n bottomVal = bottomVal * 100 / min.height.val;\n }\n var widthBiasDiffs = computeBiasValues(min.width.val - bb.w, leftVal, rightVal);\n var diffLeft = widthBiasDiffs.biasDiff;\n var diffRight = widthBiasDiffs.biasComplementDiff;\n var heightBiasDiffs = computeBiasValues(min.height.val - bb.h, topVal, bottomVal);\n var diffTop = heightBiasDiffs.biasDiff;\n var diffBottom = heightBiasDiffs.biasComplementDiff;\n _p.autoPadding = computePaddingValues(bb.w, bb.h, parent.pstyle('padding'), parent.pstyle('padding-relative-to').value);\n _p.autoWidth = Math.max(bb.w, min.width.val);\n pos.x = (-diffLeft + bb.x1 + bb.x2 + diffRight) / 2;\n _p.autoHeight = Math.max(bb.h, min.height.val);\n pos.y = (-diffTop + bb.y1 + bb.y2 + diffBottom) / 2;\n }\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n var _p = ele._private;\n if (!_p.compoundBoundsClean || force) {\n update(ele);\n if (!cy.batching()) {\n _p.compoundBoundsClean = true;\n }\n }\n }\n return this;\n};\nvar noninf = function noninf(x) {\n if (x === Infinity || x === -Infinity) {\n return 0;\n }\n return x;\n};\nvar updateBounds = function updateBounds(b, x1, y1, x2, y2) {\n // don't update with zero area boxes\n if (x2 - x1 === 0 || y2 - y1 === 0) {\n return;\n }\n\n // don't update with null dim\n if (x1 == null || y1 == null || x2 == null || y2 == null) {\n return;\n }\n b.x1 = x1 < b.x1 ? x1 : b.x1;\n b.x2 = x2 > b.x2 ? x2 : b.x2;\n b.y1 = y1 < b.y1 ? y1 : b.y1;\n b.y2 = y2 > b.y2 ? y2 : b.y2;\n b.w = b.x2 - b.x1;\n b.h = b.y2 - b.y1;\n};\nvar updateBoundsFromBox = function updateBoundsFromBox(b, b2) {\n if (b2 == null) {\n return b;\n }\n return updateBounds(b, b2.x1, b2.y1, b2.x2, b2.y2);\n};\nvar prefixedProperty = function prefixedProperty(obj, field, prefix) {\n return getPrefixedProperty(obj, field, prefix);\n};\nvar updateBoundsFromArrow = function updateBoundsFromArrow(bounds, ele, prefix) {\n if (ele.cy().headless()) {\n return;\n }\n var _p = ele._private;\n var rstyle = _p.rstyle;\n var halfArW = rstyle.arrowWidth / 2;\n var arrowType = ele.pstyle(prefix + '-arrow-shape').value;\n var x;\n var y;\n if (arrowType !== 'none') {\n if (prefix === 'source') {\n x = rstyle.srcX;\n y = rstyle.srcY;\n } else if (prefix === 'target') {\n x = rstyle.tgtX;\n y = rstyle.tgtY;\n } else {\n x = rstyle.midX;\n y = rstyle.midY;\n }\n\n // always store the individual arrow bounds\n var bbs = _p.arrowBounds = _p.arrowBounds || {};\n var bb = bbs[prefix] = bbs[prefix] || {};\n bb.x1 = x - halfArW;\n bb.y1 = y - halfArW;\n bb.x2 = x + halfArW;\n bb.y2 = y + halfArW;\n bb.w = bb.x2 - bb.x1;\n bb.h = bb.y2 - bb.y1;\n expandBoundingBox(bb, 1);\n updateBounds(bounds, bb.x1, bb.y1, bb.x2, bb.y2);\n }\n};\nvar updateBoundsFromLabel = function updateBoundsFromLabel(bounds, ele, prefix) {\n if (ele.cy().headless()) {\n return;\n }\n var prefixDash;\n if (prefix) {\n prefixDash = prefix + '-';\n } else {\n prefixDash = '';\n }\n var _p = ele._private;\n var rstyle = _p.rstyle;\n var label = ele.pstyle(prefixDash + 'label').strValue;\n if (label) {\n var halign = ele.pstyle('text-halign');\n var valign = ele.pstyle('text-valign');\n var labelWidth = prefixedProperty(rstyle, 'labelWidth', prefix);\n var labelHeight = prefixedProperty(rstyle, 'labelHeight', prefix);\n var labelX = prefixedProperty(rstyle, 'labelX', prefix);\n var labelY = prefixedProperty(rstyle, 'labelY', prefix);\n var marginX = ele.pstyle(prefixDash + 'text-margin-x').pfValue;\n var marginY = ele.pstyle(prefixDash + 'text-margin-y').pfValue;\n var isEdge = ele.isEdge();\n var rotation = ele.pstyle(prefixDash + 'text-rotation');\n var outlineWidth = ele.pstyle('text-outline-width').pfValue;\n var borderWidth = ele.pstyle('text-border-width').pfValue;\n var halfBorderWidth = borderWidth / 2;\n var padding = ele.pstyle('text-background-padding').pfValue;\n var marginOfError = 2; // expand to work around browser dimension inaccuracies\n\n var lh = labelHeight;\n var lw = labelWidth;\n var lw_2 = lw / 2;\n var lh_2 = lh / 2;\n var lx1, lx2, ly1, ly2;\n if (isEdge) {\n lx1 = labelX - lw_2;\n lx2 = labelX + lw_2;\n ly1 = labelY - lh_2;\n ly2 = labelY + lh_2;\n } else {\n switch (halign.value) {\n case 'left':\n lx1 = labelX - lw;\n lx2 = labelX;\n break;\n case 'center':\n lx1 = labelX - lw_2;\n lx2 = labelX + lw_2;\n break;\n case 'right':\n lx1 = labelX;\n lx2 = labelX + lw;\n break;\n }\n switch (valign.value) {\n case 'top':\n ly1 = labelY - lh;\n ly2 = labelY;\n break;\n case 'center':\n ly1 = labelY - lh_2;\n ly2 = labelY + lh_2;\n break;\n case 'bottom':\n ly1 = labelY;\n ly2 = labelY + lh;\n break;\n }\n }\n\n // shift by margin and expand by outline and border\n var leftPad = marginX - Math.max(outlineWidth, halfBorderWidth) - padding - marginOfError;\n var rightPad = marginX + Math.max(outlineWidth, halfBorderWidth) + padding + marginOfError;\n var topPad = marginY - Math.max(outlineWidth, halfBorderWidth) - padding - marginOfError;\n var botPad = marginY + Math.max(outlineWidth, halfBorderWidth) + padding + marginOfError;\n lx1 += leftPad;\n lx2 += rightPad;\n ly1 += topPad;\n ly2 += botPad;\n\n // always store the unrotated label bounds separately\n var bbPrefix = prefix || 'main';\n var bbs = _p.labelBounds;\n var bb = bbs[bbPrefix] = bbs[bbPrefix] || {};\n bb.x1 = lx1;\n bb.y1 = ly1;\n bb.x2 = lx2;\n bb.y2 = ly2;\n bb.w = lx2 - lx1;\n bb.h = ly2 - ly1;\n bb.leftPad = leftPad;\n bb.rightPad = rightPad;\n bb.topPad = topPad;\n bb.botPad = botPad;\n var isAutorotate = isEdge && rotation.strValue === 'autorotate';\n var isPfValue = rotation.pfValue != null && rotation.pfValue !== 0;\n if (isAutorotate || isPfValue) {\n var theta = isAutorotate ? prefixedProperty(_p.rstyle, 'labelAngle', prefix) : rotation.pfValue;\n var cos = Math.cos(theta);\n var sin = Math.sin(theta);\n\n // rotation point (default value for center-center)\n var xo = (lx1 + lx2) / 2;\n var yo = (ly1 + ly2) / 2;\n if (!isEdge) {\n switch (halign.value) {\n case 'left':\n xo = lx2;\n break;\n case 'right':\n xo = lx1;\n break;\n }\n switch (valign.value) {\n case 'top':\n yo = ly2;\n break;\n case 'bottom':\n yo = ly1;\n break;\n }\n }\n var rotate = function rotate(x, y) {\n x = x - xo;\n y = y - yo;\n return {\n x: x * cos - y * sin + xo,\n y: x * sin + y * cos + yo\n };\n };\n var px1y1 = rotate(lx1, ly1);\n var px1y2 = rotate(lx1, ly2);\n var px2y1 = rotate(lx2, ly1);\n var px2y2 = rotate(lx2, ly2);\n lx1 = Math.min(px1y1.x, px1y2.x, px2y1.x, px2y2.x);\n lx2 = Math.max(px1y1.x, px1y2.x, px2y1.x, px2y2.x);\n ly1 = Math.min(px1y1.y, px1y2.y, px2y1.y, px2y2.y);\n ly2 = Math.max(px1y1.y, px1y2.y, px2y1.y, px2y2.y);\n }\n var bbPrefixRot = bbPrefix + 'Rot';\n var bbRot = bbs[bbPrefixRot] = bbs[bbPrefixRot] || {};\n bbRot.x1 = lx1;\n bbRot.y1 = ly1;\n bbRot.x2 = lx2;\n bbRot.y2 = ly2;\n bbRot.w = lx2 - lx1;\n bbRot.h = ly2 - ly1;\n updateBounds(bounds, lx1, ly1, lx2, ly2);\n updateBounds(_p.labelBounds.all, lx1, ly1, lx2, ly2);\n }\n return bounds;\n};\nvar updateBoundsFromOutline = function updateBoundsFromOutline(bounds, ele) {\n if (ele.cy().headless()) {\n return;\n }\n var outlineOpacity = ele.pstyle('outline-opacity').value;\n var outlineWidth = ele.pstyle('outline-width').value;\n var outlineOffset = ele.pstyle('outline-offset').value;\n var expansion = outlineWidth + outlineOffset;\n updateBoundsFromMiter(bounds, ele, outlineOpacity, expansion, 'outside', expansion / 2);\n};\nvar updateBoundsFromMiter = function updateBoundsFromMiter(bounds, ele, opacity, expansionSize, expansionPosition, useFallbackValue) {\n if (opacity === 0 || expansionSize <= 0 || expansionPosition === 'inside') {\n return;\n }\n var cy = ele.cy();\n var r = cy.renderer();\n var rshape = r.nodeShapes[r.getNodeShape(ele)];\n if (!rshape) {\n return;\n }\n var _ele$position = ele.position(),\n x = _ele$position.x,\n y = _ele$position.y;\n var w = ele.width();\n var h = ele.height();\n if (rshape.hasMiterBounds) {\n if (expansionPosition === 'center') {\n expansionSize /= 2;\n }\n var mbb = rshape.miterBounds(x, y, w, h, expansionSize);\n updateBoundsFromBox(bounds, mbb);\n } else if (useFallbackValue != null && useFallbackValue > 0) {\n expandBoundingBoxSides(bounds, [useFallbackValue, useFallbackValue, useFallbackValue, useFallbackValue]);\n }\n};\nvar updateBoundsFromMiterBorder = function updateBoundsFromMiterBorder(bounds, ele) {\n if (ele.cy().headless()) {\n return;\n }\n var borderOpacity = ele.pstyle('border-opacity').value;\n var borderWidth = ele.pstyle('border-width').pfValue;\n var borderPosition = ele.pstyle('border-position').value;\n updateBoundsFromMiter(bounds, ele, borderOpacity, borderWidth, borderPosition);\n};\n\n// get the bounding box of the elements (in raw model position)\nvar boundingBoxImpl = function boundingBoxImpl(ele, options) {\n var cy = ele._private.cy;\n var styleEnabled = cy.styleEnabled();\n var headless = cy.headless();\n var bounds = makeBoundingBox();\n var _p = ele._private;\n var isNode = ele.isNode();\n var isEdge = ele.isEdge();\n var ex1, ex2, ey1, ey2; // extrema of body / lines\n var x, y; // node pos\n var rstyle = _p.rstyle;\n var manualExpansion = isNode && styleEnabled ? ele.pstyle('bounds-expansion').pfValue : [0];\n\n // must use `display` prop only, as reading `compound.width()` causes recursion\n // (other factors like width values will be considered later in this function anyway)\n var isDisplayed = function isDisplayed(ele) {\n return ele.pstyle('display').value !== 'none';\n };\n var displayed = !styleEnabled || isDisplayed(ele)\n\n // must take into account connected nodes b/c of implicit edge hiding on display:none node\n && (!isEdge || isDisplayed(ele.source()) && isDisplayed(ele.target()));\n if (displayed) {\n // displayed suffices, since we will find zero area eles anyway\n var overlayOpacity = 0;\n var overlayPadding = 0;\n if (styleEnabled && options.includeOverlays) {\n overlayOpacity = ele.pstyle('overlay-opacity').value;\n if (overlayOpacity !== 0) {\n overlayPadding = ele.pstyle('overlay-padding').value;\n }\n }\n var underlayOpacity = 0;\n var underlayPadding = 0;\n if (styleEnabled && options.includeUnderlays) {\n underlayOpacity = ele.pstyle('underlay-opacity').value;\n if (underlayOpacity !== 0) {\n underlayPadding = ele.pstyle('underlay-padding').value;\n }\n }\n var padding = Math.max(overlayPadding, underlayPadding);\n var w = 0;\n var wHalf = 0;\n if (styleEnabled) {\n w = ele.pstyle('width').pfValue;\n wHalf = w / 2;\n }\n if (isNode && options.includeNodes) {\n var pos = ele.position();\n x = pos.x;\n y = pos.y;\n var _w = ele.outerWidth();\n var halfW = _w / 2;\n var h = ele.outerHeight();\n var halfH = h / 2;\n\n // handle node dimensions\n /////////////////////////\n\n ex1 = x - halfW;\n ex2 = x + halfW;\n ey1 = y - halfH;\n ey2 = y + halfH;\n updateBounds(bounds, ex1, ey1, ex2, ey2);\n if (styleEnabled) {\n updateBoundsFromOutline(bounds, ele);\n }\n if (styleEnabled && options.includeOutlines && !headless) {\n updateBoundsFromOutline(bounds, ele);\n }\n if (styleEnabled) {\n updateBoundsFromMiterBorder(bounds, ele);\n }\n } else if (isEdge && options.includeEdges) {\n if (styleEnabled && !headless) {\n var curveStyle = ele.pstyle('curve-style').strValue;\n\n // handle edge dimensions (rough box estimate)\n //////////////////////////////////////////////\n\n ex1 = Math.min(rstyle.srcX, rstyle.midX, rstyle.tgtX);\n ex2 = Math.max(rstyle.srcX, rstyle.midX, rstyle.tgtX);\n ey1 = Math.min(rstyle.srcY, rstyle.midY, rstyle.tgtY);\n ey2 = Math.max(rstyle.srcY, rstyle.midY, rstyle.tgtY);\n\n // take into account edge width\n ex1 -= wHalf;\n ex2 += wHalf;\n ey1 -= wHalf;\n ey2 += wHalf;\n updateBounds(bounds, ex1, ey1, ex2, ey2);\n\n // precise edges\n ////////////////\n\n if (curveStyle === 'haystack') {\n var hpts = rstyle.haystackPts;\n if (hpts && hpts.length === 2) {\n ex1 = hpts[0].x;\n ey1 = hpts[0].y;\n ex2 = hpts[1].x;\n ey2 = hpts[1].y;\n if (ex1 > ex2) {\n var temp = ex1;\n ex1 = ex2;\n ex2 = temp;\n }\n if (ey1 > ey2) {\n var _temp = ey1;\n ey1 = ey2;\n ey2 = _temp;\n }\n updateBounds(bounds, ex1 - wHalf, ey1 - wHalf, ex2 + wHalf, ey2 + wHalf);\n }\n } else if (curveStyle === 'bezier' || curveStyle === 'unbundled-bezier' || endsWith(curveStyle, 'segments') || endsWith(curveStyle, 'taxi')) {\n var pts;\n switch (curveStyle) {\n case 'bezier':\n case 'unbundled-bezier':\n pts = rstyle.bezierPts;\n break;\n case 'segments':\n case 'taxi':\n case 'round-segments':\n case 'round-taxi':\n pts = rstyle.linePts;\n break;\n }\n if (pts != null) {\n for (var j = 0; j < pts.length; j++) {\n var pt = pts[j];\n ex1 = pt.x - wHalf;\n ex2 = pt.x + wHalf;\n ey1 = pt.y - wHalf;\n ey2 = pt.y + wHalf;\n updateBounds(bounds, ex1, ey1, ex2, ey2);\n }\n }\n } // bezier-like or segment-like edge\n } else {\n // headless or style disabled\n\n // fallback on source and target positions\n //////////////////////////////////////////\n\n var n1 = ele.source();\n var n1pos = n1.position();\n var n2 = ele.target();\n var n2pos = n2.position();\n ex1 = n1pos.x;\n ex2 = n2pos.x;\n ey1 = n1pos.y;\n ey2 = n2pos.y;\n if (ex1 > ex2) {\n var _temp2 = ex1;\n ex1 = ex2;\n ex2 = _temp2;\n }\n if (ey1 > ey2) {\n var _temp3 = ey1;\n ey1 = ey2;\n ey2 = _temp3;\n }\n\n // take into account edge width\n ex1 -= wHalf;\n ex2 += wHalf;\n ey1 -= wHalf;\n ey2 += wHalf;\n updateBounds(bounds, ex1, ey1, ex2, ey2);\n } // headless or style disabled\n } // edges\n\n // handle edge arrow size\n /////////////////////////\n\n if (styleEnabled && options.includeEdges && isEdge) {\n updateBoundsFromArrow(bounds, ele, 'mid-source');\n updateBoundsFromArrow(bounds, ele, 'mid-target');\n updateBoundsFromArrow(bounds, ele, 'source');\n updateBoundsFromArrow(bounds, ele, 'target');\n }\n\n // ghost\n ////////\n\n if (styleEnabled) {\n var ghost = ele.pstyle('ghost').value === 'yes';\n if (ghost) {\n var gx = ele.pstyle('ghost-offset-x').pfValue;\n var gy = ele.pstyle('ghost-offset-y').pfValue;\n updateBounds(bounds, bounds.x1 + gx, bounds.y1 + gy, bounds.x2 + gx, bounds.y2 + gy);\n }\n }\n\n // always store the body bounds separately from the labels\n var bbBody = _p.bodyBounds = _p.bodyBounds || {};\n assignBoundingBox(bbBody, bounds);\n expandBoundingBoxSides(bbBody, manualExpansion);\n expandBoundingBox(bbBody, 1); // expand to work around browser dimension inaccuracies\n\n // overlay\n //////////\n\n if (styleEnabled) {\n ex1 = bounds.x1;\n ex2 = bounds.x2;\n ey1 = bounds.y1;\n ey2 = bounds.y2;\n updateBounds(bounds, ex1 - padding, ey1 - padding, ex2 + padding, ey2 + padding);\n }\n\n // always store the body bounds separately from the labels\n var bbOverlay = _p.overlayBounds = _p.overlayBounds || {};\n assignBoundingBox(bbOverlay, bounds);\n expandBoundingBoxSides(bbOverlay, manualExpansion);\n expandBoundingBox(bbOverlay, 1); // expand to work around browser dimension inaccuracies\n\n // handle label dimensions\n //////////////////////////\n\n var bbLabels = _p.labelBounds = _p.labelBounds || {};\n if (bbLabels.all != null) {\n clearBoundingBox(bbLabels.all);\n } else {\n bbLabels.all = makeBoundingBox();\n }\n if (styleEnabled && options.includeLabels) {\n if (options.includeMainLabels) {\n updateBoundsFromLabel(bounds, ele, null);\n }\n if (isEdge) {\n if (options.includeSourceLabels) {\n updateBoundsFromLabel(bounds, ele, 'source');\n }\n if (options.includeTargetLabels) {\n updateBoundsFromLabel(bounds, ele, 'target');\n }\n }\n } // style enabled for labels\n } // if displayed\n\n bounds.x1 = noninf(bounds.x1);\n bounds.y1 = noninf(bounds.y1);\n bounds.x2 = noninf(bounds.x2);\n bounds.y2 = noninf(bounds.y2);\n bounds.w = noninf(bounds.x2 - bounds.x1);\n bounds.h = noninf(bounds.y2 - bounds.y1);\n if (bounds.w > 0 && bounds.h > 0 && displayed) {\n expandBoundingBoxSides(bounds, manualExpansion);\n\n // expand bounds by 1 because antialiasing can increase the visual/effective size by 1 on all sides\n expandBoundingBox(bounds, 1);\n }\n return bounds;\n};\nvar getKey = function getKey(opts) {\n var i = 0;\n var tf = function tf(val) {\n return (val ? 1 : 0) << i++;\n };\n var key = 0;\n key += tf(opts.incudeNodes);\n key += tf(opts.includeEdges);\n key += tf(opts.includeLabels);\n key += tf(opts.includeMainLabels);\n key += tf(opts.includeSourceLabels);\n key += tf(opts.includeTargetLabels);\n key += tf(opts.includeOverlays);\n key += tf(opts.includeOutlines);\n return key;\n};\nvar getBoundingBoxPosKey = function getBoundingBoxPosKey(ele) {\n var r = function r(x) {\n return Math.round(x);\n };\n if (ele.isEdge()) {\n var p1 = ele.source().position();\n var p2 = ele.target().position();\n return hashIntsArray([r(p1.x), r(p1.y), r(p2.x), r(p2.y)]);\n } else {\n var p = ele.position();\n return hashIntsArray([r(p.x), r(p.y)]);\n }\n};\nvar cachedBoundingBoxImpl = function cachedBoundingBoxImpl(ele, opts) {\n var _p = ele._private;\n var bb;\n var isEdge = ele.isEdge();\n var key = opts == null ? defBbOptsKey : getKey(opts);\n var usingDefOpts = key === defBbOptsKey;\n if (_p.bbCache == null) {\n bb = boundingBoxImpl(ele, defBbOpts);\n _p.bbCache = bb;\n _p.bbCachePosKey = getBoundingBoxPosKey(ele);\n } else {\n bb = _p.bbCache;\n }\n\n // not using def opts => need to build up bb from combination of sub bbs\n if (!usingDefOpts) {\n var isNode = ele.isNode();\n bb = makeBoundingBox();\n if (opts.includeNodes && isNode || opts.includeEdges && !isNode) {\n if (opts.includeOverlays) {\n updateBoundsFromBox(bb, _p.overlayBounds);\n } else {\n updateBoundsFromBox(bb, _p.bodyBounds);\n }\n }\n if (opts.includeLabels) {\n if (opts.includeMainLabels && (!isEdge || opts.includeSourceLabels && opts.includeTargetLabels)) {\n updateBoundsFromBox(bb, _p.labelBounds.all);\n } else {\n if (opts.includeMainLabels) {\n updateBoundsFromBox(bb, _p.labelBounds.mainRot);\n }\n if (opts.includeSourceLabels) {\n updateBoundsFromBox(bb, _p.labelBounds.sourceRot);\n }\n if (opts.includeTargetLabels) {\n updateBoundsFromBox(bb, _p.labelBounds.targetRot);\n }\n }\n }\n bb.w = bb.x2 - bb.x1;\n bb.h = bb.y2 - bb.y1;\n }\n return bb;\n};\nvar defBbOpts = {\n includeNodes: true,\n includeEdges: true,\n includeLabels: true,\n includeMainLabels: true,\n includeSourceLabels: true,\n includeTargetLabels: true,\n includeOverlays: true,\n includeUnderlays: true,\n includeOutlines: true,\n useCache: true\n};\nvar defBbOptsKey = getKey(defBbOpts);\nvar filledBbOpts = defaults$g(defBbOpts);\nelesfn$b.boundingBox = function (options) {\n var bounds;\n var useCache = options === undefined || options.useCache === undefined || options.useCache === true;\n var isDirty = memoize(function (ele) {\n var _p = ele._private;\n return _p.bbCache == null || _p.styleDirty || _p.bbCachePosKey !== getBoundingBoxPosKey(ele);\n }, function (ele) {\n return ele.id();\n });\n\n // the main usecase is ele.boundingBox() for a single element with no/def options\n // specified s.t. the cache is used, so check for this case to make it faster by\n // avoiding the overhead of the rest of the function\n if (useCache && this.length === 1 && !isDirty(this[0])) {\n if (options === undefined) {\n options = defBbOpts;\n } else {\n options = filledBbOpts(options);\n }\n bounds = cachedBoundingBoxImpl(this[0], options);\n } else {\n bounds = makeBoundingBox();\n options = options || defBbOpts;\n var opts = filledBbOpts(options);\n var eles = this;\n var cy = eles.cy();\n var styleEnabled = cy.styleEnabled();\n\n // cache the isDirty state for all eles, edges first since they depend on node state\n this.edges().forEach(isDirty);\n this.nodes().forEach(isDirty);\n if (styleEnabled) {\n this.recalculateRenderedStyle(useCache);\n }\n this.updateCompoundBounds(!useCache);\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n if (isDirty(ele)) {\n ele.dirtyBoundingBoxCache();\n }\n updateBoundsFromBox(bounds, cachedBoundingBoxImpl(ele, opts));\n }\n }\n bounds.x1 = noninf(bounds.x1);\n bounds.y1 = noninf(bounds.y1);\n bounds.x2 = noninf(bounds.x2);\n bounds.y2 = noninf(bounds.y2);\n bounds.w = noninf(bounds.x2 - bounds.x1);\n bounds.h = noninf(bounds.y2 - bounds.y1);\n return bounds;\n};\nelesfn$b.dirtyBoundingBoxCache = function () {\n for (var i = 0; i < this.length; i++) {\n var _p = this[i]._private;\n _p.bbCache = null;\n _p.bbCachePosKey = null;\n _p.bodyBounds = null;\n _p.overlayBounds = null;\n _p.labelBounds.all = null;\n _p.labelBounds.source = null;\n _p.labelBounds.target = null;\n _p.labelBounds.main = null;\n _p.labelBounds.sourceRot = null;\n _p.labelBounds.targetRot = null;\n _p.labelBounds.mainRot = null;\n _p.arrowBounds.source = null;\n _p.arrowBounds.target = null;\n _p.arrowBounds['mid-source'] = null;\n _p.arrowBounds['mid-target'] = null;\n }\n this.emitAndNotify('bounds');\n return this;\n};\n\n// private helper to get bounding box for custom node positions\n// - good for perf in certain cases but currently requires dirtying the rendered style\n// - would be better to not modify the nodes but the nodes are read directly everywhere in the renderer...\n// - try to use for only things like discrete layouts where the node position would change anyway\nelesfn$b.boundingBoxAt = function (fn) {\n var nodes = this.nodes();\n var cy = this.cy();\n var hasCompoundNodes = cy.hasCompoundNodes();\n var parents = cy.collection();\n if (hasCompoundNodes) {\n parents = nodes.filter(function (node) {\n return node.isParent();\n });\n nodes = nodes.not(parents);\n }\n if (plainObject(fn)) {\n var obj = fn;\n fn = function fn() {\n return obj;\n };\n }\n var storeOldPos = function storeOldPos(node, i) {\n return node._private.bbAtOldPos = fn(node, i);\n };\n var getOldPos = function getOldPos(node) {\n return node._private.bbAtOldPos;\n };\n cy.startBatch();\n nodes.forEach(storeOldPos).silentPositions(fn);\n if (hasCompoundNodes) {\n parents.dirtyCompoundBoundsCache();\n parents.dirtyBoundingBoxCache();\n parents.updateCompoundBounds(true); // force update b/c we're inside a batch cycle\n }\n var bb = copyBoundingBox(this.boundingBox({\n useCache: false\n }));\n nodes.silentPositions(getOldPos);\n if (hasCompoundNodes) {\n parents.dirtyCompoundBoundsCache();\n parents.dirtyBoundingBoxCache();\n parents.updateCompoundBounds(true); // force update b/c we're inside a batch cycle\n }\n cy.endBatch();\n return bb;\n};\nfn$3.boundingbox = fn$3.bb = fn$3.boundingBox;\nfn$3.renderedBoundingbox = fn$3.renderedBoundingBox;\nvar bounds = elesfn$b;\n\nvar fn$2, elesfn$a;\nfn$2 = elesfn$a = {};\nvar defineDimFns = function defineDimFns(opts) {\n opts.uppercaseName = capitalize(opts.name);\n opts.autoName = 'auto' + opts.uppercaseName;\n opts.labelName = 'label' + opts.uppercaseName;\n opts.outerName = 'outer' + opts.uppercaseName;\n opts.uppercaseOuterName = capitalize(opts.outerName);\n fn$2[opts.name] = function dimImpl() {\n var ele = this[0];\n var _p = ele._private;\n var cy = _p.cy;\n var styleEnabled = cy._private.styleEnabled;\n if (ele) {\n if (styleEnabled) {\n if (ele.isParent()) {\n ele.updateCompoundBounds();\n return _p[opts.autoName] || 0;\n }\n var d = ele.pstyle(opts.name);\n switch (d.strValue) {\n case 'label':\n ele.recalculateRenderedStyle();\n return _p.rstyle[opts.labelName] || 0;\n default:\n return d.pfValue;\n }\n } else {\n return 1;\n }\n }\n };\n fn$2['outer' + opts.uppercaseName] = function outerDimImpl() {\n var ele = this[0];\n var _p = ele._private;\n var cy = _p.cy;\n var styleEnabled = cy._private.styleEnabled;\n if (ele) {\n if (styleEnabled) {\n var dim = ele[opts.name]();\n var borderPos = ele.pstyle('border-position').value;\n var border;\n if (borderPos === 'center') {\n border = ele.pstyle('border-width').pfValue; // n.b. 1/2 each side\n } else if (borderPos === 'outside') {\n border = 2 * ele.pstyle('border-width').pfValue;\n } else {\n // 'inside'\n border = 0;\n }\n var padding = 2 * ele.padding();\n return dim + border + padding;\n } else {\n return 1;\n }\n }\n };\n fn$2['rendered' + opts.uppercaseName] = function renderedDimImpl() {\n var ele = this[0];\n if (ele) {\n var d = ele[opts.name]();\n return d * this.cy().zoom();\n }\n };\n fn$2['rendered' + opts.uppercaseOuterName] = function renderedOuterDimImpl() {\n var ele = this[0];\n if (ele) {\n var od = ele[opts.outerName]();\n return od * this.cy().zoom();\n }\n };\n};\ndefineDimFns({\n name: 'width'\n});\ndefineDimFns({\n name: 'height'\n});\nelesfn$a.padding = function () {\n var ele = this[0];\n var _p = ele._private;\n if (ele.isParent()) {\n ele.updateCompoundBounds();\n if (_p.autoPadding !== undefined) {\n return _p.autoPadding;\n } else {\n return ele.pstyle('padding').pfValue;\n }\n } else {\n return ele.pstyle('padding').pfValue;\n }\n};\nelesfn$a.paddedHeight = function () {\n var ele = this[0];\n return ele.height() + 2 * ele.padding();\n};\nelesfn$a.paddedWidth = function () {\n var ele = this[0];\n return ele.width() + 2 * ele.padding();\n};\nvar widthHeight = elesfn$a;\n\nvar ifEdge = function ifEdge(ele, getValue) {\n if (ele.isEdge() && ele.takesUpSpace()) {\n return getValue(ele);\n }\n};\nvar ifEdgeRenderedPosition = function ifEdgeRenderedPosition(ele, getPoint) {\n if (ele.isEdge() && ele.takesUpSpace()) {\n var cy = ele.cy();\n return modelToRenderedPosition$1(getPoint(ele), cy.zoom(), cy.pan());\n }\n};\nvar ifEdgeRenderedPositions = function ifEdgeRenderedPositions(ele, getPoints) {\n if (ele.isEdge() && ele.takesUpSpace()) {\n var cy = ele.cy();\n var pan = cy.pan();\n var zoom = cy.zoom();\n return getPoints(ele).map(function (p) {\n return modelToRenderedPosition$1(p, zoom, pan);\n });\n }\n};\nvar controlPoints = function controlPoints(ele) {\n return ele.renderer().getControlPoints(ele);\n};\nvar segmentPoints = function segmentPoints(ele) {\n return ele.renderer().getSegmentPoints(ele);\n};\nvar sourceEndpoint = function sourceEndpoint(ele) {\n return ele.renderer().getSourceEndpoint(ele);\n};\nvar targetEndpoint = function targetEndpoint(ele) {\n return ele.renderer().getTargetEndpoint(ele);\n};\nvar midpoint = function midpoint(ele) {\n return ele.renderer().getEdgeMidpoint(ele);\n};\nvar pts = {\n controlPoints: {\n get: controlPoints,\n mult: true\n },\n segmentPoints: {\n get: segmentPoints,\n mult: true\n },\n sourceEndpoint: {\n get: sourceEndpoint\n },\n targetEndpoint: {\n get: targetEndpoint\n },\n midpoint: {\n get: midpoint\n }\n};\nvar renderedName = function renderedName(name) {\n return 'rendered' + name[0].toUpperCase() + name.substr(1);\n};\nvar edgePoints = Object.keys(pts).reduce(function (obj, name) {\n var spec = pts[name];\n var rName = renderedName(name);\n obj[name] = function () {\n return ifEdge(this, spec.get);\n };\n if (spec.mult) {\n obj[rName] = function () {\n return ifEdgeRenderedPositions(this, spec.get);\n };\n } else {\n obj[rName] = function () {\n return ifEdgeRenderedPosition(this, spec.get);\n };\n }\n return obj;\n}, {});\n\nvar dimensions = extend({}, position, bounds, widthHeight, edgePoints);\n\n/*!\nEvent object based on jQuery events, MIT license\n\nhttps://jquery.org/license/\nhttps://tldrlegal.com/license/mit-license\nhttps://github.com/jquery/jquery/blob/master/src/event.js\n*/\n\nvar Event = function Event(src, props) {\n this.recycle(src, props);\n};\nfunction returnFalse() {\n return false;\n}\nfunction returnTrue() {\n return true;\n}\n\n// http://www.w3.org/TR/2003/WD-DOM-Level-3-Events-20030331/ecma-script-binding.html\nEvent.prototype = {\n instanceString: function instanceString() {\n return 'event';\n },\n recycle: function recycle(src, props) {\n this.isImmediatePropagationStopped = this.isPropagationStopped = this.isDefaultPrevented = returnFalse;\n if (src != null && src.preventDefault) {\n // Browser Event object\n this.type = src.type;\n\n // Events bubbling up the document may have been marked as prevented\n // by a handler lower down the tree; reflect the correct value.\n this.isDefaultPrevented = src.defaultPrevented ? returnTrue : returnFalse;\n } else if (src != null && src.type) {\n // Plain object containing all event details\n props = src;\n } else {\n // Event string\n this.type = src;\n }\n\n // Put explicitly provided properties onto the event object\n if (props != null) {\n // more efficient to manually copy fields we use\n this.originalEvent = props.originalEvent;\n this.type = props.type != null ? props.type : this.type;\n this.cy = props.cy;\n this.target = props.target;\n this.position = props.position;\n this.renderedPosition = props.renderedPosition;\n this.namespace = props.namespace;\n this.layout = props.layout;\n }\n if (this.cy != null && this.position != null && this.renderedPosition == null) {\n // create a rendered position based on the passed position\n var pos = this.position;\n var zoom = this.cy.zoom();\n var pan = this.cy.pan();\n this.renderedPosition = {\n x: pos.x * zoom + pan.x,\n y: pos.y * zoom + pan.y\n };\n }\n\n // Create a timestamp if incoming event doesn't have one\n this.timeStamp = src && src.timeStamp || Date.now();\n },\n preventDefault: function preventDefault() {\n this.isDefaultPrevented = returnTrue;\n var e = this.originalEvent;\n if (!e) {\n return;\n }\n\n // if preventDefault exists run it on the original event\n if (e.preventDefault) {\n e.preventDefault();\n }\n },\n stopPropagation: function stopPropagation() {\n this.isPropagationStopped = returnTrue;\n var e = this.originalEvent;\n if (!e) {\n return;\n }\n\n // if stopPropagation exists run it on the original event\n if (e.stopPropagation) {\n e.stopPropagation();\n }\n },\n stopImmediatePropagation: function stopImmediatePropagation() {\n this.isImmediatePropagationStopped = returnTrue;\n this.stopPropagation();\n },\n isDefaultPrevented: returnFalse,\n isPropagationStopped: returnFalse,\n isImmediatePropagationStopped: returnFalse\n};\n\nvar eventRegex = /^([^.]+)(\\.(?:[^.]+))?$/; // regex for matching event strings (e.g. \"click.namespace\")\nvar universalNamespace = '.*'; // matches as if no namespace specified and prevents users from unbinding accidentally\n\nvar defaults$8 = {\n qualifierCompare: function qualifierCompare(q1, q2) {\n return q1 === q2;\n },\n eventMatches: function eventMatches(/*context, listener, eventObj*/\n ) {\n return true;\n },\n addEventFields: function addEventFields(/*context, evt*/\n ) {},\n callbackContext: function callbackContext(context /*, listener, eventObj*/) {\n return context;\n },\n beforeEmit: function beforeEmit(/* context, listener, eventObj */\n ) {},\n afterEmit: function afterEmit(/* context, listener, eventObj */\n ) {},\n bubble: function bubble(/*context*/\n ) {\n return false;\n },\n parent: function parent(/*context*/\n ) {\n return null;\n },\n context: null\n};\nvar defaultsKeys = Object.keys(defaults$8);\nvar emptyOpts = {};\nfunction Emitter() {\n var opts = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : emptyOpts;\n var context = arguments.length > 1 ? arguments[1] : undefined;\n // micro-optimisation vs Object.assign() -- reduces Element instantiation time\n for (var i = 0; i < defaultsKeys.length; i++) {\n var key = defaultsKeys[i];\n this[key] = opts[key] || defaults$8[key];\n }\n this.context = context || this.context;\n this.listeners = [];\n this.emitting = 0;\n}\nvar p = Emitter.prototype;\nvar forEachEvent = function forEachEvent(self, handler, events, qualifier, callback, conf, confOverrides) {\n if (fn$6(qualifier)) {\n callback = qualifier;\n qualifier = null;\n }\n if (confOverrides) {\n if (conf == null) {\n conf = confOverrides;\n } else {\n conf = extend({}, conf, confOverrides);\n }\n }\n var eventList = array(events) ? events : events.split(/\\s+/);\n for (var i = 0; i < eventList.length; i++) {\n var evt = eventList[i];\n if (emptyString(evt)) {\n continue;\n }\n var match = evt.match(eventRegex); // type[.namespace]\n\n if (match) {\n var type = match[1];\n var namespace = match[2] ? match[2] : null;\n var ret = handler(self, evt, type, namespace, qualifier, callback, conf);\n if (ret === false) {\n break;\n } // allow exiting early\n }\n }\n};\nvar makeEventObj = function makeEventObj(self, obj) {\n self.addEventFields(self.context, obj);\n return new Event(obj.type, obj);\n};\nvar forEachEventObj = function forEachEventObj(self, handler, events) {\n if (event(events)) {\n handler(self, events);\n return;\n } else if (plainObject(events)) {\n handler(self, makeEventObj(self, events));\n return;\n }\n var eventList = array(events) ? events : events.split(/\\s+/);\n for (var i = 0; i < eventList.length; i++) {\n var evt = eventList[i];\n if (emptyString(evt)) {\n continue;\n }\n var match = evt.match(eventRegex); // type[.namespace]\n\n if (match) {\n var type = match[1];\n var namespace = match[2] ? match[2] : null;\n var eventObj = makeEventObj(self, {\n type: type,\n namespace: namespace,\n target: self.context\n });\n handler(self, eventObj);\n }\n }\n};\np.on = p.addListener = function (events, qualifier, callback, conf, confOverrides) {\n forEachEvent(this, function (self, event, type, namespace, qualifier, callback, conf) {\n if (fn$6(callback)) {\n self.listeners.push({\n event: event,\n // full event string\n callback: callback,\n // callback to run\n type: type,\n // the event type (e.g. 'click')\n namespace: namespace,\n // the event namespace (e.g. \".foo\")\n qualifier: qualifier,\n // a restriction on whether to match this emitter\n conf: conf // additional configuration\n });\n }\n }, events, qualifier, callback, conf, confOverrides);\n return this;\n};\np.one = function (events, qualifier, callback, conf) {\n return this.on(events, qualifier, callback, conf, {\n one: true\n });\n};\np.removeListener = p.off = function (events, qualifier, callback, conf) {\n var _this = this;\n if (this.emitting !== 0) {\n this.listeners = copyArray(this.listeners);\n }\n var listeners = this.listeners;\n var _loop = function _loop(i) {\n var listener = listeners[i];\n forEachEvent(_this, function (self, event, type, namespace, qualifier, callback /*, conf*/) {\n if ((listener.type === type || events === '*') && (!namespace && listener.namespace !== '.*' || listener.namespace === namespace) && (!qualifier || self.qualifierCompare(listener.qualifier, qualifier)) && (!callback || listener.callback === callback)) {\n listeners.splice(i, 1);\n return false;\n }\n }, events, qualifier, callback, conf);\n };\n for (var i = listeners.length - 1; i >= 0; i--) {\n _loop(i);\n }\n return this;\n};\np.removeAllListeners = function () {\n return this.removeListener('*');\n};\np.emit = p.trigger = function (events, extraParams, manualCallback) {\n var listeners = this.listeners;\n var numListenersBeforeEmit = listeners.length;\n this.emitting++;\n if (!array(extraParams)) {\n extraParams = [extraParams];\n }\n forEachEventObj(this, function (self, eventObj) {\n if (manualCallback != null) {\n listeners = [{\n event: eventObj.event,\n type: eventObj.type,\n namespace: eventObj.namespace,\n callback: manualCallback\n }];\n numListenersBeforeEmit = listeners.length;\n }\n var _loop2 = function _loop2() {\n var listener = listeners[i];\n if (listener.type === eventObj.type && (!listener.namespace || listener.namespace === eventObj.namespace || listener.namespace === universalNamespace) && self.eventMatches(self.context, listener, eventObj)) {\n var args = [eventObj];\n if (extraParams != null) {\n push(args, extraParams);\n }\n self.beforeEmit(self.context, listener, eventObj);\n if (listener.conf && listener.conf.one) {\n self.listeners = self.listeners.filter(function (l) {\n return l !== listener;\n });\n }\n var context = self.callbackContext(self.context, listener, eventObj);\n var ret = listener.callback.apply(context, args);\n self.afterEmit(self.context, listener, eventObj);\n if (ret === false) {\n eventObj.stopPropagation();\n eventObj.preventDefault();\n }\n } // if listener matches\n };\n for (var i = 0; i < numListenersBeforeEmit; i++) {\n _loop2();\n } // for listener\n\n if (self.bubble(self.context) && !eventObj.isPropagationStopped()) {\n self.parent(self.context).emit(eventObj, extraParams);\n }\n }, events);\n this.emitting--;\n return this;\n};\n\nvar emitterOptions$1 = {\n qualifierCompare: function qualifierCompare(selector1, selector2) {\n if (selector1 == null || selector2 == null) {\n return selector1 == null && selector2 == null;\n } else {\n return selector1.sameText(selector2);\n }\n },\n eventMatches: function eventMatches(ele, listener, eventObj) {\n var selector = listener.qualifier;\n if (selector != null) {\n return ele !== eventObj.target && element(eventObj.target) && selector.matches(eventObj.target);\n }\n return true;\n },\n addEventFields: function addEventFields(ele, evt) {\n evt.cy = ele.cy();\n evt.target = ele;\n },\n callbackContext: function callbackContext(ele, listener, eventObj) {\n return listener.qualifier != null ? eventObj.target : ele;\n },\n beforeEmit: function beforeEmit(context, listener /*, eventObj*/) {\n if (listener.conf && listener.conf.once) {\n listener.conf.onceCollection.removeListener(listener.event, listener.qualifier, listener.callback);\n }\n },\n bubble: function bubble() {\n return true;\n },\n parent: function parent(ele) {\n return ele.isChild() ? ele.parent() : ele.cy();\n }\n};\nvar argSelector$1 = function argSelector(arg) {\n if (string(arg)) {\n return new Selector(arg);\n } else {\n return arg;\n }\n};\nvar elesfn$9 = {\n createEmitter: function createEmitter() {\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n var _p = ele._private;\n if (!_p.emitter) {\n _p.emitter = new Emitter(emitterOptions$1, ele);\n }\n }\n return this;\n },\n emitter: function emitter() {\n return this._private.emitter;\n },\n on: function on(events, selector, callback) {\n var argSel = argSelector$1(selector);\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n ele.emitter().on(events, argSel, callback);\n }\n return this;\n },\n removeListener: function removeListener(events, selector, callback) {\n var argSel = argSelector$1(selector);\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n ele.emitter().removeListener(events, argSel, callback);\n }\n return this;\n },\n removeAllListeners: function removeAllListeners() {\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n ele.emitter().removeAllListeners();\n }\n return this;\n },\n one: function one(events, selector, callback) {\n var argSel = argSelector$1(selector);\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n ele.emitter().one(events, argSel, callback);\n }\n return this;\n },\n once: function once(events, selector, callback) {\n var argSel = argSelector$1(selector);\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n ele.emitter().on(events, argSel, callback, {\n once: true,\n onceCollection: this\n });\n }\n },\n emit: function emit(events, extraParams) {\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n ele.emitter().emit(events, extraParams);\n }\n return this;\n },\n emitAndNotify: function emitAndNotify(event, extraParams) {\n // for internal use only\n if (this.length === 0) {\n return;\n } // empty collections don't need to notify anything\n\n // notify renderer\n this.cy().notify(event, this);\n this.emit(event, extraParams);\n return this;\n }\n};\ndefine.eventAliasesOn(elesfn$9);\n\nvar elesfn$8 = {\n nodes: function nodes(selector) {\n return this.filter(function (ele) {\n return ele.isNode();\n }).filter(selector);\n },\n edges: function edges(selector) {\n return this.filter(function (ele) {\n return ele.isEdge();\n }).filter(selector);\n },\n // internal helper to get nodes and edges as separate collections with single iteration over elements\n byGroup: function byGroup() {\n var nodes = this.spawn();\n var edges = this.spawn();\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n if (ele.isNode()) {\n nodes.push(ele);\n } else {\n edges.push(ele);\n }\n }\n return {\n nodes: nodes,\n edges: edges\n };\n },\n filter: function filter(_filter, thisArg) {\n if (_filter === undefined) {\n // check this first b/c it's the most common/performant case\n return this;\n } else if (string(_filter) || elementOrCollection(_filter)) {\n return new Selector(_filter).filter(this);\n } else if (fn$6(_filter)) {\n var filterEles = this.spawn();\n var eles = this;\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n var include = thisArg ? _filter.apply(thisArg, [ele, i, eles]) : _filter(ele, i, eles);\n if (include) {\n filterEles.push(ele);\n }\n }\n return filterEles;\n }\n return this.spawn(); // if not handled by above, give 'em an empty collection\n },\n not: function not(toRemove) {\n if (!toRemove) {\n return this;\n } else {\n if (string(toRemove)) {\n toRemove = this.filter(toRemove);\n }\n var elements = this.spawn();\n for (var i = 0; i < this.length; i++) {\n var element = this[i];\n var remove = toRemove.has(element);\n if (!remove) {\n elements.push(element);\n }\n }\n return elements;\n }\n },\n absoluteComplement: function absoluteComplement() {\n var cy = this.cy();\n return cy.mutableElements().not(this);\n },\n intersect: function intersect(other) {\n // if a selector is specified, then filter by it instead\n if (string(other)) {\n var selector = other;\n return this.filter(selector);\n }\n var elements = this.spawn();\n var col1 = this;\n var col2 = other;\n var col1Smaller = this.length < other.length;\n var colS = col1Smaller ? col1 : col2;\n var colL = col1Smaller ? col2 : col1;\n for (var i = 0; i < colS.length; i++) {\n var ele = colS[i];\n if (colL.has(ele)) {\n elements.push(ele);\n }\n }\n return elements;\n },\n xor: function xor(other) {\n var cy = this._private.cy;\n if (string(other)) {\n other = cy.$(other);\n }\n var elements = this.spawn();\n var col1 = this;\n var col2 = other;\n var add = function add(col, other) {\n for (var i = 0; i < col.length; i++) {\n var ele = col[i];\n var id = ele._private.data.id;\n var inOther = other.hasElementWithId(id);\n if (!inOther) {\n elements.push(ele);\n }\n }\n };\n add(col1, col2);\n add(col2, col1);\n return elements;\n },\n diff: function diff(other) {\n var cy = this._private.cy;\n if (string(other)) {\n other = cy.$(other);\n }\n var left = this.spawn();\n var right = this.spawn();\n var both = this.spawn();\n var col1 = this;\n var col2 = other;\n var add = function add(col, other, retEles) {\n for (var i = 0; i < col.length; i++) {\n var ele = col[i];\n var id = ele._private.data.id;\n var inOther = other.hasElementWithId(id);\n if (inOther) {\n both.merge(ele);\n } else {\n retEles.push(ele);\n }\n }\n };\n add(col1, col2, left);\n add(col2, col1, right);\n return {\n left: left,\n right: right,\n both: both\n };\n },\n add: function add(toAdd) {\n var cy = this._private.cy;\n if (!toAdd) {\n return this;\n }\n if (string(toAdd)) {\n var selector = toAdd;\n toAdd = cy.mutableElements().filter(selector);\n }\n var elements = this.spawnSelf();\n for (var i = 0; i < toAdd.length; i++) {\n var ele = toAdd[i];\n var add = !this.has(ele);\n if (add) {\n elements.push(ele);\n }\n }\n return elements;\n },\n // in place merge on calling collection\n merge: function merge(toAdd) {\n var _p = this._private;\n var cy = _p.cy;\n if (!toAdd) {\n return this;\n }\n if (toAdd && string(toAdd)) {\n var selector = toAdd;\n toAdd = cy.mutableElements().filter(selector);\n }\n var map = _p.map;\n for (var i = 0; i < toAdd.length; i++) {\n var toAddEle = toAdd[i];\n var id = toAddEle._private.data.id;\n var add = !map.has(id);\n if (add) {\n var index = this.length++;\n this[index] = toAddEle;\n map.set(id, {\n ele: toAddEle,\n index: index\n });\n }\n }\n return this; // chaining\n },\n unmergeAt: function unmergeAt(i) {\n var ele = this[i];\n var id = ele.id();\n var _p = this._private;\n var map = _p.map;\n\n // remove ele\n this[i] = undefined;\n map[\"delete\"](id);\n var unmergedLastEle = i === this.length - 1;\n\n // replace empty spot with last ele in collection\n if (this.length > 1 && !unmergedLastEle) {\n var lastEleI = this.length - 1;\n var lastEle = this[lastEleI];\n var lastEleId = lastEle._private.data.id;\n this[lastEleI] = undefined;\n this[i] = lastEle;\n map.set(lastEleId, {\n ele: lastEle,\n index: i\n });\n }\n\n // the collection is now 1 ele smaller\n this.length--;\n return this;\n },\n // remove single ele in place in calling collection\n unmergeOne: function unmergeOne(ele) {\n ele = ele[0];\n var _p = this._private;\n var id = ele._private.data.id;\n var map = _p.map;\n var entry = map.get(id);\n if (!entry) {\n return this; // no need to remove\n }\n var i = entry.index;\n this.unmergeAt(i);\n return this;\n },\n // remove eles in place on calling collection\n unmerge: function unmerge(toRemove) {\n var cy = this._private.cy;\n if (!toRemove) {\n return this;\n }\n if (toRemove && string(toRemove)) {\n var selector = toRemove;\n toRemove = cy.mutableElements().filter(selector);\n }\n for (var i = 0; i < toRemove.length; i++) {\n this.unmergeOne(toRemove[i]);\n }\n return this; // chaining\n },\n unmergeBy: function unmergeBy(toRmFn) {\n for (var i = this.length - 1; i >= 0; i--) {\n var ele = this[i];\n if (toRmFn(ele)) {\n this.unmergeAt(i);\n }\n }\n return this;\n },\n map: function map(mapFn, thisArg) {\n var arr = [];\n var eles = this;\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n var ret = thisArg ? mapFn.apply(thisArg, [ele, i, eles]) : mapFn(ele, i, eles);\n arr.push(ret);\n }\n return arr;\n },\n reduce: function reduce(fn, initialValue) {\n var val = initialValue;\n var eles = this;\n for (var i = 0; i < eles.length; i++) {\n val = fn(val, eles[i], i, eles);\n }\n return val;\n },\n max: function max(valFn, thisArg) {\n var max = -Infinity;\n var maxEle;\n var eles = this;\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n var val = thisArg ? valFn.apply(thisArg, [ele, i, eles]) : valFn(ele, i, eles);\n if (val > max) {\n max = val;\n maxEle = ele;\n }\n }\n return {\n value: max,\n ele: maxEle\n };\n },\n min: function min(valFn, thisArg) {\n var min = Infinity;\n var minEle;\n var eles = this;\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n var val = thisArg ? valFn.apply(thisArg, [ele, i, eles]) : valFn(ele, i, eles);\n if (val < min) {\n min = val;\n minEle = ele;\n }\n }\n return {\n value: min,\n ele: minEle\n };\n }\n};\n\n// aliases\nvar fn$1 = elesfn$8;\nfn$1['u'] = fn$1['|'] = fn$1['+'] = fn$1.union = fn$1.or = fn$1.add;\nfn$1['\\\\'] = fn$1['!'] = fn$1['-'] = fn$1.difference = fn$1.relativeComplement = fn$1.subtract = fn$1.not;\nfn$1['n'] = fn$1['&'] = fn$1['.'] = fn$1.and = fn$1.intersection = fn$1.intersect;\nfn$1['^'] = fn$1['(+)'] = fn$1['(-)'] = fn$1.symmetricDifference = fn$1.symdiff = fn$1.xor;\nfn$1.fnFilter = fn$1.filterFn = fn$1.stdFilter = fn$1.filter;\nfn$1.complement = fn$1.abscomp = fn$1.absoluteComplement;\n\nvar elesfn$7 = {\n isNode: function isNode() {\n return this.group() === 'nodes';\n },\n isEdge: function isEdge() {\n return this.group() === 'edges';\n },\n isLoop: function isLoop() {\n return this.isEdge() && this.source()[0] === this.target()[0];\n },\n isSimple: function isSimple() {\n return this.isEdge() && this.source()[0] !== this.target()[0];\n },\n group: function group() {\n var ele = this[0];\n if (ele) {\n return ele._private.group;\n }\n }\n};\n\n/**\n * Elements are drawn in a specific order based on compound depth (low to high), the element type (nodes above edges),\n * and z-index (low to high). These styles affect how this applies:\n *\n * z-compound-depth: May be `bottom | orphan | auto | top`. The first drawn is `bottom`, then `orphan` which is the\n * same depth as the root of the compound graph, followed by the default value `auto` which draws in order from\n * root to leaves of the compound graph. The last drawn is `top`.\n * z-index-compare: May be `auto | manual`. The default value is `auto` which always draws edges under nodes.\n * `manual` ignores this convention and draws based on the `z-index` value setting.\n * z-index: An integer value that affects the relative draw order of elements. In general, an element with a higher\n * `z-index` will be drawn on top of an element with a lower `z-index`.\n */\nvar zIndexSort = function zIndexSort(a, b) {\n var cy = a.cy();\n var hasCompoundNodes = cy.hasCompoundNodes();\n function getDepth(ele) {\n var style = ele.pstyle('z-compound-depth');\n if (style.value === 'auto') {\n return hasCompoundNodes ? ele.zDepth() : 0;\n } else if (style.value === 'bottom') {\n return -1;\n } else if (style.value === 'top') {\n return MAX_INT$1;\n }\n // 'orphan'\n return 0;\n }\n var depthDiff = getDepth(a) - getDepth(b);\n if (depthDiff !== 0) {\n return depthDiff;\n }\n function getEleDepth(ele) {\n var style = ele.pstyle('z-index-compare');\n if (style.value === 'auto') {\n return ele.isNode() ? 1 : 0;\n }\n // 'manual'\n return 0;\n }\n var eleDiff = getEleDepth(a) - getEleDepth(b);\n if (eleDiff !== 0) {\n return eleDiff;\n }\n var zDiff = a.pstyle('z-index').value - b.pstyle('z-index').value;\n if (zDiff !== 0) {\n return zDiff;\n }\n // compare indices in the core (order added to graph w/ last on top)\n return a.poolIndex() - b.poolIndex();\n};\n\nvar elesfn$6 = {\n forEach: function forEach(fn, thisArg) {\n if (fn$6(fn)) {\n var N = this.length;\n for (var i = 0; i < N; i++) {\n var ele = this[i];\n var ret = thisArg ? fn.apply(thisArg, [ele, i, this]) : fn(ele, i, this);\n if (ret === false) {\n break;\n } // exit each early on return false\n }\n }\n return this;\n },\n toArray: function toArray() {\n var array = [];\n for (var i = 0; i < this.length; i++) {\n array.push(this[i]);\n }\n return array;\n },\n slice: function slice(start, end) {\n var array = [];\n var thisSize = this.length;\n if (end == null) {\n end = thisSize;\n }\n if (start == null) {\n start = 0;\n }\n if (start < 0) {\n start = thisSize + start;\n }\n if (end < 0) {\n end = thisSize + end;\n }\n for (var i = start; i >= 0 && i < end && i < thisSize; i++) {\n array.push(this[i]);\n }\n return this.spawn(array);\n },\n size: function size() {\n return this.length;\n },\n eq: function eq(i) {\n return this[i] || this.spawn();\n },\n first: function first() {\n return this[0] || this.spawn();\n },\n last: function last() {\n return this[this.length - 1] || this.spawn();\n },\n empty: function empty() {\n return this.length === 0;\n },\n nonempty: function nonempty() {\n return !this.empty();\n },\n sort: function sort(sortFn) {\n if (!fn$6(sortFn)) {\n return this;\n }\n var sorted = this.toArray().sort(sortFn);\n return this.spawn(sorted);\n },\n sortByZIndex: function sortByZIndex() {\n return this.sort(zIndexSort);\n },\n zDepth: function zDepth() {\n var ele = this[0];\n if (!ele) {\n return undefined;\n }\n\n // let cy = ele.cy();\n var _p = ele._private;\n var group = _p.group;\n if (group === 'nodes') {\n var depth = _p.data.parent ? ele.parents().size() : 0;\n if (!ele.isParent()) {\n return MAX_INT$1 - 1; // childless nodes always on top\n }\n return depth;\n } else {\n var src = _p.source;\n var tgt = _p.target;\n var srcDepth = src.zDepth();\n var tgtDepth = tgt.zDepth();\n return Math.max(srcDepth, tgtDepth, 0); // depth of deepest parent\n }\n }\n};\nelesfn$6.each = elesfn$6.forEach;\nvar defineSymbolIterator = function defineSymbolIterator() {\n var typeofUndef = \"undefined\" ;\n var isIteratorSupported = (typeof Symbol === \"undefined\" ? \"undefined\" : _typeof(Symbol)) != typeofUndef && _typeof(Symbol.iterator) != typeofUndef;\n if (isIteratorSupported) {\n elesfn$6[Symbol.iterator] = function () {\n var _this = this;\n var entry = {\n value: undefined,\n done: false\n };\n var i = 0;\n var length = this.length;\n return _defineProperty$1({\n next: function next() {\n if (i < length) {\n entry.value = _this[i++];\n } else {\n entry.value = undefined;\n entry.done = true;\n }\n return entry;\n }\n }, Symbol.iterator, function () {\n return this;\n });\n };\n }\n};\ndefineSymbolIterator();\n\nvar getLayoutDimensionOptions = defaults$g({\n nodeDimensionsIncludeLabels: false\n});\nvar elesfn$5 = {\n // Calculates and returns node dimensions { x, y } based on options given\n layoutDimensions: function layoutDimensions(options) {\n options = getLayoutDimensionOptions(options);\n var dims;\n if (!this.takesUpSpace()) {\n dims = {\n w: 0,\n h: 0\n };\n } else if (options.nodeDimensionsIncludeLabels) {\n var bbDim = this.boundingBox();\n dims = {\n w: bbDim.w,\n h: bbDim.h\n };\n } else {\n dims = {\n w: this.outerWidth(),\n h: this.outerHeight()\n };\n }\n\n // sanitise the dimensions for external layouts (avoid division by zero)\n if (dims.w === 0 || dims.h === 0) {\n dims.w = dims.h = 1;\n }\n return dims;\n },\n // using standard layout options, apply position function (w/ or w/o animation)\n layoutPositions: function layoutPositions(layout, options, fn) {\n var nodes = this.nodes().filter(function (n) {\n return !n.isParent();\n });\n var cy = this.cy();\n var layoutEles = options.eles; // nodes & edges\n var getMemoizeKey = function getMemoizeKey(node) {\n return node.id();\n };\n var fnMem = memoize(fn, getMemoizeKey); // memoized version of position function\n\n layout.emit({\n type: 'layoutstart',\n layout: layout\n });\n layout.animations = [];\n var calculateSpacing = function calculateSpacing(spacing, nodesBb, pos) {\n var center = {\n x: nodesBb.x1 + nodesBb.w / 2,\n y: nodesBb.y1 + nodesBb.h / 2\n };\n var spacingVector = {\n // scale from center of bounding box (not necessarily 0,0)\n x: (pos.x - center.x) * spacing,\n y: (pos.y - center.y) * spacing\n };\n return {\n x: center.x + spacingVector.x,\n y: center.y + spacingVector.y\n };\n };\n var useSpacingFactor = options.spacingFactor && options.spacingFactor !== 1;\n var spacingBb = function spacingBb() {\n if (!useSpacingFactor) {\n return null;\n }\n var bb = makeBoundingBox();\n for (var i = 0; i < nodes.length; i++) {\n var node = nodes[i];\n var pos = fnMem(node, i);\n expandBoundingBoxByPoint(bb, pos.x, pos.y);\n }\n return bb;\n };\n var bb = spacingBb();\n var getFinalPos = memoize(function (node, i) {\n var newPos = fnMem(node, i);\n if (useSpacingFactor) {\n var spacing = Math.abs(options.spacingFactor);\n newPos = calculateSpacing(spacing, bb, newPos);\n }\n if (options.transform != null) {\n newPos = options.transform(node, newPos);\n }\n return newPos;\n }, getMemoizeKey);\n if (options.animate) {\n for (var i = 0; i < nodes.length; i++) {\n var node = nodes[i];\n var newPos = getFinalPos(node, i);\n var animateNode = options.animateFilter == null || options.animateFilter(node, i);\n if (animateNode) {\n var ani = node.animation({\n position: newPos,\n duration: options.animationDuration,\n easing: options.animationEasing\n });\n layout.animations.push(ani);\n } else {\n node.position(newPos);\n }\n }\n if (options.fit) {\n var fitAni = cy.animation({\n fit: {\n boundingBox: layoutEles.boundingBoxAt(getFinalPos),\n padding: options.padding\n },\n duration: options.animationDuration,\n easing: options.animationEasing\n });\n layout.animations.push(fitAni);\n } else if (options.zoom !== undefined && options.pan !== undefined) {\n var zoomPanAni = cy.animation({\n zoom: options.zoom,\n pan: options.pan,\n duration: options.animationDuration,\n easing: options.animationEasing\n });\n layout.animations.push(zoomPanAni);\n }\n layout.animations.forEach(function (ani) {\n return ani.play();\n });\n layout.one('layoutready', options.ready);\n layout.emit({\n type: 'layoutready',\n layout: layout\n });\n Promise$1.all(layout.animations.map(function (ani) {\n return ani.promise();\n })).then(function () {\n layout.one('layoutstop', options.stop);\n layout.emit({\n type: 'layoutstop',\n layout: layout\n });\n });\n } else {\n nodes.positions(getFinalPos);\n if (options.fit) {\n cy.fit(options.eles, options.padding);\n }\n if (options.zoom != null) {\n cy.zoom(options.zoom);\n }\n if (options.pan) {\n cy.pan(options.pan);\n }\n layout.one('layoutready', options.ready);\n layout.emit({\n type: 'layoutready',\n layout: layout\n });\n layout.one('layoutstop', options.stop);\n layout.emit({\n type: 'layoutstop',\n layout: layout\n });\n }\n return this; // chaining\n },\n layout: function layout(options) {\n var cy = this.cy();\n return cy.makeLayout(extend({}, options, {\n eles: this\n }));\n }\n};\n\n// aliases:\nelesfn$5.createLayout = elesfn$5.makeLayout = elesfn$5.layout;\n\nfunction styleCache(key, fn, ele) {\n var _p = ele._private;\n var cache = _p.styleCache = _p.styleCache || [];\n var val;\n if ((val = cache[key]) != null) {\n return val;\n } else {\n val = cache[key] = fn(ele);\n return val;\n }\n}\nfunction cacheStyleFunction(key, fn) {\n key = hashString(key);\n return function cachedStyleFunction(ele) {\n return styleCache(key, fn, ele);\n };\n}\nfunction cachePrototypeStyleFunction(key, fn) {\n key = hashString(key);\n var selfFn = function selfFn(ele) {\n return fn.call(ele);\n };\n return function cachedPrototypeStyleFunction() {\n var ele = this[0];\n if (ele) {\n return styleCache(key, selfFn, ele);\n }\n };\n}\nvar elesfn$4 = {\n recalculateRenderedStyle: function recalculateRenderedStyle(useCache) {\n var cy = this.cy();\n var renderer = cy.renderer();\n var styleEnabled = cy.styleEnabled();\n if (renderer && styleEnabled) {\n renderer.recalculateRenderedStyle(this, useCache);\n }\n return this;\n },\n dirtyStyleCache: function dirtyStyleCache() {\n var cy = this.cy();\n var dirty = function dirty(ele) {\n return ele._private.styleCache = null;\n };\n if (cy.hasCompoundNodes()) {\n var eles;\n eles = this.spawnSelf().merge(this.descendants()).merge(this.parents());\n eles.merge(eles.connectedEdges());\n eles.forEach(dirty);\n } else {\n this.forEach(function (ele) {\n dirty(ele);\n ele.connectedEdges().forEach(dirty);\n });\n }\n return this;\n },\n // fully updates (recalculates) the style for the elements\n updateStyle: function updateStyle(notifyRenderer) {\n var cy = this._private.cy;\n if (!cy.styleEnabled()) {\n return this;\n }\n if (cy.batching()) {\n var bEles = cy._private.batchStyleEles;\n bEles.merge(this);\n return this; // chaining and exit early when batching\n }\n var hasCompounds = cy.hasCompoundNodes();\n var updatedEles = this;\n notifyRenderer = notifyRenderer || notifyRenderer === undefined ? true : false;\n if (hasCompounds) {\n // then add everything up and down for compound selector checks\n updatedEles = this.spawnSelf().merge(this.descendants()).merge(this.parents());\n }\n\n // let changedEles = style.apply( updatedEles );\n var changedEles = updatedEles;\n if (notifyRenderer) {\n changedEles.emitAndNotify('style'); // let renderer know we changed style\n } else {\n changedEles.emit('style'); // just fire the event\n }\n updatedEles.forEach(function (ele) {\n return ele._private.styleDirty = true;\n });\n return this; // chaining\n },\n // private: clears dirty flag and recalculates style\n cleanStyle: function cleanStyle() {\n var cy = this.cy();\n if (!cy.styleEnabled()) {\n return;\n }\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n if (ele._private.styleDirty) {\n // n.b. this flag should be set before apply() to avoid potential infinite recursion\n ele._private.styleDirty = false;\n cy.style().apply(ele);\n }\n }\n },\n // get the internal parsed style object for the specified property\n parsedStyle: function parsedStyle(property) {\n var includeNonDefault = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : true;\n var ele = this[0];\n var cy = ele.cy();\n if (!cy.styleEnabled()) {\n return;\n }\n if (ele) {\n // this.cleanStyle();\n\n // Inline the important part of cleanStyle(), for raw performance\n if (ele._private.styleDirty) {\n // n.b. this flag should be set before apply() to avoid potential infinite recursion\n ele._private.styleDirty = false;\n cy.style().apply(ele);\n }\n var overriddenStyle = ele._private.style[property];\n if (overriddenStyle != null) {\n return overriddenStyle;\n } else if (includeNonDefault) {\n return cy.style().getDefaultProperty(property);\n } else {\n return null;\n }\n }\n },\n numericStyle: function numericStyle(property) {\n var ele = this[0];\n if (!ele.cy().styleEnabled()) {\n return;\n }\n if (ele) {\n var pstyle = ele.pstyle(property);\n return pstyle.pfValue !== undefined ? pstyle.pfValue : pstyle.value;\n }\n },\n numericStyleUnits: function numericStyleUnits(property) {\n var ele = this[0];\n if (!ele.cy().styleEnabled()) {\n return;\n }\n if (ele) {\n return ele.pstyle(property).units;\n }\n },\n // get the specified css property as a rendered value (i.e. on-screen value)\n // or get the whole rendered style if no property specified (NB doesn't allow setting)\n renderedStyle: function renderedStyle(property) {\n var cy = this.cy();\n if (!cy.styleEnabled()) {\n return this;\n }\n var ele = this[0];\n if (ele) {\n return cy.style().getRenderedStyle(ele, property);\n }\n },\n // read the calculated css style of the element or override the style (via a bypass)\n style: function style(name, value) {\n var cy = this.cy();\n if (!cy.styleEnabled()) {\n return this;\n }\n var updateTransitions = false;\n var style = cy.style();\n if (plainObject(name)) {\n // then extend the bypass\n var props = name;\n style.applyBypass(this, props, updateTransitions);\n this.emitAndNotify('style'); // let the renderer know we've updated style\n } else if (string(name)) {\n if (value === undefined) {\n // then get the property from the style\n var ele = this[0];\n if (ele) {\n return style.getStylePropertyValue(ele, name);\n } else {\n // empty collection => can't get any value\n return;\n }\n } else {\n // then set the bypass with the property value\n style.applyBypass(this, name, value, updateTransitions);\n this.emitAndNotify('style'); // let the renderer know we've updated style\n }\n } else if (name === undefined) {\n var _ele = this[0];\n if (_ele) {\n return style.getRawStyle(_ele);\n } else {\n // empty collection => can't get any value\n return;\n }\n }\n return this; // chaining\n },\n removeStyle: function removeStyle(names) {\n var cy = this.cy();\n if (!cy.styleEnabled()) {\n return this;\n }\n var updateTransitions = false;\n var style = cy.style();\n var eles = this;\n if (names === undefined) {\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n style.removeAllBypasses(ele, updateTransitions);\n }\n } else {\n names = names.split(/\\s+/);\n for (var _i = 0; _i < eles.length; _i++) {\n var _ele2 = eles[_i];\n style.removeBypasses(_ele2, names, updateTransitions);\n }\n }\n this.emitAndNotify('style'); // let the renderer know we've updated style\n\n return this; // chaining\n },\n show: function show() {\n this.css('display', 'element');\n return this; // chaining\n },\n hide: function hide() {\n this.css('display', 'none');\n return this; // chaining\n },\n effectiveOpacity: function effectiveOpacity() {\n var cy = this.cy();\n if (!cy.styleEnabled()) {\n return 1;\n }\n var hasCompoundNodes = cy.hasCompoundNodes();\n var ele = this[0];\n if (ele) {\n var _p = ele._private;\n var parentOpacity = ele.pstyle('opacity').value;\n if (!hasCompoundNodes) {\n return parentOpacity;\n }\n var parents = !_p.data.parent ? null : ele.parents();\n if (parents) {\n for (var i = 0; i < parents.length; i++) {\n var parent = parents[i];\n var opacity = parent.pstyle('opacity').value;\n parentOpacity = opacity * parentOpacity;\n }\n }\n return parentOpacity;\n }\n },\n transparent: function transparent() {\n var cy = this.cy();\n if (!cy.styleEnabled()) {\n return false;\n }\n var ele = this[0];\n var hasCompoundNodes = ele.cy().hasCompoundNodes();\n if (ele) {\n if (!hasCompoundNodes) {\n return ele.pstyle('opacity').value === 0;\n } else {\n return ele.effectiveOpacity() === 0;\n }\n }\n },\n backgrounding: function backgrounding() {\n var cy = this.cy();\n if (!cy.styleEnabled()) {\n return false;\n }\n var ele = this[0];\n return ele._private.backgrounding ? true : false;\n }\n};\nfunction checkCompound(ele, parentOk) {\n var _p = ele._private;\n var parents = _p.data.parent ? ele.parents() : null;\n if (parents) {\n for (var i = 0; i < parents.length; i++) {\n var parent = parents[i];\n if (!parentOk(parent)) {\n return false;\n }\n }\n }\n return true;\n}\nfunction defineDerivedStateFunction(specs) {\n var ok = specs.ok;\n var edgeOkViaNode = specs.edgeOkViaNode || specs.ok;\n var parentOk = specs.parentOk || specs.ok;\n return function () {\n var cy = this.cy();\n if (!cy.styleEnabled()) {\n return true;\n }\n var ele = this[0];\n var hasCompoundNodes = cy.hasCompoundNodes();\n if (ele) {\n var _p = ele._private;\n if (!ok(ele)) {\n return false;\n }\n if (ele.isNode()) {\n return !hasCompoundNodes || checkCompound(ele, parentOk);\n } else {\n var src = _p.source;\n var tgt = _p.target;\n return edgeOkViaNode(src) && (!hasCompoundNodes || checkCompound(src, edgeOkViaNode)) && (src === tgt || edgeOkViaNode(tgt) && (!hasCompoundNodes || checkCompound(tgt, edgeOkViaNode)));\n }\n }\n };\n}\nvar eleTakesUpSpace = cacheStyleFunction('eleTakesUpSpace', function (ele) {\n return ele.pstyle('display').value === 'element' && ele.width() !== 0 && (ele.isNode() ? ele.height() !== 0 : true);\n});\nelesfn$4.takesUpSpace = cachePrototypeStyleFunction('takesUpSpace', defineDerivedStateFunction({\n ok: eleTakesUpSpace\n}));\nvar eleInteractive = cacheStyleFunction('eleInteractive', function (ele) {\n return ele.pstyle('events').value === 'yes' && ele.pstyle('visibility').value === 'visible' && eleTakesUpSpace(ele);\n});\nvar parentInteractive = cacheStyleFunction('parentInteractive', function (parent) {\n return parent.pstyle('visibility').value === 'visible' && eleTakesUpSpace(parent);\n});\nelesfn$4.interactive = cachePrototypeStyleFunction('interactive', defineDerivedStateFunction({\n ok: eleInteractive,\n parentOk: parentInteractive,\n edgeOkViaNode: eleTakesUpSpace\n}));\nelesfn$4.noninteractive = function () {\n var ele = this[0];\n if (ele) {\n return !ele.interactive();\n }\n};\nvar eleVisible = cacheStyleFunction('eleVisible', function (ele) {\n return ele.pstyle('visibility').value === 'visible' && ele.pstyle('opacity').pfValue !== 0 && eleTakesUpSpace(ele);\n});\nvar edgeVisibleViaNode = eleTakesUpSpace;\nelesfn$4.visible = cachePrototypeStyleFunction('visible', defineDerivedStateFunction({\n ok: eleVisible,\n edgeOkViaNode: edgeVisibleViaNode\n}));\nelesfn$4.hidden = function () {\n var ele = this[0];\n if (ele) {\n return !ele.visible();\n }\n};\nelesfn$4.isBundledBezier = cachePrototypeStyleFunction('isBundledBezier', function () {\n if (!this.cy().styleEnabled()) {\n return false;\n }\n return !this.removed() && this.pstyle('curve-style').value === 'bezier' && this.takesUpSpace();\n});\nelesfn$4.bypass = elesfn$4.css = elesfn$4.style;\nelesfn$4.renderedCss = elesfn$4.renderedStyle;\nelesfn$4.removeBypass = elesfn$4.removeCss = elesfn$4.removeStyle;\nelesfn$4.pstyle = elesfn$4.parsedStyle;\n\nvar elesfn$3 = {};\nfunction defineSwitchFunction(params) {\n return function () {\n var args = arguments;\n var changedEles = [];\n\n // e.g. cy.nodes().select( data, handler )\n if (args.length === 2) {\n var data = args[0];\n var handler = args[1];\n this.on(params.event, data, handler);\n }\n\n // e.g. cy.nodes().select( handler )\n else if (args.length === 1 && fn$6(args[0])) {\n var _handler = args[0];\n this.on(params.event, _handler);\n }\n\n // e.g. cy.nodes().select()\n // e.g. (private) cy.nodes().select(['tapselect'])\n else if (args.length === 0 || args.length === 1 && array(args[0])) {\n var addlEvents = args.length === 1 ? args[0] : null;\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n var able = !params.ableField || ele._private[params.ableField];\n var changed = ele._private[params.field] != params.value;\n if (params.overrideAble) {\n var overrideAble = params.overrideAble(ele);\n if (overrideAble !== undefined) {\n able = overrideAble;\n if (!overrideAble) {\n return this;\n } // to save cycles assume not able for all on override\n }\n }\n if (able) {\n ele._private[params.field] = params.value;\n if (changed) {\n changedEles.push(ele);\n }\n }\n }\n var changedColl = this.spawn(changedEles);\n changedColl.updateStyle(); // change of state => possible change of style\n changedColl.emit(params.event);\n if (addlEvents) {\n changedColl.emit(addlEvents);\n }\n }\n return this;\n };\n}\nfunction defineSwitchSet(params) {\n elesfn$3[params.field] = function () {\n var ele = this[0];\n if (ele) {\n if (params.overrideField) {\n var val = params.overrideField(ele);\n if (val !== undefined) {\n return val;\n }\n }\n return ele._private[params.field];\n }\n };\n elesfn$3[params.on] = defineSwitchFunction({\n event: params.on,\n field: params.field,\n ableField: params.ableField,\n overrideAble: params.overrideAble,\n value: true\n });\n elesfn$3[params.off] = defineSwitchFunction({\n event: params.off,\n field: params.field,\n ableField: params.ableField,\n overrideAble: params.overrideAble,\n value: false\n });\n}\ndefineSwitchSet({\n field: 'locked',\n overrideField: function overrideField(ele) {\n return ele.cy().autolock() ? true : undefined;\n },\n on: 'lock',\n off: 'unlock'\n});\ndefineSwitchSet({\n field: 'grabbable',\n overrideField: function overrideField(ele) {\n return ele.cy().autoungrabify() || ele.pannable() ? false : undefined;\n },\n on: 'grabify',\n off: 'ungrabify'\n});\ndefineSwitchSet({\n field: 'selected',\n ableField: 'selectable',\n overrideAble: function overrideAble(ele) {\n return ele.cy().autounselectify() ? false : undefined;\n },\n on: 'select',\n off: 'unselect'\n});\ndefineSwitchSet({\n field: 'selectable',\n overrideField: function overrideField(ele) {\n return ele.cy().autounselectify() ? false : undefined;\n },\n on: 'selectify',\n off: 'unselectify'\n});\nelesfn$3.deselect = elesfn$3.unselect;\nelesfn$3.grabbed = function () {\n var ele = this[0];\n if (ele) {\n return ele._private.grabbed;\n }\n};\ndefineSwitchSet({\n field: 'active',\n on: 'activate',\n off: 'unactivate'\n});\ndefineSwitchSet({\n field: 'pannable',\n on: 'panify',\n off: 'unpanify'\n});\nelesfn$3.inactive = function () {\n var ele = this[0];\n if (ele) {\n return !ele._private.active;\n }\n};\n\nvar elesfn$2 = {};\n\n// DAG functions\n////////////////\n\nvar defineDagExtremity = function defineDagExtremity(params) {\n return function dagExtremityImpl(selector) {\n var eles = this;\n var ret = [];\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n if (!ele.isNode()) {\n continue;\n }\n var disqualified = false;\n var edges = ele.connectedEdges();\n for (var j = 0; j < edges.length; j++) {\n var edge = edges[j];\n var src = edge.source();\n var tgt = edge.target();\n if (params.noIncomingEdges && tgt === ele && src !== ele || params.noOutgoingEdges && src === ele && tgt !== ele) {\n disqualified = true;\n break;\n }\n }\n if (!disqualified) {\n ret.push(ele);\n }\n }\n return this.spawn(ret, true).filter(selector);\n };\n};\nvar defineDagOneHop = function defineDagOneHop(params) {\n return function (selector) {\n var eles = this;\n var oEles = [];\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n if (!ele.isNode()) {\n continue;\n }\n var edges = ele.connectedEdges();\n for (var j = 0; j < edges.length; j++) {\n var edge = edges[j];\n var src = edge.source();\n var tgt = edge.target();\n if (params.outgoing && src === ele) {\n oEles.push(edge);\n oEles.push(tgt);\n } else if (params.incoming && tgt === ele) {\n oEles.push(edge);\n oEles.push(src);\n }\n }\n }\n return this.spawn(oEles, true).filter(selector);\n };\n};\nvar defineDagAllHops = function defineDagAllHops(params) {\n return function (selector) {\n var eles = this;\n var sEles = [];\n var sElesIds = {};\n for (;;) {\n var next = params.outgoing ? eles.outgoers() : eles.incomers();\n if (next.length === 0) {\n break;\n } // done if none left\n\n var newNext = false;\n for (var i = 0; i < next.length; i++) {\n var n = next[i];\n var nid = n.id();\n if (!sElesIds[nid]) {\n sElesIds[nid] = true;\n sEles.push(n);\n newNext = true;\n }\n }\n if (!newNext) {\n break;\n } // done if touched all outgoers already\n\n eles = next;\n }\n return this.spawn(sEles, true).filter(selector);\n };\n};\nelesfn$2.clearTraversalCache = function () {\n for (var i = 0; i < this.length; i++) {\n this[i]._private.traversalCache = null;\n }\n};\nextend(elesfn$2, {\n // get the root nodes in the DAG\n roots: defineDagExtremity({\n noIncomingEdges: true\n }),\n // get the leaf nodes in the DAG\n leaves: defineDagExtremity({\n noOutgoingEdges: true\n }),\n // normally called children in graph theory\n // these nodes =edges=> outgoing nodes\n outgoers: cache(defineDagOneHop({\n outgoing: true\n }), 'outgoers'),\n // aka DAG descendants\n successors: defineDagAllHops({\n outgoing: true\n }),\n // normally called parents in graph theory\n // these nodes <=edges= incoming nodes\n incomers: cache(defineDagOneHop({\n incoming: true\n }), 'incomers'),\n // aka DAG ancestors\n predecessors: defineDagAllHops({\n })\n});\n\n// Neighbourhood functions\n//////////////////////////\n\nextend(elesfn$2, {\n neighborhood: cache(function (selector) {\n var elements = [];\n var nodes = this.nodes();\n for (var i = 0; i < nodes.length; i++) {\n // for all nodes\n var node = nodes[i];\n var connectedEdges = node.connectedEdges();\n\n // for each connected edge, add the edge and the other node\n for (var j = 0; j < connectedEdges.length; j++) {\n var edge = connectedEdges[j];\n var src = edge.source();\n var tgt = edge.target();\n var otherNode = node === src ? tgt : src;\n\n // need check in case of loop\n if (otherNode.length > 0) {\n elements.push(otherNode[0]); // add node 1 hop away\n }\n\n // add connected edge\n elements.push(edge[0]);\n }\n }\n return this.spawn(elements, true).filter(selector);\n }, 'neighborhood'),\n closedNeighborhood: function closedNeighborhood(selector) {\n return this.neighborhood().add(this).filter(selector);\n },\n openNeighborhood: function openNeighborhood(selector) {\n return this.neighborhood(selector);\n }\n});\n\n// aliases\nelesfn$2.neighbourhood = elesfn$2.neighborhood;\nelesfn$2.closedNeighbourhood = elesfn$2.closedNeighborhood;\nelesfn$2.openNeighbourhood = elesfn$2.openNeighborhood;\n\n// Edge functions\n/////////////////\n\nextend(elesfn$2, {\n source: cache(function sourceImpl(selector) {\n var ele = this[0];\n var src;\n if (ele) {\n src = ele._private.source || ele.cy().collection();\n }\n return src && selector ? src.filter(selector) : src;\n }, 'source'),\n target: cache(function targetImpl(selector) {\n var ele = this[0];\n var tgt;\n if (ele) {\n tgt = ele._private.target || ele.cy().collection();\n }\n return tgt && selector ? tgt.filter(selector) : tgt;\n }, 'target'),\n sources: defineSourceFunction({\n attr: 'source'\n }),\n targets: defineSourceFunction({\n attr: 'target'\n })\n});\nfunction defineSourceFunction(params) {\n return function sourceImpl(selector) {\n var sources = [];\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n var src = ele._private[params.attr];\n if (src) {\n sources.push(src);\n }\n }\n return this.spawn(sources, true).filter(selector);\n };\n}\nextend(elesfn$2, {\n edgesWith: cache(defineEdgesWithFunction(), 'edgesWith'),\n edgesTo: cache(defineEdgesWithFunction({\n thisIsSrc: true\n }), 'edgesTo')\n});\nfunction defineEdgesWithFunction(params) {\n return function edgesWithImpl(otherNodes) {\n var elements = [];\n var cy = this._private.cy;\n var p = params || {};\n\n // get elements if a selector is specified\n if (string(otherNodes)) {\n otherNodes = cy.$(otherNodes);\n }\n for (var h = 0; h < otherNodes.length; h++) {\n var edges = otherNodes[h]._private.edges;\n for (var i = 0; i < edges.length; i++) {\n var edge = edges[i];\n var edgeData = edge._private.data;\n var thisToOther = this.hasElementWithId(edgeData.source) && otherNodes.hasElementWithId(edgeData.target);\n var otherToThis = otherNodes.hasElementWithId(edgeData.source) && this.hasElementWithId(edgeData.target);\n var edgeConnectsThisAndOther = thisToOther || otherToThis;\n if (!edgeConnectsThisAndOther) {\n continue;\n }\n if (p.thisIsSrc || p.thisIsTgt) {\n if (p.thisIsSrc && !thisToOther) {\n continue;\n }\n if (p.thisIsTgt && !otherToThis) {\n continue;\n }\n }\n elements.push(edge);\n }\n }\n return this.spawn(elements, true);\n };\n}\nextend(elesfn$2, {\n connectedEdges: cache(function (selector) {\n var retEles = [];\n var eles = this;\n for (var i = 0; i < eles.length; i++) {\n var node = eles[i];\n if (!node.isNode()) {\n continue;\n }\n var edges = node._private.edges;\n for (var j = 0; j < edges.length; j++) {\n var edge = edges[j];\n retEles.push(edge);\n }\n }\n return this.spawn(retEles, true).filter(selector);\n }, 'connectedEdges'),\n connectedNodes: cache(function (selector) {\n var retEles = [];\n var eles = this;\n for (var i = 0; i < eles.length; i++) {\n var edge = eles[i];\n if (!edge.isEdge()) {\n continue;\n }\n retEles.push(edge.source()[0]);\n retEles.push(edge.target()[0]);\n }\n return this.spawn(retEles, true).filter(selector);\n }, 'connectedNodes'),\n parallelEdges: cache(defineParallelEdgesFunction(), 'parallelEdges'),\n codirectedEdges: cache(defineParallelEdgesFunction({\n codirected: true\n }), 'codirectedEdges')\n});\nfunction defineParallelEdgesFunction(params) {\n var defaults = {\n codirected: false\n };\n params = extend({}, defaults, params);\n return function parallelEdgesImpl(selector) {\n // micro-optimised for renderer\n var elements = [];\n var edges = this.edges();\n var p = params;\n\n // look at all the edges in the collection\n for (var i = 0; i < edges.length; i++) {\n var edge1 = edges[i];\n var edge1_p = edge1._private;\n var src1 = edge1_p.source;\n var srcid1 = src1._private.data.id;\n var tgtid1 = edge1_p.data.target;\n var srcEdges1 = src1._private.edges;\n\n // look at edges connected to the src node of this edge\n for (var j = 0; j < srcEdges1.length; j++) {\n var edge2 = srcEdges1[j];\n var edge2data = edge2._private.data;\n var tgtid2 = edge2data.target;\n var srcid2 = edge2data.source;\n var codirected = tgtid2 === tgtid1 && srcid2 === srcid1;\n var oppdirected = srcid1 === tgtid2 && tgtid1 === srcid2;\n if (p.codirected && codirected || !p.codirected && (codirected || oppdirected)) {\n elements.push(edge2);\n }\n }\n }\n return this.spawn(elements, true).filter(selector);\n };\n}\n\n// Misc functions\n/////////////////\n\nextend(elesfn$2, {\n components: function components(root) {\n var self = this;\n var cy = self.cy();\n var visited = cy.collection();\n var unvisited = root == null ? self.nodes() : root.nodes();\n var components = [];\n if (root != null && unvisited.empty()) {\n // root may contain only edges\n unvisited = root.sources(); // doesn't matter which node to use (undirected), so just use the source sides\n }\n var visitInComponent = function visitInComponent(node, component) {\n visited.merge(node);\n unvisited.unmerge(node);\n component.merge(node);\n };\n if (unvisited.empty()) {\n return self.spawn();\n }\n var _loop = function _loop() {\n // each iteration yields a component\n var cmpt = cy.collection();\n components.push(cmpt);\n var root = unvisited[0];\n visitInComponent(root, cmpt);\n self.bfs({\n directed: false,\n roots: root,\n visit: function visit(v) {\n return visitInComponent(v, cmpt);\n }\n });\n cmpt.forEach(function (node) {\n node.connectedEdges().forEach(function (e) {\n // connectedEdges() usually cached\n if (self.has(e) && cmpt.has(e.source()) && cmpt.has(e.target())) {\n // has() is cheap\n cmpt.merge(e); // forEach() only considers nodes -- sets N at call time\n }\n });\n });\n };\n do {\n _loop();\n } while (unvisited.length > 0);\n return components;\n },\n component: function component() {\n var ele = this[0];\n return ele.cy().mutableElements().components(ele)[0];\n }\n});\nelesfn$2.componentsOf = elesfn$2.components;\n\n// represents a set of nodes, edges, or both together\nvar Collection = function Collection(cy, elements) {\n var unique = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : false;\n var removed = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : false;\n if (cy === undefined) {\n error('A collection must have a reference to the core');\n return;\n }\n var map = new Map$1();\n var createdElements = false;\n if (!elements) {\n elements = [];\n } else if (elements.length > 0 && plainObject(elements[0]) && !element(elements[0])) {\n createdElements = true;\n\n // make elements from json and restore all at once later\n var eles = [];\n var elesIds = new Set$1();\n for (var i = 0, l = elements.length; i < l; i++) {\n var json = elements[i];\n if (json.data == null) {\n json.data = {};\n }\n var _data = json.data;\n\n // make sure newly created elements have valid ids\n if (_data.id == null) {\n _data.id = uuid();\n } else if (cy.hasElementWithId(_data.id) || elesIds.has(_data.id)) {\n continue; // can't create element if prior id already exists\n }\n var ele = new Element(cy, json, false);\n eles.push(ele);\n elesIds.add(_data.id);\n }\n elements = eles;\n }\n this.length = 0;\n for (var _i = 0, _l = elements.length; _i < _l; _i++) {\n var element$1 = elements[_i][0]; // [0] in case elements is an array of collections, rather than array of elements\n if (element$1 == null) {\n continue;\n }\n var id = element$1._private.data.id;\n if (!unique || !map.has(id)) {\n if (unique) {\n map.set(id, {\n index: this.length,\n ele: element$1\n });\n }\n this[this.length] = element$1;\n this.length++;\n }\n }\n this._private = {\n eles: this,\n cy: cy,\n get map() {\n if (this.lazyMap == null) {\n this.rebuildMap();\n }\n return this.lazyMap;\n },\n set map(m) {\n this.lazyMap = m;\n },\n rebuildMap: function rebuildMap() {\n var m = this.lazyMap = new Map$1();\n var eles = this.eles;\n for (var _i2 = 0; _i2 < eles.length; _i2++) {\n var _ele = eles[_i2];\n m.set(_ele.id(), {\n index: _i2,\n ele: _ele\n });\n }\n }\n };\n if (unique) {\n this._private.map = map;\n }\n\n // restore the elements if we created them from json\n if (createdElements && !removed) {\n this.restore();\n }\n};\n\n// Functions\n////////////////////////////////////////////////////////////////////////////////////////////////////\n\n// keep the prototypes in sync (an element has the same functions as a collection)\n// and use elefn and elesfn as shorthands to the prototypes\nvar elesfn$1 = Element.prototype = Collection.prototype = Object.create(Array.prototype);\nelesfn$1.instanceString = function () {\n return 'collection';\n};\nelesfn$1.spawn = function (eles, unique) {\n return new Collection(this.cy(), eles, unique);\n};\nelesfn$1.spawnSelf = function () {\n return this.spawn(this);\n};\nelesfn$1.cy = function () {\n return this._private.cy;\n};\nelesfn$1.renderer = function () {\n return this._private.cy.renderer();\n};\nelesfn$1.element = function () {\n return this[0];\n};\nelesfn$1.collection = function () {\n if (collection(this)) {\n return this;\n } else {\n // an element\n return new Collection(this._private.cy, [this]);\n }\n};\nelesfn$1.unique = function () {\n return new Collection(this._private.cy, this, true);\n};\nelesfn$1.hasElementWithId = function (id) {\n id = '' + id; // id must be string\n\n return this._private.map.has(id);\n};\nelesfn$1.getElementById = function (id) {\n id = '' + id; // id must be string\n\n var cy = this._private.cy;\n var entry = this._private.map.get(id);\n return entry ? entry.ele : new Collection(cy); // get ele or empty collection\n};\nelesfn$1.$id = elesfn$1.getElementById;\nelesfn$1.poolIndex = function () {\n var cy = this._private.cy;\n var eles = cy._private.elements;\n var id = this[0]._private.data.id;\n return eles._private.map.get(id).index;\n};\nelesfn$1.indexOf = function (ele) {\n var id = ele[0]._private.data.id;\n return this._private.map.get(id).index;\n};\nelesfn$1.indexOfId = function (id) {\n id = '' + id; // id must be string\n\n return this._private.map.get(id).index;\n};\nelesfn$1.json = function (obj) {\n var ele = this.element();\n var cy = this.cy();\n if (ele == null && obj) {\n return this;\n } // can't set to no eles\n\n if (ele == null) {\n return undefined;\n } // can't get from no eles\n\n var p = ele._private;\n if (plainObject(obj)) {\n // set\n\n cy.startBatch();\n if (obj.data) {\n ele.data(obj.data);\n var _data2 = p.data;\n if (ele.isEdge()) {\n // source and target are immutable via data()\n var move = false;\n var spec = {};\n var src = obj.data.source;\n var tgt = obj.data.target;\n if (src != null && src != _data2.source) {\n spec.source = '' + src; // id must be string\n move = true;\n }\n if (tgt != null && tgt != _data2.target) {\n spec.target = '' + tgt; // id must be string\n move = true;\n }\n if (move) {\n ele = ele.move(spec);\n }\n } else {\n // parent is immutable via data()\n var newParentValSpecd = 'parent' in obj.data;\n var parent = obj.data.parent;\n if (newParentValSpecd && (parent != null || _data2.parent != null) && parent != _data2.parent) {\n if (parent === undefined) {\n // can't set undefined imperatively, so use null\n parent = null;\n }\n if (parent != null) {\n parent = '' + parent; // id must be string\n }\n ele = ele.move({\n parent: parent\n });\n }\n }\n }\n if (obj.position) {\n ele.position(obj.position);\n }\n\n // ignore group -- immutable\n\n var checkSwitch = function checkSwitch(k, trueFnName, falseFnName) {\n var obj_k = obj[k];\n if (obj_k != null && obj_k !== p[k]) {\n if (obj_k) {\n ele[trueFnName]();\n } else {\n ele[falseFnName]();\n }\n }\n };\n checkSwitch('removed', 'remove', 'restore');\n checkSwitch('selected', 'select', 'unselect');\n checkSwitch('selectable', 'selectify', 'unselectify');\n checkSwitch('locked', 'lock', 'unlock');\n checkSwitch('grabbable', 'grabify', 'ungrabify');\n checkSwitch('pannable', 'panify', 'unpanify');\n if (obj.classes != null) {\n ele.classes(obj.classes);\n }\n cy.endBatch();\n return this;\n } else if (obj === undefined) {\n // get\n\n var json = {\n data: copy(p.data),\n position: copy(p.position),\n group: p.group,\n removed: p.removed,\n selected: p.selected,\n selectable: p.selectable,\n locked: p.locked,\n grabbable: p.grabbable,\n pannable: p.pannable,\n classes: null\n };\n json.classes = '';\n var i = 0;\n p.classes.forEach(function (cls) {\n return json.classes += i++ === 0 ? cls : ' ' + cls;\n });\n return json;\n }\n};\nelesfn$1.jsons = function () {\n var jsons = [];\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n var json = ele.json();\n jsons.push(json);\n }\n return jsons;\n};\nelesfn$1.clone = function () {\n var cy = this.cy();\n var elesArr = [];\n for (var i = 0; i < this.length; i++) {\n var ele = this[i];\n var json = ele.json();\n var clone = new Element(cy, json, false); // NB no restore\n\n elesArr.push(clone);\n }\n return new Collection(cy, elesArr);\n};\nelesfn$1.copy = elesfn$1.clone;\nelesfn$1.restore = function () {\n var notifyRenderer = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : true;\n var addToPool = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : true;\n var self = this;\n var cy = self.cy();\n var cy_p = cy._private;\n\n // create arrays of nodes and edges, since we need to\n // restore the nodes first\n var nodes = [];\n var edges = [];\n var elements;\n for (var _i3 = 0, l = self.length; _i3 < l; _i3++) {\n var ele = self[_i3];\n if (addToPool && !ele.removed()) {\n // don't need to handle this ele\n continue;\n }\n\n // keep nodes first in the array and edges after\n if (ele.isNode()) {\n // put to front of array if node\n nodes.push(ele);\n } else {\n // put to end of array if edge\n edges.push(ele);\n }\n }\n elements = nodes.concat(edges);\n var i;\n var removeFromElements = function removeFromElements() {\n elements.splice(i, 1);\n i--;\n };\n\n // now, restore each element\n for (i = 0; i < elements.length; i++) {\n var _ele2 = elements[i];\n var _private = _ele2._private;\n var _data3 = _private.data;\n\n // the traversal cache should start fresh when ele is added\n _ele2.clearTraversalCache();\n\n // set id and validate\n if (!addToPool && !_private.removed) ; else if (_data3.id === undefined) {\n _data3.id = uuid();\n } else if (number$1(_data3.id)) {\n _data3.id = '' + _data3.id; // now it's a string\n } else if (emptyString(_data3.id) || !string(_data3.id)) {\n error('Can not create element with invalid string ID `' + _data3.id + '`');\n\n // can't create element if it has empty string as id or non-string id\n removeFromElements();\n continue;\n } else if (cy.hasElementWithId(_data3.id)) {\n error('Can not create second element with ID `' + _data3.id + '`');\n\n // can't create element if one already has that id\n removeFromElements();\n continue;\n }\n var id = _data3.id; // id is finalised, now let's keep a ref\n\n if (_ele2.isNode()) {\n // extra checks for nodes\n var pos = _private.position;\n\n // make sure the nodes have a defined position\n\n if (pos.x == null) {\n pos.x = 0;\n }\n if (pos.y == null) {\n pos.y = 0;\n }\n }\n if (_ele2.isEdge()) {\n // extra checks for edges\n\n var edge = _ele2;\n var fields = ['source', 'target'];\n var fieldsLength = fields.length;\n var badSourceOrTarget = false;\n for (var j = 0; j < fieldsLength; j++) {\n var field = fields[j];\n var val = _data3[field];\n if (number$1(val)) {\n val = _data3[field] = '' + _data3[field]; // now string\n }\n if (val == null || val === '') {\n // can't create if source or target is not defined properly\n error('Can not create edge `' + id + '` with unspecified ' + field);\n badSourceOrTarget = true;\n } else if (!cy.hasElementWithId(val)) {\n // can't create edge if one of its nodes doesn't exist\n error('Can not create edge `' + id + '` with nonexistent ' + field + ' `' + val + '`');\n badSourceOrTarget = true;\n }\n }\n if (badSourceOrTarget) {\n removeFromElements();\n continue;\n } // can't create this\n\n var src = cy.getElementById(_data3.source);\n var tgt = cy.getElementById(_data3.target);\n\n // only one edge in node if loop\n if (src.same(tgt)) {\n src._private.edges.push(edge);\n } else {\n src._private.edges.push(edge);\n tgt._private.edges.push(edge);\n }\n edge._private.source = src;\n edge._private.target = tgt;\n } // if is edge\n\n // create mock ids / indexes maps for element so it can be used like collections\n _private.map = new Map$1();\n _private.map.set(id, {\n ele: _ele2,\n index: 0\n });\n _private.removed = false;\n if (addToPool) {\n cy.addToPool(_ele2);\n }\n } // for each element\n\n // do compound node sanity checks\n for (var _i4 = 0; _i4 < nodes.length; _i4++) {\n // each node\n var node = nodes[_i4];\n var _data4 = node._private.data;\n if (number$1(_data4.parent)) {\n // then automake string\n _data4.parent = '' + _data4.parent;\n }\n var parentId = _data4.parent;\n var specifiedParent = parentId != null;\n if (specifiedParent || node._private.parent) {\n var parent = node._private.parent ? cy.collection().merge(node._private.parent) : cy.getElementById(parentId);\n if (parent.empty()) {\n // non-existant parent; just remove it\n _data4.parent = undefined;\n } else if (parent[0].removed()) {\n warn('Node added with missing parent, reference to parent removed');\n _data4.parent = undefined;\n node._private.parent = null;\n } else {\n var selfAsParent = false;\n var ancestor = parent;\n while (!ancestor.empty()) {\n if (node.same(ancestor)) {\n // mark self as parent and remove from data\n selfAsParent = true;\n _data4.parent = undefined; // remove parent reference\n\n // exit or we loop forever\n break;\n }\n ancestor = ancestor.parent();\n }\n if (!selfAsParent) {\n // connect with children\n parent[0]._private.children.push(node);\n node._private.parent = parent[0];\n\n // let the core know we have a compound graph\n cy_p.hasCompoundNodes = true;\n }\n } // else\n } // if specified parent\n } // for each node\n\n if (elements.length > 0) {\n var restored = elements.length === self.length ? self : new Collection(cy, elements);\n for (var _i5 = 0; _i5 < restored.length; _i5++) {\n var _ele3 = restored[_i5];\n if (_ele3.isNode()) {\n continue;\n }\n\n // adding an edge invalidates the traversal caches for the parallel edges\n _ele3.parallelEdges().clearTraversalCache();\n\n // adding an edge invalidates the traversal cache for the connected nodes\n _ele3.source().clearTraversalCache();\n _ele3.target().clearTraversalCache();\n }\n var toUpdateStyle;\n if (cy_p.hasCompoundNodes) {\n toUpdateStyle = cy.collection().merge(restored).merge(restored.connectedNodes()).merge(restored.parent());\n } else {\n toUpdateStyle = restored;\n }\n toUpdateStyle.dirtyCompoundBoundsCache().dirtyBoundingBoxCache().updateStyle(notifyRenderer);\n if (notifyRenderer) {\n restored.emitAndNotify('add');\n } else if (addToPool) {\n restored.emit('add');\n }\n }\n return self; // chainability\n};\nelesfn$1.removed = function () {\n var ele = this[0];\n return ele && ele._private.removed;\n};\nelesfn$1.inside = function () {\n var ele = this[0];\n return ele && !ele._private.removed;\n};\nelesfn$1.remove = function () {\n var notifyRenderer = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : true;\n var removeFromPool = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : true;\n var self = this;\n var elesToRemove = [];\n var elesToRemoveIds = {};\n var cy = self._private.cy;\n\n // add connected edges\n function addConnectedEdges(node) {\n var edges = node._private.edges;\n for (var i = 0; i < edges.length; i++) {\n add(edges[i]);\n }\n }\n\n // add descendant nodes\n function addChildren(node) {\n var children = node._private.children;\n for (var i = 0; i < children.length; i++) {\n add(children[i]);\n }\n }\n function add(ele) {\n var alreadyAdded = elesToRemoveIds[ele.id()];\n if (removeFromPool && ele.removed() || alreadyAdded) {\n return;\n } else {\n elesToRemoveIds[ele.id()] = true;\n }\n if (ele.isNode()) {\n elesToRemove.push(ele); // nodes are removed last\n\n addConnectedEdges(ele);\n addChildren(ele);\n } else {\n elesToRemove.unshift(ele); // edges are removed first\n }\n }\n\n // make the list of elements to remove\n // (may be removing more than specified due to connected edges etc)\n\n for (var i = 0, l = self.length; i < l; i++) {\n var ele = self[i];\n add(ele);\n }\n function removeEdgeRef(node, edge) {\n var connectedEdges = node._private.edges;\n removeFromArray(connectedEdges, edge);\n\n // removing an edges invalidates the traversal cache for its nodes\n node.clearTraversalCache();\n }\n function removeParallelRef(pllEdge) {\n // removing an edge invalidates the traversal caches for the parallel edges\n pllEdge.clearTraversalCache();\n }\n var alteredParents = [];\n alteredParents.ids = {};\n function removeChildRef(parent, ele) {\n ele = ele[0];\n parent = parent[0];\n var children = parent._private.children;\n var pid = parent.id();\n removeFromArray(children, ele); // remove parent => child ref\n\n ele._private.parent = null; // remove child => parent ref\n\n if (!alteredParents.ids[pid]) {\n alteredParents.ids[pid] = true;\n alteredParents.push(parent);\n }\n }\n self.dirtyCompoundBoundsCache();\n if (removeFromPool) {\n cy.removeFromPool(elesToRemove); // remove from core pool\n }\n for (var _i6 = 0; _i6 < elesToRemove.length; _i6++) {\n var _ele4 = elesToRemove[_i6];\n if (_ele4.isEdge()) {\n // remove references to this edge in its connected nodes\n var src = _ele4.source()[0];\n var tgt = _ele4.target()[0];\n removeEdgeRef(src, _ele4);\n removeEdgeRef(tgt, _ele4);\n var pllEdges = _ele4.parallelEdges();\n for (var j = 0; j < pllEdges.length; j++) {\n var pllEdge = pllEdges[j];\n removeParallelRef(pllEdge);\n if (pllEdge.isBundledBezier()) {\n pllEdge.dirtyBoundingBoxCache();\n }\n }\n } else {\n // remove reference to parent\n var parent = _ele4.parent();\n if (parent.length !== 0) {\n removeChildRef(parent, _ele4);\n }\n }\n if (removeFromPool) {\n // mark as removed\n _ele4._private.removed = true;\n }\n }\n\n // check to see if we have a compound graph or not\n var elesStillInside = cy._private.elements;\n cy._private.hasCompoundNodes = false;\n for (var _i7 = 0; _i7 < elesStillInside.length; _i7++) {\n var _ele5 = elesStillInside[_i7];\n if (_ele5.isParent()) {\n cy._private.hasCompoundNodes = true;\n break;\n }\n }\n var removedElements = new Collection(this.cy(), elesToRemove);\n if (removedElements.size() > 0) {\n // must manually notify since trigger won't do this automatically once removed\n\n if (notifyRenderer) {\n removedElements.emitAndNotify('remove');\n } else if (removeFromPool) {\n removedElements.emit('remove');\n }\n }\n\n // the parents who were modified by the removal need their style updated\n for (var _i8 = 0; _i8 < alteredParents.length; _i8++) {\n var _ele6 = alteredParents[_i8];\n if (!removeFromPool || !_ele6.removed()) {\n _ele6.updateStyle();\n }\n }\n return removedElements;\n};\nelesfn$1.move = function (struct) {\n var cy = this._private.cy;\n var eles = this;\n\n // just clean up refs, caches, etc. in the same way as when removing and then restoring\n // (our calls to remove/restore do not remove from the graph or make events)\n var notifyRenderer = false;\n var modifyPool = false;\n var toString = function toString(id) {\n return id == null ? id : '' + id;\n }; // id must be string\n\n if (struct.source !== undefined || struct.target !== undefined) {\n var srcId = toString(struct.source);\n var tgtId = toString(struct.target);\n var srcExists = srcId != null && cy.hasElementWithId(srcId);\n var tgtExists = tgtId != null && cy.hasElementWithId(tgtId);\n if (srcExists || tgtExists) {\n cy.batch(function () {\n // avoid duplicate style updates\n eles.remove(notifyRenderer, modifyPool); // clean up refs etc.\n eles.emitAndNotify('moveout');\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n var _data5 = ele._private.data;\n if (ele.isEdge()) {\n if (srcExists) {\n _data5.source = srcId;\n }\n if (tgtExists) {\n _data5.target = tgtId;\n }\n }\n }\n eles.restore(notifyRenderer, modifyPool); // make new refs, style, etc.\n });\n eles.emitAndNotify('move');\n }\n } else if (struct.parent !== undefined) {\n // move node to new parent\n var parentId = toString(struct.parent);\n var parentExists = parentId === null || cy.hasElementWithId(parentId);\n if (parentExists) {\n var pidToAssign = parentId === null ? undefined : parentId;\n cy.batch(function () {\n // avoid duplicate style updates\n var updated = eles.remove(notifyRenderer, modifyPool); // clean up refs etc.\n updated.emitAndNotify('moveout');\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n var _data6 = ele._private.data;\n if (ele.isNode()) {\n _data6.parent = pidToAssign;\n }\n }\n updated.restore(notifyRenderer, modifyPool); // make new refs, style, etc.\n });\n eles.emitAndNotify('move');\n }\n }\n return this;\n};\n[elesfn$j, elesfn$i, elesfn$h, elesfn$g, elesfn$f, data, elesfn$d, dimensions, elesfn$9, elesfn$8, elesfn$7, elesfn$6, elesfn$5, elesfn$4, elesfn$3, elesfn$2].forEach(function (props) {\n extend(elesfn$1, props);\n});\n\nvar corefn$9 = {\n add: function add(opts) {\n var elements;\n var cy = this;\n\n // add the elements\n if (elementOrCollection(opts)) {\n var eles = opts;\n if (eles._private.cy === cy) {\n // same instance => just restore\n elements = eles.restore();\n } else {\n // otherwise, copy from json\n var jsons = [];\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n jsons.push(ele.json());\n }\n elements = new Collection(cy, jsons);\n }\n }\n\n // specify an array of options\n else if (array(opts)) {\n var _jsons = opts;\n elements = new Collection(cy, _jsons);\n }\n\n // specify via opts.nodes and opts.edges\n else if (plainObject(opts) && (array(opts.nodes) || array(opts.edges))) {\n var elesByGroup = opts;\n var _jsons2 = [];\n var grs = ['nodes', 'edges'];\n for (var _i = 0, il = grs.length; _i < il; _i++) {\n var group = grs[_i];\n var elesArray = elesByGroup[group];\n if (array(elesArray)) {\n for (var j = 0, jl = elesArray.length; j < jl; j++) {\n var json = extend({\n group: group\n }, elesArray[j]);\n _jsons2.push(json);\n }\n }\n }\n elements = new Collection(cy, _jsons2);\n }\n\n // specify options for one element\n else {\n var _json = opts;\n elements = new Element(cy, _json).collection();\n }\n return elements;\n },\n remove: function remove(collection) {\n if (elementOrCollection(collection)) ; else if (string(collection)) {\n var selector = collection;\n collection = this.$(selector);\n }\n return collection.remove();\n }\n};\n\n/* global Float32Array */\n\n/*! Bezier curve function generator. Copyright Gaetan Renaudeau. MIT License: http://en.wikipedia.org/wiki/MIT_License */\nfunction generateCubicBezier(mX1, mY1, mX2, mY2) {\n var NEWTON_ITERATIONS = 4,\n NEWTON_MIN_SLOPE = 0.001,\n SUBDIVISION_PRECISION = 0.0000001,\n SUBDIVISION_MAX_ITERATIONS = 10,\n kSplineTableSize = 11,\n kSampleStepSize = 1.0 / (kSplineTableSize - 1.0),\n float32ArraySupported = typeof Float32Array !== 'undefined';\n\n /* Must contain four arguments. */\n if (arguments.length !== 4) {\n return false;\n }\n\n /* Arguments must be numbers. */\n for (var i = 0; i < 4; ++i) {\n if (typeof arguments[i] !== \"number\" || isNaN(arguments[i]) || !isFinite(arguments[i])) {\n return false;\n }\n }\n\n /* X values must be in the [0, 1] range. */\n mX1 = Math.min(mX1, 1);\n mX2 = Math.min(mX2, 1);\n mX1 = Math.max(mX1, 0);\n mX2 = Math.max(mX2, 0);\n var mSampleValues = float32ArraySupported ? new Float32Array(kSplineTableSize) : new Array(kSplineTableSize);\n function A(aA1, aA2) {\n return 1.0 - 3.0 * aA2 + 3.0 * aA1;\n }\n function B(aA1, aA2) {\n return 3.0 * aA2 - 6.0 * aA1;\n }\n function C(aA1) {\n return 3.0 * aA1;\n }\n function calcBezier(aT, aA1, aA2) {\n return ((A(aA1, aA2) * aT + B(aA1, aA2)) * aT + C(aA1)) * aT;\n }\n function getSlope(aT, aA1, aA2) {\n return 3.0 * A(aA1, aA2) * aT * aT + 2.0 * B(aA1, aA2) * aT + C(aA1);\n }\n function newtonRaphsonIterate(aX, aGuessT) {\n for (var _i = 0; _i < NEWTON_ITERATIONS; ++_i) {\n var currentSlope = getSlope(aGuessT, mX1, mX2);\n if (currentSlope === 0.0) {\n return aGuessT;\n }\n var currentX = calcBezier(aGuessT, mX1, mX2) - aX;\n aGuessT -= currentX / currentSlope;\n }\n return aGuessT;\n }\n function calcSampleValues() {\n for (var _i2 = 0; _i2 < kSplineTableSize; ++_i2) {\n mSampleValues[_i2] = calcBezier(_i2 * kSampleStepSize, mX1, mX2);\n }\n }\n function binarySubdivide(aX, aA, aB) {\n var currentX,\n currentT,\n i = 0;\n do {\n currentT = aA + (aB - aA) / 2.0;\n currentX = calcBezier(currentT, mX1, mX2) - aX;\n if (currentX > 0.0) {\n aB = currentT;\n } else {\n aA = currentT;\n }\n } while (Math.abs(currentX) > SUBDIVISION_PRECISION && ++i < SUBDIVISION_MAX_ITERATIONS);\n return currentT;\n }\n function getTForX(aX) {\n var intervalStart = 0.0,\n currentSample = 1,\n lastSample = kSplineTableSize - 1;\n for (; currentSample !== lastSample && mSampleValues[currentSample] <= aX; ++currentSample) {\n intervalStart += kSampleStepSize;\n }\n --currentSample;\n var dist = (aX - mSampleValues[currentSample]) / (mSampleValues[currentSample + 1] - mSampleValues[currentSample]),\n guessForT = intervalStart + dist * kSampleStepSize,\n initialSlope = getSlope(guessForT, mX1, mX2);\n if (initialSlope >= NEWTON_MIN_SLOPE) {\n return newtonRaphsonIterate(aX, guessForT);\n } else if (initialSlope === 0.0) {\n return guessForT;\n } else {\n return binarySubdivide(aX, intervalStart, intervalStart + kSampleStepSize);\n }\n }\n var _precomputed = false;\n function precompute() {\n _precomputed = true;\n if (mX1 !== mY1 || mX2 !== mY2) {\n calcSampleValues();\n }\n }\n var f = function f(aX) {\n if (!_precomputed) {\n precompute();\n }\n if (mX1 === mY1 && mX2 === mY2) {\n return aX;\n }\n if (aX === 0) {\n return 0;\n }\n if (aX === 1) {\n return 1;\n }\n return calcBezier(getTForX(aX), mY1, mY2);\n };\n f.getControlPoints = function () {\n return [{\n x: mX1,\n y: mY1\n }, {\n x: mX2,\n y: mY2\n }];\n };\n var str = \"generateBezier(\" + [mX1, mY1, mX2, mY2] + \")\";\n f.toString = function () {\n return str;\n };\n return f;\n}\n\n/*! Runge-Kutta spring physics function generator. Adapted from Framer.js, copyright Koen Bok. MIT License: http://en.wikipedia.org/wiki/MIT_License */\n/* Given a tension, friction, and duration, a simulation at 60FPS will first run without a defined duration in order to calculate the full path. A second pass\n then adjusts the time delta -- using the relation between actual time and duration -- to calculate the path for the duration-constrained animation. */\nvar generateSpringRK4 = function () {\n function springAccelerationForState(state) {\n return -state.tension * state.x - state.friction * state.v;\n }\n function springEvaluateStateWithDerivative(initialState, dt, derivative) {\n var state = {\n x: initialState.x + derivative.dx * dt,\n v: initialState.v + derivative.dv * dt,\n tension: initialState.tension,\n friction: initialState.friction\n };\n return {\n dx: state.v,\n dv: springAccelerationForState(state)\n };\n }\n function springIntegrateState(state, dt) {\n var a = {\n dx: state.v,\n dv: springAccelerationForState(state)\n },\n b = springEvaluateStateWithDerivative(state, dt * 0.5, a),\n c = springEvaluateStateWithDerivative(state, dt * 0.5, b),\n d = springEvaluateStateWithDerivative(state, dt, c),\n dxdt = 1.0 / 6.0 * (a.dx + 2.0 * (b.dx + c.dx) + d.dx),\n dvdt = 1.0 / 6.0 * (a.dv + 2.0 * (b.dv + c.dv) + d.dv);\n state.x = state.x + dxdt * dt;\n state.v = state.v + dvdt * dt;\n return state;\n }\n return function springRK4Factory(tension, friction, duration) {\n var initState = {\n x: -1,\n v: 0,\n tension: null,\n friction: null\n },\n path = [0],\n time_lapsed = 0,\n tolerance = 1 / 10000,\n DT = 16 / 1000,\n have_duration,\n dt,\n last_state;\n tension = parseFloat(tension) || 500;\n friction = parseFloat(friction) || 20;\n duration = duration || null;\n initState.tension = tension;\n initState.friction = friction;\n have_duration = duration !== null;\n\n /* Calculate the actual time it takes for this animation to complete with the provided conditions. */\n if (have_duration) {\n /* Run the simulation without a duration. */\n time_lapsed = springRK4Factory(tension, friction);\n /* Compute the adjusted time delta. */\n dt = time_lapsed / duration * DT;\n } else {\n dt = DT;\n }\n for (;;) {\n /* Next/step function .*/\n last_state = springIntegrateState(last_state || initState, dt);\n /* Store the position. */\n path.push(1 + last_state.x);\n time_lapsed += 16;\n /* If the change threshold is reached, break. */\n if (!(Math.abs(last_state.x) > tolerance && Math.abs(last_state.v) > tolerance)) {\n break;\n }\n }\n\n /* If duration is not defined, return the actual time required for completing this animation. Otherwise, return a closure that holds the\n computed path and returns a snapshot of the position according to a given percentComplete. */\n return !have_duration ? time_lapsed : function (percentComplete) {\n return path[percentComplete * (path.length - 1) | 0];\n };\n };\n}();\n\nvar cubicBezier = function cubicBezier(t1, p1, t2, p2) {\n var bezier = generateCubicBezier(t1, p1, t2, p2);\n return function (start, end, percent) {\n return start + (end - start) * bezier(percent);\n };\n};\nvar easings = {\n 'linear': function linear(start, end, percent) {\n return start + (end - start) * percent;\n },\n // default easings\n 'ease': cubicBezier(0.25, 0.1, 0.25, 1),\n 'ease-in': cubicBezier(0.42, 0, 1, 1),\n 'ease-out': cubicBezier(0, 0, 0.58, 1),\n 'ease-in-out': cubicBezier(0.42, 0, 0.58, 1),\n // sine\n 'ease-in-sine': cubicBezier(0.47, 0, 0.745, 0.715),\n 'ease-out-sine': cubicBezier(0.39, 0.575, 0.565, 1),\n 'ease-in-out-sine': cubicBezier(0.445, 0.05, 0.55, 0.95),\n // quad\n 'ease-in-quad': cubicBezier(0.55, 0.085, 0.68, 0.53),\n 'ease-out-quad': cubicBezier(0.25, 0.46, 0.45, 0.94),\n 'ease-in-out-quad': cubicBezier(0.455, 0.03, 0.515, 0.955),\n // cubic\n 'ease-in-cubic': cubicBezier(0.55, 0.055, 0.675, 0.19),\n 'ease-out-cubic': cubicBezier(0.215, 0.61, 0.355, 1),\n 'ease-in-out-cubic': cubicBezier(0.645, 0.045, 0.355, 1),\n // quart\n 'ease-in-quart': cubicBezier(0.895, 0.03, 0.685, 0.22),\n 'ease-out-quart': cubicBezier(0.165, 0.84, 0.44, 1),\n 'ease-in-out-quart': cubicBezier(0.77, 0, 0.175, 1),\n // quint\n 'ease-in-quint': cubicBezier(0.755, 0.05, 0.855, 0.06),\n 'ease-out-quint': cubicBezier(0.23, 1, 0.32, 1),\n 'ease-in-out-quint': cubicBezier(0.86, 0, 0.07, 1),\n // expo\n 'ease-in-expo': cubicBezier(0.95, 0.05, 0.795, 0.035),\n 'ease-out-expo': cubicBezier(0.19, 1, 0.22, 1),\n 'ease-in-out-expo': cubicBezier(1, 0, 0, 1),\n // circ\n 'ease-in-circ': cubicBezier(0.6, 0.04, 0.98, 0.335),\n 'ease-out-circ': cubicBezier(0.075, 0.82, 0.165, 1),\n 'ease-in-out-circ': cubicBezier(0.785, 0.135, 0.15, 0.86),\n // user param easings...\n\n 'spring': function spring(tension, friction, duration) {\n if (duration === 0) {\n // can't get a spring w/ duration 0\n return easings.linear; // duration 0 => jump to end so impl doesn't matter\n }\n var spring = generateSpringRK4(tension, friction, duration);\n return function (start, end, percent) {\n return start + (end - start) * spring(percent);\n };\n },\n 'cubic-bezier': cubicBezier\n};\n\nfunction getEasedValue(type, start, end, percent, easingFn) {\n if (percent === 1) {\n return end;\n }\n if (start === end) {\n return end;\n }\n var val = easingFn(start, end, percent);\n if (type == null) {\n return val;\n }\n if (type.roundValue || type.color) {\n val = Math.round(val);\n }\n if (type.min !== undefined) {\n val = Math.max(val, type.min);\n }\n if (type.max !== undefined) {\n val = Math.min(val, type.max);\n }\n return val;\n}\nfunction getValue(prop, spec) {\n if (prop.pfValue != null || prop.value != null) {\n if (prop.pfValue != null && (spec == null || spec.type.units !== '%')) {\n return prop.pfValue;\n } else {\n return prop.value;\n }\n } else {\n return prop;\n }\n}\nfunction ease(startProp, endProp, percent, easingFn, propSpec) {\n var type = propSpec != null ? propSpec.type : null;\n if (percent < 0) {\n percent = 0;\n } else if (percent > 1) {\n percent = 1;\n }\n var start = getValue(startProp, propSpec);\n var end = getValue(endProp, propSpec);\n if (number$1(start) && number$1(end)) {\n return getEasedValue(type, start, end, percent, easingFn);\n } else if (array(start) && array(end)) {\n var easedArr = [];\n for (var i = 0; i < end.length; i++) {\n var si = start[i];\n var ei = end[i];\n if (si != null && ei != null) {\n var val = getEasedValue(type, si, ei, percent, easingFn);\n easedArr.push(val);\n } else {\n easedArr.push(ei);\n }\n }\n return easedArr;\n }\n return undefined;\n}\n\nfunction step$1(self, ani, now, isCore) {\n var isEles = !isCore;\n var _p = self._private;\n var ani_p = ani._private;\n var pEasing = ani_p.easing;\n var startTime = ani_p.startTime;\n var cy = isCore ? self : self.cy();\n var style = cy.style();\n if (!ani_p.easingImpl) {\n if (pEasing == null) {\n // use default\n ani_p.easingImpl = easings['linear'];\n } else {\n // then define w/ name\n var easingVals;\n if (string(pEasing)) {\n var easingProp = style.parse('transition-timing-function', pEasing);\n easingVals = easingProp.value;\n } else {\n // then assume preparsed array\n easingVals = pEasing;\n }\n var name, args;\n if (string(easingVals)) {\n name = easingVals;\n args = [];\n } else {\n name = easingVals[1];\n args = easingVals.slice(2).map(function (n) {\n return +n;\n });\n }\n if (args.length > 0) {\n // create with args\n if (name === 'spring') {\n args.push(ani_p.duration); // need duration to generate spring\n }\n ani_p.easingImpl = easings[name].apply(null, args);\n } else {\n // static impl by name\n ani_p.easingImpl = easings[name];\n }\n }\n }\n var easing = ani_p.easingImpl;\n var percent;\n if (ani_p.duration === 0) {\n percent = 1;\n } else {\n percent = (now - startTime) / ani_p.duration;\n }\n if (ani_p.applying) {\n percent = ani_p.progress;\n }\n if (percent < 0) {\n percent = 0;\n } else if (percent > 1) {\n percent = 1;\n }\n if (ani_p.delay == null) {\n // then update\n\n var startPos = ani_p.startPosition;\n var endPos = ani_p.position;\n if (endPos && isEles && !self.locked()) {\n var newPos = {};\n if (valid(startPos.x, endPos.x)) {\n newPos.x = ease(startPos.x, endPos.x, percent, easing);\n }\n if (valid(startPos.y, endPos.y)) {\n newPos.y = ease(startPos.y, endPos.y, percent, easing);\n }\n self.position(newPos);\n }\n var startPan = ani_p.startPan;\n var endPan = ani_p.pan;\n var pan = _p.pan;\n var animatingPan = endPan != null && isCore;\n if (animatingPan) {\n if (valid(startPan.x, endPan.x)) {\n pan.x = ease(startPan.x, endPan.x, percent, easing);\n }\n if (valid(startPan.y, endPan.y)) {\n pan.y = ease(startPan.y, endPan.y, percent, easing);\n }\n self.emit('pan');\n }\n var startZoom = ani_p.startZoom;\n var endZoom = ani_p.zoom;\n var animatingZoom = endZoom != null && isCore;\n if (animatingZoom) {\n if (valid(startZoom, endZoom)) {\n _p.zoom = bound(_p.minZoom, ease(startZoom, endZoom, percent, easing), _p.maxZoom);\n }\n self.emit('zoom');\n }\n if (animatingPan || animatingZoom) {\n self.emit('viewport');\n }\n var props = ani_p.style;\n if (props && props.length > 0 && isEles) {\n for (var i = 0; i < props.length; i++) {\n var prop = props[i];\n var _name = prop.name;\n var end = prop;\n var start = ani_p.startStyle[_name];\n var propSpec = style.properties[start.name];\n var easedVal = ease(start, end, percent, easing, propSpec);\n style.overrideBypass(self, _name, easedVal);\n } // for props\n\n self.emit('style');\n } // if\n }\n ani_p.progress = percent;\n return percent;\n}\nfunction valid(start, end) {\n if (start == null || end == null) {\n return false;\n }\n if (number$1(start) && number$1(end)) {\n return true;\n } else if (start && end) {\n return true;\n }\n return false;\n}\n\nfunction startAnimation(self, ani, now, isCore) {\n var ani_p = ani._private;\n ani_p.started = true;\n ani_p.startTime = now - ani_p.progress * ani_p.duration;\n}\n\nfunction stepAll(now, cy) {\n var eles = cy._private.aniEles;\n var doneEles = [];\n function stepOne(ele, isCore) {\n var _p = ele._private;\n var current = _p.animation.current;\n var queue = _p.animation.queue;\n var ranAnis = false;\n\n // if nothing currently animating, get something from the queue\n if (current.length === 0) {\n var next = queue.shift();\n if (next) {\n current.push(next);\n }\n }\n var callbacks = function callbacks(_callbacks) {\n for (var j = _callbacks.length - 1; j >= 0; j--) {\n var cb = _callbacks[j];\n cb();\n }\n _callbacks.splice(0, _callbacks.length);\n };\n\n // step and remove if done\n for (var i = current.length - 1; i >= 0; i--) {\n var ani = current[i];\n var ani_p = ani._private;\n if (ani_p.stopped) {\n current.splice(i, 1);\n ani_p.hooked = false;\n ani_p.playing = false;\n ani_p.started = false;\n callbacks(ani_p.frames);\n continue;\n }\n if (!ani_p.playing && !ani_p.applying) {\n continue;\n }\n\n // an apply() while playing shouldn't do anything\n if (ani_p.playing && ani_p.applying) {\n ani_p.applying = false;\n }\n if (!ani_p.started) {\n startAnimation(ele, ani, now);\n }\n step$1(ele, ani, now, isCore);\n if (ani_p.applying) {\n ani_p.applying = false;\n }\n callbacks(ani_p.frames);\n if (ani_p.step != null) {\n ani_p.step(now);\n }\n if (ani.completed()) {\n current.splice(i, 1);\n ani_p.hooked = false;\n ani_p.playing = false;\n ani_p.started = false;\n callbacks(ani_p.completes);\n }\n ranAnis = true;\n }\n if (!isCore && current.length === 0 && queue.length === 0) {\n doneEles.push(ele);\n }\n return ranAnis;\n } // stepElement\n\n // handle all eles\n var ranEleAni = false;\n for (var e = 0; e < eles.length; e++) {\n var ele = eles[e];\n var handledThisEle = stepOne(ele);\n ranEleAni = ranEleAni || handledThisEle;\n } // each element\n\n var ranCoreAni = stepOne(cy, true);\n\n // notify renderer\n if (ranEleAni || ranCoreAni) {\n if (eles.length > 0) {\n cy.notify('draw', eles);\n } else {\n cy.notify('draw');\n }\n }\n\n // remove elements from list of currently animating if its queues are empty\n eles.unmerge(doneEles);\n cy.emit('step');\n} // stepAll\n\nvar corefn$8 = {\n // pull in animation functions\n animate: define.animate(),\n animation: define.animation(),\n animated: define.animated(),\n clearQueue: define.clearQueue(),\n delay: define.delay(),\n delayAnimation: define.delayAnimation(),\n stop: define.stop(),\n addToAnimationPool: function addToAnimationPool(eles) {\n var cy = this;\n if (!cy.styleEnabled()) {\n return;\n } // save cycles when no style used\n\n cy._private.aniEles.merge(eles);\n },\n stopAnimationLoop: function stopAnimationLoop() {\n this._private.animationsRunning = false;\n },\n startAnimationLoop: function startAnimationLoop() {\n var cy = this;\n cy._private.animationsRunning = true;\n if (!cy.styleEnabled()) {\n return;\n } // save cycles when no style used\n\n // NB the animation loop will exec in headless environments if style enabled\n // and explicit cy.destroy() is necessary to stop the loop\n\n function headlessStep() {\n if (!cy._private.animationsRunning) {\n return;\n }\n requestAnimationFrame(function animationStep(now) {\n stepAll(now, cy);\n headlessStep();\n });\n }\n var renderer = cy.renderer();\n if (renderer && renderer.beforeRender) {\n // let the renderer schedule animations\n renderer.beforeRender(function rendererAnimationStep(willDraw, now) {\n stepAll(now, cy);\n }, renderer.beforeRenderPriorities.animations);\n } else {\n // manage the animation loop ourselves\n headlessStep(); // first call\n }\n }\n};\n\nvar emitterOptions = {\n qualifierCompare: function qualifierCompare(selector1, selector2) {\n if (selector1 == null || selector2 == null) {\n return selector1 == null && selector2 == null;\n } else {\n return selector1.sameText(selector2);\n }\n },\n eventMatches: function eventMatches(cy, listener, eventObj) {\n var selector = listener.qualifier;\n if (selector != null) {\n return cy !== eventObj.target && element(eventObj.target) && selector.matches(eventObj.target);\n }\n return true;\n },\n addEventFields: function addEventFields(cy, evt) {\n evt.cy = cy;\n evt.target = cy;\n },\n callbackContext: function callbackContext(cy, listener, eventObj) {\n return listener.qualifier != null ? eventObj.target : cy;\n }\n};\nvar argSelector = function argSelector(arg) {\n if (string(arg)) {\n return new Selector(arg);\n } else {\n return arg;\n }\n};\nvar elesfn = {\n createEmitter: function createEmitter() {\n var _p = this._private;\n if (!_p.emitter) {\n _p.emitter = new Emitter(emitterOptions, this);\n }\n return this;\n },\n emitter: function emitter() {\n return this._private.emitter;\n },\n on: function on(events, selector, callback) {\n this.emitter().on(events, argSelector(selector), callback);\n return this;\n },\n removeListener: function removeListener(events, selector, callback) {\n this.emitter().removeListener(events, argSelector(selector), callback);\n return this;\n },\n removeAllListeners: function removeAllListeners() {\n this.emitter().removeAllListeners();\n return this;\n },\n one: function one(events, selector, callback) {\n this.emitter().one(events, argSelector(selector), callback);\n return this;\n },\n once: function once(events, selector, callback) {\n this.emitter().one(events, argSelector(selector), callback);\n return this;\n },\n emit: function emit(events, extraParams) {\n this.emitter().emit(events, extraParams);\n return this;\n },\n emitAndNotify: function emitAndNotify(event, eles) {\n this.emit(event);\n this.notify(event, eles);\n return this;\n }\n};\ndefine.eventAliasesOn(elesfn);\n\nvar corefn$7 = {\n png: function png(options) {\n var renderer = this._private.renderer;\n options = options || {};\n return renderer.png(options);\n },\n jpg: function jpg(options) {\n var renderer = this._private.renderer;\n options = options || {};\n options.bg = options.bg || '#fff';\n return renderer.jpg(options);\n }\n};\ncorefn$7.jpeg = corefn$7.jpg;\n\nvar corefn$6 = {\n layout: function layout(options) {\n var cy = this;\n if (options == null) {\n error('Layout options must be specified to make a layout');\n return;\n }\n if (options.name == null) {\n error('A `name` must be specified to make a layout');\n return;\n }\n var name = options.name;\n var Layout = cy.extension('layout', name);\n if (Layout == null) {\n error('No such layout `' + name + '` found. Did you forget to import it and `cytoscape.use()` it?');\n return;\n }\n var eles;\n if (string(options.eles)) {\n eles = cy.$(options.eles);\n } else {\n eles = options.eles != null ? options.eles : cy.$();\n }\n var layout = new Layout(extend({}, options, {\n cy: cy,\n eles: eles\n }));\n return layout;\n }\n};\ncorefn$6.createLayout = corefn$6.makeLayout = corefn$6.layout;\n\nvar corefn$5 = {\n notify: function notify(eventName, eventEles) {\n var _p = this._private;\n if (this.batching()) {\n _p.batchNotifications = _p.batchNotifications || {};\n var eles = _p.batchNotifications[eventName] = _p.batchNotifications[eventName] || this.collection();\n if (eventEles != null) {\n eles.merge(eventEles);\n }\n return; // notifications are disabled during batching\n }\n if (!_p.notificationsEnabled) {\n return;\n } // exit on disabled\n\n var renderer = this.renderer();\n\n // exit if destroy() called on core or renderer in between frames #1499 #1528\n if (this.destroyed() || !renderer) {\n return;\n }\n renderer.notify(eventName, eventEles);\n },\n notifications: function notifications(bool) {\n var p = this._private;\n if (bool === undefined) {\n return p.notificationsEnabled;\n } else {\n p.notificationsEnabled = bool ? true : false;\n }\n return this;\n },\n noNotifications: function noNotifications(callback) {\n this.notifications(false);\n callback();\n this.notifications(true);\n },\n batching: function batching() {\n return this._private.batchCount > 0;\n },\n startBatch: function startBatch() {\n var _p = this._private;\n if (_p.batchCount == null) {\n _p.batchCount = 0;\n }\n if (_p.batchCount === 0) {\n _p.batchStyleEles = this.collection();\n _p.batchNotifications = {};\n }\n _p.batchCount++;\n return this;\n },\n endBatch: function endBatch() {\n var _p = this._private;\n if (_p.batchCount === 0) {\n return this;\n }\n _p.batchCount--;\n if (_p.batchCount === 0) {\n // update style for dirty eles\n _p.batchStyleEles.updateStyle();\n var renderer = this.renderer();\n\n // notify the renderer of queued eles and event types\n Object.keys(_p.batchNotifications).forEach(function (eventName) {\n var eles = _p.batchNotifications[eventName];\n if (eles.empty()) {\n renderer.notify(eventName);\n } else {\n renderer.notify(eventName, eles);\n }\n });\n }\n return this;\n },\n batch: function batch(callback) {\n this.startBatch();\n callback();\n this.endBatch();\n return this;\n },\n // for backwards compatibility\n batchData: function batchData(map) {\n var cy = this;\n return this.batch(function () {\n var ids = Object.keys(map);\n for (var i = 0; i < ids.length; i++) {\n var id = ids[i];\n var data = map[id];\n var ele = cy.getElementById(id);\n ele.data(data);\n }\n });\n }\n};\n\nvar rendererDefaults = defaults$g({\n hideEdgesOnViewport: false,\n textureOnViewport: false,\n motionBlur: false,\n motionBlurOpacity: 0.05,\n pixelRatio: undefined,\n desktopTapThreshold: 4,\n touchTapThreshold: 8,\n wheelSensitivity: 1,\n debug: false,\n showFps: false,\n // webgl options\n webgl: false,\n webglDebug: false,\n webglDebugShowAtlases: false,\n // defaults good for mobile\n webglTexSize: 2048,\n webglTexRows: 36,\n webglTexRowsNodes: 18,\n webglBatchSize: 2048,\n webglTexPerBatch: 14,\n webglBgColor: [255, 255, 255]\n});\nvar corefn$4 = {\n renderTo: function renderTo(context, zoom, pan, pxRatio) {\n var r = this._private.renderer;\n r.renderTo(context, zoom, pan, pxRatio);\n return this;\n },\n renderer: function renderer() {\n return this._private.renderer;\n },\n forceRender: function forceRender() {\n this.notify('draw');\n return this;\n },\n resize: function resize() {\n this.invalidateSize();\n this.emitAndNotify('resize');\n return this;\n },\n initRenderer: function initRenderer(options) {\n var cy = this;\n var RendererProto = cy.extension('renderer', options.name);\n if (RendererProto == null) {\n error(\"Can not initialise: No such renderer `\".concat(options.name, \"` found. Did you forget to import it and `cytoscape.use()` it?\"));\n return;\n }\n if (options.wheelSensitivity !== undefined) {\n warn(\"You have set a custom wheel sensitivity. This will make your app zoom unnaturally when using mainstream mice. You should change this value from the default only if you can guarantee that all your users will use the same hardware and OS configuration as your current machine.\");\n }\n var rOpts = rendererDefaults(options);\n rOpts.cy = cy;\n cy._private.renderer = new RendererProto(rOpts);\n this.notify('init');\n },\n destroyRenderer: function destroyRenderer() {\n var cy = this;\n cy.notify('destroy'); // destroy the renderer\n\n var domEle = cy.container();\n if (domEle) {\n domEle._cyreg = null;\n while (domEle.childNodes.length > 0) {\n domEle.removeChild(domEle.childNodes[0]);\n }\n }\n cy._private.renderer = null; // to be extra safe, remove the ref\n cy.mutableElements().forEach(function (ele) {\n var _p = ele._private;\n _p.rscratch = {};\n _p.rstyle = {};\n _p.animation.current = [];\n _p.animation.queue = [];\n });\n },\n onRender: function onRender(fn) {\n return this.on('render', fn);\n },\n offRender: function offRender(fn) {\n return this.off('render', fn);\n }\n};\ncorefn$4.invalidateDimensions = corefn$4.resize;\n\nvar corefn$3 = {\n // get a collection\n // - empty collection on no args\n // - collection of elements in the graph on selector arg\n // - guarantee a returned collection when elements or collection specified\n collection: function collection(eles, opts) {\n if (string(eles)) {\n return this.$(eles);\n } else if (elementOrCollection(eles)) {\n return eles.collection();\n } else if (array(eles)) {\n if (!opts) {\n opts = {};\n }\n return new Collection(this, eles, opts.unique, opts.removed);\n }\n return new Collection(this);\n },\n nodes: function nodes(selector) {\n var nodes = this.$(function (ele) {\n return ele.isNode();\n });\n if (selector) {\n return nodes.filter(selector);\n }\n return nodes;\n },\n edges: function edges(selector) {\n var edges = this.$(function (ele) {\n return ele.isEdge();\n });\n if (selector) {\n return edges.filter(selector);\n }\n return edges;\n },\n // search the graph like jQuery\n $: function $(selector) {\n var eles = this._private.elements;\n if (selector) {\n return eles.filter(selector);\n } else {\n return eles.spawnSelf();\n }\n },\n mutableElements: function mutableElements() {\n return this._private.elements;\n }\n};\n\n// aliases\ncorefn$3.elements = corefn$3.filter = corefn$3.$;\n\nvar styfn$8 = {};\n\n// keys for style blocks, e.g. ttfftt\nvar TRUE = 't';\nvar FALSE = 'f';\n\n// (potentially expensive calculation)\n// apply the style to the element based on\n// - its bypass\n// - what selectors match it\nstyfn$8.apply = function (eles) {\n var self = this;\n var _p = self._private;\n var cy = _p.cy;\n var updatedEles = cy.collection();\n for (var ie = 0; ie < eles.length; ie++) {\n var ele = eles[ie];\n var cxtMeta = self.getContextMeta(ele);\n if (cxtMeta.empty) {\n continue;\n }\n var cxtStyle = self.getContextStyle(cxtMeta);\n var app = self.applyContextStyle(cxtMeta, cxtStyle, ele);\n if (ele._private.appliedInitStyle) {\n self.updateTransitions(ele, app.diffProps);\n } else {\n ele._private.appliedInitStyle = true;\n }\n var hintsDiff = self.updateStyleHints(ele);\n if (hintsDiff) {\n updatedEles.push(ele);\n }\n } // for elements\n\n return updatedEles;\n};\nstyfn$8.getPropertiesDiff = function (oldCxtKey, newCxtKey) {\n var self = this;\n var cache = self._private.propDiffs = self._private.propDiffs || {};\n var dualCxtKey = oldCxtKey + '-' + newCxtKey;\n var cachedVal = cache[dualCxtKey];\n if (cachedVal) {\n return cachedVal;\n }\n var diffProps = [];\n var addedProp = {};\n for (var i = 0; i < self.length; i++) {\n var cxt = self[i];\n var oldHasCxt = oldCxtKey[i] === TRUE;\n var newHasCxt = newCxtKey[i] === TRUE;\n var cxtHasDiffed = oldHasCxt !== newHasCxt;\n var cxtHasMappedProps = cxt.mappedProperties.length > 0;\n if (cxtHasDiffed || newHasCxt && cxtHasMappedProps) {\n var props = undefined;\n if (cxtHasDiffed && cxtHasMappedProps) {\n props = cxt.properties; // suffices b/c mappedProperties is a subset of properties\n } else if (cxtHasDiffed) {\n props = cxt.properties; // need to check them all\n } else if (cxtHasMappedProps) {\n props = cxt.mappedProperties; // only need to check mapped\n }\n for (var j = 0; j < props.length; j++) {\n var prop = props[j];\n var name = prop.name;\n\n // if a later context overrides this property, then the fact that this context has switched/diffed doesn't matter\n // (semi expensive check since it makes this function O(n^2) on context length, but worth it since overall result\n // is cached)\n var laterCxtOverrides = false;\n for (var k = i + 1; k < self.length; k++) {\n var laterCxt = self[k];\n var hasLaterCxt = newCxtKey[k] === TRUE;\n if (!hasLaterCxt) {\n continue;\n } // can't override unless the context is active\n\n laterCxtOverrides = laterCxt.properties[prop.name] != null;\n if (laterCxtOverrides) {\n break;\n } // exit early as long as one later context overrides\n }\n if (!addedProp[name] && !laterCxtOverrides) {\n addedProp[name] = true;\n diffProps.push(name);\n }\n } // for props\n } // if\n } // for contexts\n\n cache[dualCxtKey] = diffProps;\n return diffProps;\n};\nstyfn$8.getContextMeta = function (ele) {\n var self = this;\n var cxtKey = '';\n var diffProps;\n var prevKey = ele._private.styleCxtKey || '';\n\n // get the cxt key\n for (var i = 0; i < self.length; i++) {\n var context = self[i];\n var contextSelectorMatches = context.selector && context.selector.matches(ele); // NB: context.selector may be null for 'core'\n\n if (contextSelectorMatches) {\n cxtKey += TRUE;\n } else {\n cxtKey += FALSE;\n }\n } // for context\n\n diffProps = self.getPropertiesDiff(prevKey, cxtKey);\n ele._private.styleCxtKey = cxtKey;\n return {\n key: cxtKey,\n diffPropNames: diffProps,\n empty: diffProps.length === 0\n };\n};\n\n// gets a computed ele style object based on matched contexts\nstyfn$8.getContextStyle = function (cxtMeta) {\n var cxtKey = cxtMeta.key;\n var self = this;\n var cxtStyles = this._private.contextStyles = this._private.contextStyles || {};\n\n // if already computed style, returned cached copy\n if (cxtStyles[cxtKey]) {\n return cxtStyles[cxtKey];\n }\n var style = {\n _private: {\n key: cxtKey\n }\n };\n for (var i = 0; i < self.length; i++) {\n var cxt = self[i];\n var hasCxt = cxtKey[i] === TRUE;\n if (!hasCxt) {\n continue;\n }\n for (var j = 0; j < cxt.properties.length; j++) {\n var prop = cxt.properties[j];\n style[prop.name] = prop;\n }\n }\n cxtStyles[cxtKey] = style;\n return style;\n};\nstyfn$8.applyContextStyle = function (cxtMeta, cxtStyle, ele) {\n var self = this;\n var diffProps = cxtMeta.diffPropNames;\n var retDiffProps = {};\n var types = self.types;\n for (var i = 0; i < diffProps.length; i++) {\n var diffPropName = diffProps[i];\n var cxtProp = cxtStyle[diffPropName];\n var eleProp = ele.pstyle(diffPropName);\n if (!cxtProp) {\n // no context prop means delete\n if (!eleProp) {\n continue; // no existing prop means nothing needs to be removed\n // nb affects initial application on mapped values like control-point-distances\n } else if (eleProp.bypass) {\n cxtProp = {\n name: diffPropName,\n deleteBypassed: true\n };\n } else {\n cxtProp = {\n name: diffPropName,\n \"delete\": true\n };\n }\n }\n\n // save cycles when the context prop doesn't need to be applied\n if (eleProp === cxtProp) {\n continue;\n }\n\n // save cycles when a mapped context prop doesn't need to be applied\n if (cxtProp.mapped === types.fn // context prop is function mapper\n && eleProp != null // some props can be null even by default (e.g. a prop that overrides another one)\n && eleProp.mapping != null // ele prop is a concrete value from from a mapper\n && eleProp.mapping.value === cxtProp.value // the current prop on the ele is a flat prop value for the function mapper\n ) {\n // NB don't write to cxtProp, as it's shared among eles (stored in stylesheet)\n var mapping = eleProp.mapping; // can write to mapping, as it's a per-ele copy\n var fnValue = mapping.fnValue = cxtProp.value(ele); // temporarily cache the value in case of a miss\n\n if (fnValue === mapping.prevFnValue) {\n continue;\n }\n }\n var retDiffProp = retDiffProps[diffPropName] = {\n prev: eleProp\n };\n self.applyParsedProperty(ele, cxtProp);\n retDiffProp.next = ele.pstyle(diffPropName);\n if (retDiffProp.next && retDiffProp.next.bypass) {\n retDiffProp.next = retDiffProp.next.bypassed;\n }\n }\n return {\n diffProps: retDiffProps\n };\n};\nstyfn$8.updateStyleHints = function (ele) {\n var _p = ele._private;\n var self = this;\n var propNames = self.propertyGroupNames;\n var propGrKeys = self.propertyGroupKeys;\n var propHash = function propHash(ele, propNames, seedKey) {\n return self.getPropertiesHash(ele, propNames, seedKey);\n };\n var oldStyleKey = _p.styleKey;\n if (ele.removed()) {\n return false;\n }\n var isNode = _p.group === 'nodes';\n\n // get the style key hashes per prop group\n // but lazily -- only use non-default prop values to reduce the number of hashes\n //\n\n var overriddenStyles = ele._private.style;\n propNames = Object.keys(overriddenStyles);\n for (var i = 0; i < propGrKeys.length; i++) {\n var grKey = propGrKeys[i];\n _p.styleKeys[grKey] = [DEFAULT_HASH_SEED, DEFAULT_HASH_SEED_ALT];\n }\n var updateGrKey1 = function updateGrKey1(val, grKey) {\n return _p.styleKeys[grKey][0] = hashInt(val, _p.styleKeys[grKey][0]);\n };\n var updateGrKey2 = function updateGrKey2(val, grKey) {\n return _p.styleKeys[grKey][1] = hashIntAlt(val, _p.styleKeys[grKey][1]);\n };\n var updateGrKey = function updateGrKey(val, grKey) {\n updateGrKey1(val, grKey);\n updateGrKey2(val, grKey);\n };\n var updateGrKeyWStr = function updateGrKeyWStr(strVal, grKey) {\n for (var j = 0; j < strVal.length; j++) {\n var ch = strVal.charCodeAt(j);\n updateGrKey1(ch, grKey);\n updateGrKey2(ch, grKey);\n }\n };\n\n // - hashing works on 32 bit ints b/c we use bitwise ops\n // - small numbers get cut off (e.g. 0.123 is seen as 0 by the hashing function)\n // - raise up small numbers so more significant digits are seen by hashing\n // - make small numbers larger than a normal value to avoid collisions\n // - works in practice and it's relatively cheap\n var N = 2000000000;\n var cleanNum = function cleanNum(val) {\n return -128 < val && val < 128 && Math.floor(val) !== val ? N - (val * 1024 | 0) : val;\n };\n for (var _i = 0; _i < propNames.length; _i++) {\n var name = propNames[_i];\n var parsedProp = overriddenStyles[name];\n if (parsedProp == null) {\n continue;\n }\n var propInfo = this.properties[name];\n var type = propInfo.type;\n var _grKey = propInfo.groupKey;\n var normalizedNumberVal = undefined;\n if (propInfo.hashOverride != null) {\n normalizedNumberVal = propInfo.hashOverride(ele, parsedProp);\n } else if (parsedProp.pfValue != null) {\n normalizedNumberVal = parsedProp.pfValue;\n }\n\n // might not be a number if it allows enums\n var numberVal = propInfo.enums == null ? parsedProp.value : null;\n var haveNormNum = normalizedNumberVal != null;\n var haveUnitedNum = numberVal != null;\n var haveNum = haveNormNum || haveUnitedNum;\n var units = parsedProp.units;\n\n // numbers are cheaper to hash than strings\n // 1 hash op vs n hash ops (for length n string)\n if (type.number && haveNum && !type.multiple) {\n var v = haveNormNum ? normalizedNumberVal : numberVal;\n updateGrKey(cleanNum(v), _grKey);\n if (!haveNormNum && units != null) {\n updateGrKeyWStr(units, _grKey);\n }\n } else {\n updateGrKeyWStr(parsedProp.strValue, _grKey);\n }\n }\n\n // overall style key\n //\n\n var hash = [DEFAULT_HASH_SEED, DEFAULT_HASH_SEED_ALT];\n for (var _i2 = 0; _i2 < propGrKeys.length; _i2++) {\n var _grKey2 = propGrKeys[_i2];\n var grHash = _p.styleKeys[_grKey2];\n hash[0] = hashInt(grHash[0], hash[0]);\n hash[1] = hashIntAlt(grHash[1], hash[1]);\n }\n _p.styleKey = combineHashes(hash[0], hash[1]);\n\n // label dims\n //\n\n var sk = _p.styleKeys;\n _p.labelDimsKey = combineHashesArray(sk.labelDimensions);\n var labelKeys = propHash(ele, ['label'], sk.labelDimensions);\n _p.labelKey = combineHashesArray(labelKeys);\n _p.labelStyleKey = combineHashesArray(hashArrays(sk.commonLabel, labelKeys));\n if (!isNode) {\n var sourceLabelKeys = propHash(ele, ['source-label'], sk.labelDimensions);\n _p.sourceLabelKey = combineHashesArray(sourceLabelKeys);\n _p.sourceLabelStyleKey = combineHashesArray(hashArrays(sk.commonLabel, sourceLabelKeys));\n var targetLabelKeys = propHash(ele, ['target-label'], sk.labelDimensions);\n _p.targetLabelKey = combineHashesArray(targetLabelKeys);\n _p.targetLabelStyleKey = combineHashesArray(hashArrays(sk.commonLabel, targetLabelKeys));\n }\n\n // node\n //\n\n if (isNode) {\n var _p$styleKeys = _p.styleKeys,\n nodeBody = _p$styleKeys.nodeBody,\n nodeBorder = _p$styleKeys.nodeBorder,\n nodeOutline = _p$styleKeys.nodeOutline,\n backgroundImage = _p$styleKeys.backgroundImage,\n compound = _p$styleKeys.compound,\n pie = _p$styleKeys.pie,\n stripe = _p$styleKeys.stripe;\n var nodeKeys = [nodeBody, nodeBorder, nodeOutline, backgroundImage, compound, pie, stripe].filter(function (k) {\n return k != null;\n }).reduce(hashArrays, [DEFAULT_HASH_SEED, DEFAULT_HASH_SEED_ALT]);\n _p.nodeKey = combineHashesArray(nodeKeys);\n _p.hasPie = pie != null && pie[0] !== DEFAULT_HASH_SEED && pie[1] !== DEFAULT_HASH_SEED_ALT;\n _p.hasStripe = stripe != null && stripe[0] !== DEFAULT_HASH_SEED && stripe[1] !== DEFAULT_HASH_SEED_ALT;\n }\n return oldStyleKey !== _p.styleKey;\n};\nstyfn$8.clearStyleHints = function (ele) {\n var _p = ele._private;\n _p.styleCxtKey = '';\n _p.styleKeys = {};\n _p.styleKey = null;\n _p.labelKey = null;\n _p.labelStyleKey = null;\n _p.sourceLabelKey = null;\n _p.sourceLabelStyleKey = null;\n _p.targetLabelKey = null;\n _p.targetLabelStyleKey = null;\n _p.nodeKey = null;\n _p.hasPie = null;\n _p.hasStripe = null;\n};\n\n// apply a property to the style (for internal use)\n// returns whether application was successful\n//\n// now, this function flattens the property, and here's how:\n//\n// for parsedProp:{ bypass: true, deleteBypass: true }\n// no property is generated, instead the bypass property in the\n// element's style is replaced by what's pointed to by the `bypassed`\n// field in the bypass property (i.e. restoring the property the\n// bypass was overriding)\n//\n// for parsedProp:{ mapped: truthy }\n// the generated flattenedProp:{ mapping: prop }\n//\n// for parsedProp:{ bypass: true }\n// the generated flattenedProp:{ bypassed: parsedProp }\nstyfn$8.applyParsedProperty = function (ele, parsedProp) {\n var self = this;\n var prop = parsedProp;\n var style = ele._private.style;\n var flatProp;\n var types = self.types;\n var type = self.properties[prop.name].type;\n var propIsBypass = prop.bypass;\n var origProp = style[prop.name];\n var origPropIsBypass = origProp && origProp.bypass;\n var _p = ele._private;\n var flatPropMapping = 'mapping';\n var getVal = function getVal(p) {\n if (p == null) {\n return null;\n } else if (p.pfValue != null) {\n return p.pfValue;\n } else {\n return p.value;\n }\n };\n var checkTriggers = function checkTriggers() {\n var fromVal = getVal(origProp);\n var toVal = getVal(prop);\n self.checkTriggers(ele, prop.name, fromVal, toVal);\n };\n\n // edge sanity checks to prevent the client from making serious mistakes\n if (parsedProp.name === 'curve-style' && ele.isEdge() && (\n // loops must be bundled beziers\n parsedProp.value !== 'bezier' && ele.isLoop() ||\n // edges connected to compound nodes can not be haystacks\n parsedProp.value === 'haystack' && (ele.source().isParent() || ele.target().isParent()))) {\n prop = parsedProp = this.parse(parsedProp.name, 'bezier', propIsBypass);\n }\n if (prop[\"delete\"]) {\n // delete the property and use the default value on falsey value\n style[prop.name] = undefined;\n checkTriggers();\n return true;\n }\n if (prop.deleteBypassed) {\n // delete the property that the\n if (!origProp) {\n checkTriggers();\n return true; // can't delete if no prop\n } else if (origProp.bypass) {\n // delete bypassed\n origProp.bypassed = undefined;\n checkTriggers();\n return true;\n } else {\n return false; // we're unsuccessful deleting the bypassed\n }\n }\n\n // check if we need to delete the current bypass\n if (prop.deleteBypass) {\n // then this property is just here to indicate we need to delete\n if (!origProp) {\n checkTriggers();\n return true; // property is already not defined\n } else if (origProp.bypass) {\n // then replace the bypass property with the original\n // because the bypassed property was already applied (and therefore parsed), we can just replace it (no reapplying necessary)\n style[prop.name] = origProp.bypassed;\n checkTriggers();\n return true;\n } else {\n return false; // we're unsuccessful deleting the bypass\n }\n }\n var printMappingErr = function printMappingErr() {\n warn('Do not assign mappings to elements without corresponding data (i.e. ele `' + ele.id() + '` has no mapping for property `' + prop.name + '` with data field `' + prop.field + '`); try a `[' + prop.field + ']` selector to limit scope to elements with `' + prop.field + '` defined');\n };\n\n // put the property in the style objects\n switch (prop.mapped) {\n // flatten the property if mapped\n case types.mapData:\n {\n // flatten the field (e.g. data.foo.bar)\n var fields = prop.field.split('.');\n var fieldVal = _p.data;\n for (var i = 0; i < fields.length && fieldVal; i++) {\n var field = fields[i];\n fieldVal = fieldVal[field];\n }\n if (fieldVal == null) {\n printMappingErr();\n return false;\n }\n var percent;\n if (!number$1(fieldVal)) {\n // then don't apply and fall back on the existing style\n warn('Do not use continuous mappers without specifying numeric data (i.e. `' + prop.field + ': ' + fieldVal + '` for `' + ele.id() + '` is non-numeric)');\n return false;\n } else {\n var fieldWidth = prop.fieldMax - prop.fieldMin;\n if (fieldWidth === 0) {\n // safety check -- not strictly necessary as no props of zero range should be passed here\n percent = 0;\n } else {\n percent = (fieldVal - prop.fieldMin) / fieldWidth;\n }\n }\n\n // make sure to bound percent value\n if (percent < 0) {\n percent = 0;\n } else if (percent > 1) {\n percent = 1;\n }\n if (type.color) {\n var r1 = prop.valueMin[0];\n var r2 = prop.valueMax[0];\n var g1 = prop.valueMin[1];\n var g2 = prop.valueMax[1];\n var b1 = prop.valueMin[2];\n var b2 = prop.valueMax[2];\n var a1 = prop.valueMin[3] == null ? 1 : prop.valueMin[3];\n var a2 = prop.valueMax[3] == null ? 1 : prop.valueMax[3];\n var clr = [Math.round(r1 + (r2 - r1) * percent), Math.round(g1 + (g2 - g1) * percent), Math.round(b1 + (b2 - b1) * percent), Math.round(a1 + (a2 - a1) * percent)];\n flatProp = {\n // colours are simple, so just create the flat property instead of expensive string parsing\n bypass: prop.bypass,\n // we're a bypass if the mapping property is a bypass\n name: prop.name,\n value: clr,\n strValue: 'rgb(' + clr[0] + ', ' + clr[1] + ', ' + clr[2] + ')'\n };\n } else if (type.number) {\n var calcValue = prop.valueMin + (prop.valueMax - prop.valueMin) * percent;\n flatProp = this.parse(prop.name, calcValue, prop.bypass, flatPropMapping);\n } else {\n return false; // can only map to colours and numbers\n }\n if (!flatProp) {\n // if we can't flatten the property, then don't apply the property and fall back on the existing style\n printMappingErr();\n return false;\n }\n flatProp.mapping = prop; // keep a reference to the mapping\n prop = flatProp; // the flattened (mapped) property is the one we want\n\n break;\n }\n\n // direct mapping\n case types.data:\n {\n // flatten the field (e.g. data.foo.bar)\n var _fields = prop.field.split('.');\n var _fieldVal = _p.data;\n for (var _i3 = 0; _i3 < _fields.length && _fieldVal; _i3++) {\n var _field = _fields[_i3];\n _fieldVal = _fieldVal[_field];\n }\n if (_fieldVal != null) {\n flatProp = this.parse(prop.name, _fieldVal, prop.bypass, flatPropMapping);\n }\n if (!flatProp) {\n // if we can't flatten the property, then don't apply and fall back on the existing style\n printMappingErr();\n return false;\n }\n flatProp.mapping = prop; // keep a reference to the mapping\n prop = flatProp; // the flattened (mapped) property is the one we want\n\n break;\n }\n case types.fn:\n {\n var fn = prop.value;\n var fnRetVal = prop.fnValue != null ? prop.fnValue : fn(ele); // check for cached value before calling function\n\n prop.prevFnValue = fnRetVal;\n if (fnRetVal == null) {\n warn('Custom function mappers may not return null (i.e. `' + prop.name + '` for ele `' + ele.id() + '` is null)');\n return false;\n }\n flatProp = this.parse(prop.name, fnRetVal, prop.bypass, flatPropMapping);\n if (!flatProp) {\n warn('Custom function mappers may not return invalid values for the property type (i.e. `' + prop.name + '` for ele `' + ele.id() + '` is invalid)');\n return false;\n }\n flatProp.mapping = copy(prop); // keep a reference to the mapping\n prop = flatProp; // the flattened (mapped) property is the one we want\n\n break;\n }\n case undefined:\n break;\n // just set the property\n\n default:\n return false;\n // not a valid mapping\n }\n\n // if the property is a bypass property, then link the resultant property to the original one\n if (propIsBypass) {\n if (origPropIsBypass) {\n // then this bypass overrides the existing one\n prop.bypassed = origProp.bypassed; // steal bypassed prop from old bypass\n } else {\n // then link the orig prop to the new bypass\n prop.bypassed = origProp;\n }\n style[prop.name] = prop; // and set\n } else {\n // prop is not bypass\n if (origPropIsBypass) {\n // then keep the orig prop (since it's a bypass) and link to the new prop\n origProp.bypassed = prop;\n } else {\n // then just replace the old prop with the new one\n style[prop.name] = prop;\n }\n }\n checkTriggers();\n return true;\n};\nstyfn$8.cleanElements = function (eles, keepBypasses) {\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n this.clearStyleHints(ele);\n ele.dirtyCompoundBoundsCache();\n ele.dirtyBoundingBoxCache();\n if (!keepBypasses) {\n ele._private.style = {};\n } else {\n var style = ele._private.style;\n var propNames = Object.keys(style);\n for (var j = 0; j < propNames.length; j++) {\n var propName = propNames[j];\n var eleProp = style[propName];\n if (eleProp != null) {\n if (eleProp.bypass) {\n eleProp.bypassed = null;\n } else {\n style[propName] = null;\n }\n }\n }\n }\n }\n};\n\n// updates the visual style for all elements (useful for manual style modification after init)\nstyfn$8.update = function () {\n var cy = this._private.cy;\n var eles = cy.mutableElements();\n eles.updateStyle();\n};\n\n// diffProps : { name => { prev, next } }\nstyfn$8.updateTransitions = function (ele, diffProps) {\n var self = this;\n var _p = ele._private;\n var props = ele.pstyle('transition-property').value;\n var duration = ele.pstyle('transition-duration').pfValue;\n var delay = ele.pstyle('transition-delay').pfValue;\n if (props.length > 0 && duration > 0) {\n var style = {};\n\n // build up the style to animate towards\n var anyPrev = false;\n for (var i = 0; i < props.length; i++) {\n var prop = props[i];\n var styProp = ele.pstyle(prop);\n var diffProp = diffProps[prop];\n if (!diffProp) {\n continue;\n }\n var prevProp = diffProp.prev;\n var fromProp = prevProp;\n var toProp = diffProp.next != null ? diffProp.next : styProp;\n var diff = false;\n var initVal = undefined;\n var initDt = 0.000001; // delta time % value for initVal (allows animating out of init zero opacity)\n\n if (!fromProp) {\n continue;\n }\n\n // consider px values\n if (number$1(fromProp.pfValue) && number$1(toProp.pfValue)) {\n diff = toProp.pfValue - fromProp.pfValue; // nonzero is truthy\n initVal = fromProp.pfValue + initDt * diff;\n\n // consider numerical values\n } else if (number$1(fromProp.value) && number$1(toProp.value)) {\n diff = toProp.value - fromProp.value; // nonzero is truthy\n initVal = fromProp.value + initDt * diff;\n\n // consider colour values\n } else if (array(fromProp.value) && array(toProp.value)) {\n diff = fromProp.value[0] !== toProp.value[0] || fromProp.value[1] !== toProp.value[1] || fromProp.value[2] !== toProp.value[2];\n initVal = fromProp.strValue;\n }\n\n // the previous value is good for an animation only if it's different\n if (diff) {\n style[prop] = toProp.strValue; // to val\n this.applyBypass(ele, prop, initVal); // from val\n anyPrev = true;\n }\n } // end if props allow ani\n\n // can't transition if there's nothing previous to transition from\n if (!anyPrev) {\n return;\n }\n _p.transitioning = true;\n new Promise$1(function (resolve) {\n if (delay > 0) {\n ele.delayAnimation(delay).play().promise().then(resolve);\n } else {\n resolve();\n }\n }).then(function () {\n return ele.animation({\n style: style,\n duration: duration,\n easing: ele.pstyle('transition-timing-function').value,\n queue: false\n }).play().promise();\n }).then(function () {\n // if( !isBypass ){\n self.removeBypasses(ele, props);\n ele.emitAndNotify('style');\n // }\n\n _p.transitioning = false;\n });\n } else if (_p.transitioning) {\n this.removeBypasses(ele, props);\n ele.emitAndNotify('style');\n _p.transitioning = false;\n }\n};\nstyfn$8.checkTrigger = function (ele, name, fromValue, toValue, getTrigger, onTrigger) {\n var prop = this.properties[name];\n var triggerCheck = getTrigger(prop);\n if (ele.removed()) {\n return;\n }\n if (triggerCheck != null && triggerCheck(fromValue, toValue, ele)) {\n onTrigger(prop);\n }\n};\nstyfn$8.checkZOrderTrigger = function (ele, name, fromValue, toValue) {\n var _this = this;\n this.checkTrigger(ele, name, fromValue, toValue, function (prop) {\n return prop.triggersZOrder;\n }, function () {\n _this._private.cy.notify('zorder', ele);\n });\n};\nstyfn$8.checkBoundsTrigger = function (ele, name, fromValue, toValue) {\n this.checkTrigger(ele, name, fromValue, toValue, function (prop) {\n return prop.triggersBounds;\n }, function (prop) {\n ele.dirtyCompoundBoundsCache();\n ele.dirtyBoundingBoxCache();\n });\n};\nstyfn$8.checkConnectedEdgesBoundsTrigger = function (ele, name, fromValue, toValue) {\n this.checkTrigger(ele, name, fromValue, toValue, function (prop) {\n return prop.triggersBoundsOfConnectedEdges;\n }, function (prop) {\n ele.connectedEdges().forEach(function (edge) {\n edge.dirtyBoundingBoxCache();\n });\n });\n};\nstyfn$8.checkParallelEdgesBoundsTrigger = function (ele, name, fromValue, toValue) {\n this.checkTrigger(ele, name, fromValue, toValue, function (prop) {\n return prop.triggersBoundsOfParallelEdges;\n }, function (prop) {\n ele.parallelEdges().forEach(function (pllEdge) {\n pllEdge.dirtyBoundingBoxCache();\n });\n });\n};\nstyfn$8.checkTriggers = function (ele, name, fromValue, toValue) {\n ele.dirtyStyleCache();\n this.checkZOrderTrigger(ele, name, fromValue, toValue);\n this.checkBoundsTrigger(ele, name, fromValue, toValue);\n this.checkConnectedEdgesBoundsTrigger(ele, name, fromValue, toValue);\n this.checkParallelEdgesBoundsTrigger(ele, name, fromValue, toValue);\n};\n\nvar styfn$7 = {};\n\n// bypasses are applied to an existing style on an element, and just tacked on temporarily\n// returns true iff application was successful for at least 1 specified property\nstyfn$7.applyBypass = function (eles, name, value, updateTransitions) {\n var self = this;\n var props = [];\n var isBypass = true;\n\n // put all the properties (can specify one or many) in an array after parsing them\n if (name === '*' || name === '**') {\n // apply to all property names\n\n if (value !== undefined) {\n for (var i = 0; i < self.properties.length; i++) {\n var prop = self.properties[i];\n var _name = prop.name;\n var parsedProp = this.parse(_name, value, true);\n if (parsedProp) {\n props.push(parsedProp);\n }\n }\n }\n } else if (string(name)) {\n // then parse the single property\n var _parsedProp = this.parse(name, value, true);\n if (_parsedProp) {\n props.push(_parsedProp);\n }\n } else if (plainObject(name)) {\n // then parse each property\n var specifiedProps = name;\n updateTransitions = value;\n var names = Object.keys(specifiedProps);\n for (var _i = 0; _i < names.length; _i++) {\n var _name2 = names[_i];\n var _value = specifiedProps[_name2];\n if (_value === undefined) {\n // try camel case name too\n _value = specifiedProps[dash2camel(_name2)];\n }\n if (_value !== undefined) {\n var _parsedProp2 = this.parse(_name2, _value, true);\n if (_parsedProp2) {\n props.push(_parsedProp2);\n }\n }\n }\n } else {\n // can't do anything without well defined properties\n return false;\n }\n\n // we've failed if there are no valid properties\n if (props.length === 0) {\n return false;\n }\n\n // now, apply the bypass properties on the elements\n var ret = false; // return true if at least one succesful bypass applied\n for (var _i2 = 0; _i2 < eles.length; _i2++) {\n // for each ele\n var ele = eles[_i2];\n var diffProps = {};\n var diffProp = undefined;\n for (var j = 0; j < props.length; j++) {\n // for each prop\n var _prop = props[j];\n if (updateTransitions) {\n var prevProp = ele.pstyle(_prop.name);\n diffProp = diffProps[_prop.name] = {\n prev: prevProp\n };\n }\n ret = this.applyParsedProperty(ele, copy(_prop)) || ret;\n if (updateTransitions) {\n diffProp.next = ele.pstyle(_prop.name);\n }\n } // for props\n\n if (ret) {\n this.updateStyleHints(ele);\n }\n if (updateTransitions) {\n this.updateTransitions(ele, diffProps, isBypass);\n }\n } // for eles\n\n return ret;\n};\n\n// only useful in specific cases like animation\nstyfn$7.overrideBypass = function (eles, name, value) {\n name = camel2dash(name);\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n var prop = ele._private.style[name];\n var type = this.properties[name].type;\n var isColor = type.color;\n var isMulti = type.mutiple;\n var oldValue = !prop ? null : prop.pfValue != null ? prop.pfValue : prop.value;\n if (!prop || !prop.bypass) {\n // need a bypass if one doesn't exist\n this.applyBypass(ele, name, value);\n } else {\n prop.value = value;\n if (prop.pfValue != null) {\n prop.pfValue = value;\n }\n if (isColor) {\n prop.strValue = 'rgb(' + value.join(',') + ')';\n } else if (isMulti) {\n prop.strValue = value.join(' ');\n } else {\n prop.strValue = '' + value;\n }\n this.updateStyleHints(ele);\n }\n this.checkTriggers(ele, name, oldValue, value);\n }\n};\nstyfn$7.removeAllBypasses = function (eles, updateTransitions) {\n return this.removeBypasses(eles, this.propertyNames, updateTransitions);\n};\nstyfn$7.removeBypasses = function (eles, props, updateTransitions) {\n var isBypass = true;\n for (var j = 0; j < eles.length; j++) {\n var ele = eles[j];\n var diffProps = {};\n for (var i = 0; i < props.length; i++) {\n var name = props[i];\n var prop = this.properties[name];\n var prevProp = ele.pstyle(prop.name);\n if (!prevProp || !prevProp.bypass) {\n // if a bypass doesn't exist for the prop, nothing needs to be removed\n continue;\n }\n var value = ''; // empty => remove bypass\n var parsedProp = this.parse(name, value, true);\n var diffProp = diffProps[prop.name] = {\n prev: prevProp\n };\n this.applyParsedProperty(ele, parsedProp);\n diffProp.next = ele.pstyle(prop.name);\n } // for props\n\n this.updateStyleHints(ele);\n if (updateTransitions) {\n this.updateTransitions(ele, diffProps, isBypass);\n }\n } // for eles\n};\n\nvar styfn$6 = {};\n\n// gets what an em size corresponds to in pixels relative to a dom element\nstyfn$6.getEmSizeInPixels = function () {\n var px = this.containerCss('font-size');\n if (px != null) {\n return parseFloat(px);\n } else {\n return 1; // for headless\n }\n};\n\n// gets css property from the core container\nstyfn$6.containerCss = function (propName) {\n var cy = this._private.cy;\n var domElement = cy.container();\n var containerWindow = cy.window();\n if (containerWindow && domElement && containerWindow.getComputedStyle) {\n return containerWindow.getComputedStyle(domElement).getPropertyValue(propName);\n }\n};\n\nvar styfn$5 = {};\n\n// gets the rendered style for an element\nstyfn$5.getRenderedStyle = function (ele, prop) {\n if (prop) {\n return this.getStylePropertyValue(ele, prop, true);\n } else {\n return this.getRawStyle(ele, true);\n }\n};\n\n// gets the raw style for an element\nstyfn$5.getRawStyle = function (ele, isRenderedVal) {\n var self = this;\n ele = ele[0]; // insure it's an element\n\n if (ele) {\n var rstyle = {};\n for (var i = 0; i < self.properties.length; i++) {\n var prop = self.properties[i];\n var val = self.getStylePropertyValue(ele, prop.name, isRenderedVal);\n if (val != null) {\n rstyle[prop.name] = val;\n rstyle[dash2camel(prop.name)] = val;\n }\n }\n return rstyle;\n }\n};\nstyfn$5.getIndexedStyle = function (ele, property, subproperty, index) {\n var pstyle = ele.pstyle(property)[subproperty][index];\n return pstyle != null ? pstyle : ele.cy().style().getDefaultProperty(property)[subproperty][0];\n};\nstyfn$5.getStylePropertyValue = function (ele, propName, isRenderedVal) {\n var self = this;\n ele = ele[0]; // insure it's an element\n\n if (ele) {\n var prop = self.properties[propName];\n if (prop.alias) {\n prop = prop.pointsTo;\n }\n var type = prop.type;\n var styleProp = ele.pstyle(prop.name);\n if (styleProp) {\n var value = styleProp.value,\n units = styleProp.units,\n strValue = styleProp.strValue;\n if (isRenderedVal && type.number && value != null && number$1(value)) {\n var zoom = ele.cy().zoom();\n var getRenderedValue = function getRenderedValue(val) {\n return val * zoom;\n };\n var getValueStringWithUnits = function getValueStringWithUnits(val, units) {\n return getRenderedValue(val) + units;\n };\n var isArrayValue = array(value);\n var haveUnits = isArrayValue ? units.every(function (u) {\n return u != null;\n }) : units != null;\n if (haveUnits) {\n if (isArrayValue) {\n return value.map(function (v, i) {\n return getValueStringWithUnits(v, units[i]);\n }).join(' ');\n } else {\n return getValueStringWithUnits(value, units);\n }\n } else {\n if (isArrayValue) {\n return value.map(function (v) {\n return string(v) ? v : '' + getRenderedValue(v);\n }).join(' ');\n } else {\n return '' + getRenderedValue(value);\n }\n }\n } else if (strValue != null) {\n return strValue;\n }\n }\n return null;\n }\n};\nstyfn$5.getAnimationStartStyle = function (ele, aniProps) {\n var rstyle = {};\n for (var i = 0; i < aniProps.length; i++) {\n var aniProp = aniProps[i];\n var name = aniProp.name;\n var styleProp = ele.pstyle(name);\n if (styleProp !== undefined) {\n // then make a prop of it\n if (plainObject(styleProp)) {\n styleProp = this.parse(name, styleProp.strValue);\n } else {\n styleProp = this.parse(name, styleProp);\n }\n }\n if (styleProp) {\n rstyle[name] = styleProp;\n }\n }\n return rstyle;\n};\nstyfn$5.getPropsList = function (propsObj) {\n var self = this;\n var rstyle = [];\n var style = propsObj;\n var props = self.properties;\n if (style) {\n var names = Object.keys(style);\n for (var i = 0; i < names.length; i++) {\n var name = names[i];\n var val = style[name];\n var prop = props[name] || props[camel2dash(name)];\n var styleProp = this.parse(prop.name, val);\n if (styleProp) {\n rstyle.push(styleProp);\n }\n }\n }\n return rstyle;\n};\nstyfn$5.getNonDefaultPropertiesHash = function (ele, propNames, seed) {\n var hash = seed.slice();\n var name, val, strVal, chVal;\n var i, j;\n for (i = 0; i < propNames.length; i++) {\n name = propNames[i];\n val = ele.pstyle(name, false);\n if (val == null) {\n continue;\n } else if (val.pfValue != null) {\n hash[0] = hashInt(chVal, hash[0]);\n hash[1] = hashIntAlt(chVal, hash[1]);\n } else {\n strVal = val.strValue;\n for (j = 0; j < strVal.length; j++) {\n chVal = strVal.charCodeAt(j);\n hash[0] = hashInt(chVal, hash[0]);\n hash[1] = hashIntAlt(chVal, hash[1]);\n }\n }\n }\n return hash;\n};\nstyfn$5.getPropertiesHash = styfn$5.getNonDefaultPropertiesHash;\n\nvar styfn$4 = {};\nstyfn$4.appendFromJson = function (json) {\n var style = this;\n for (var i = 0; i < json.length; i++) {\n var context = json[i];\n var selector = context.selector;\n var props = context.style || context.css;\n var names = Object.keys(props);\n style.selector(selector); // apply selector\n\n for (var j = 0; j < names.length; j++) {\n var name = names[j];\n var value = props[name];\n style.css(name, value); // apply property\n }\n }\n return style;\n};\n\n// accessible cy.style() function\nstyfn$4.fromJson = function (json) {\n var style = this;\n style.resetToDefault();\n style.appendFromJson(json);\n return style;\n};\n\n// get json from cy.style() api\nstyfn$4.json = function () {\n var json = [];\n for (var i = this.defaultLength; i < this.length; i++) {\n var cxt = this[i];\n var selector = cxt.selector;\n var props = cxt.properties;\n var css = {};\n for (var j = 0; j < props.length; j++) {\n var prop = props[j];\n css[prop.name] = prop.strValue;\n }\n json.push({\n selector: !selector ? 'core' : selector.toString(),\n style: css\n });\n }\n return json;\n};\n\nvar styfn$3 = {};\nstyfn$3.appendFromString = function (string) {\n var self = this;\n var style = this;\n var remaining = '' + string;\n var selAndBlockStr;\n var blockRem;\n var propAndValStr;\n\n // remove comments from the style string\n remaining = remaining.replace(/[/][*](\\s|.)+?[*][/]/g, '');\n function removeSelAndBlockFromRemaining() {\n // remove the parsed selector and block from the remaining text to parse\n if (remaining.length > selAndBlockStr.length) {\n remaining = remaining.substr(selAndBlockStr.length);\n } else {\n remaining = '';\n }\n }\n function removePropAndValFromRem() {\n // remove the parsed property and value from the remaining block text to parse\n if (blockRem.length > propAndValStr.length) {\n blockRem = blockRem.substr(propAndValStr.length);\n } else {\n blockRem = '';\n }\n }\n for (;;) {\n var nothingLeftToParse = remaining.match(/^\\s*$/);\n if (nothingLeftToParse) {\n break;\n }\n var selAndBlock = remaining.match(/^\\s*((?:.|\\s)+?)\\s*\\{((?:.|\\s)+?)\\}/);\n if (!selAndBlock) {\n warn('Halting stylesheet parsing: String stylesheet contains more to parse but no selector and block found in: ' + remaining);\n break;\n }\n selAndBlockStr = selAndBlock[0];\n\n // parse the selector\n var selectorStr = selAndBlock[1];\n if (selectorStr !== 'core') {\n var selector = new Selector(selectorStr);\n if (selector.invalid) {\n warn('Skipping parsing of block: Invalid selector found in string stylesheet: ' + selectorStr);\n\n // skip this selector and block\n removeSelAndBlockFromRemaining();\n continue;\n }\n }\n\n // parse the block of properties and values\n var blockStr = selAndBlock[2];\n var invalidBlock = false;\n blockRem = blockStr;\n var props = [];\n for (;;) {\n var _nothingLeftToParse = blockRem.match(/^\\s*$/);\n if (_nothingLeftToParse) {\n break;\n }\n var propAndVal = blockRem.match(/^\\s*(.+?)\\s*:\\s*(.+?)(?:\\s*;|\\s*$)/);\n if (!propAndVal) {\n warn('Skipping parsing of block: Invalid formatting of style property and value definitions found in:' + blockStr);\n invalidBlock = true;\n break;\n }\n propAndValStr = propAndVal[0];\n var propStr = propAndVal[1];\n var valStr = propAndVal[2];\n var prop = self.properties[propStr];\n if (!prop) {\n warn('Skipping property: Invalid property name in: ' + propAndValStr);\n\n // skip this property in the block\n removePropAndValFromRem();\n continue;\n }\n var parsedProp = style.parse(propStr, valStr);\n if (!parsedProp) {\n warn('Skipping property: Invalid property definition in: ' + propAndValStr);\n\n // skip this property in the block\n removePropAndValFromRem();\n continue;\n }\n props.push({\n name: propStr,\n val: valStr\n });\n removePropAndValFromRem();\n }\n if (invalidBlock) {\n removeSelAndBlockFromRemaining();\n break;\n }\n\n // put the parsed block in the style\n style.selector(selectorStr);\n for (var i = 0; i < props.length; i++) {\n var _prop = props[i];\n style.css(_prop.name, _prop.val);\n }\n removeSelAndBlockFromRemaining();\n }\n return style;\n};\nstyfn$3.fromString = function (string) {\n var style = this;\n style.resetToDefault();\n style.appendFromString(string);\n return style;\n};\n\nvar styfn$2 = {};\n(function () {\n var number$1 = number;\n var rgba = rgbaNoBackRefs;\n var hsla = hslaNoBackRefs;\n var hex3$1 = hex3;\n var hex6$1 = hex6;\n var data = function data(prefix) {\n return '^' + prefix + '\\\\s*\\\\(\\\\s*([\\\\w\\\\.]+)\\\\s*\\\\)$';\n };\n var mapData = function mapData(prefix) {\n var mapArg = number$1 + '|\\\\w+|' + rgba + '|' + hsla + '|' + hex3$1 + '|' + hex6$1;\n return '^' + prefix + '\\\\s*\\\\(([\\\\w\\\\.]+)\\\\s*\\\\,\\\\s*(' + number$1 + ')\\\\s*\\\\,\\\\s*(' + number$1 + ')\\\\s*,\\\\s*(' + mapArg + ')\\\\s*\\\\,\\\\s*(' + mapArg + ')\\\\)$';\n };\n var urlRegexes = ['^url\\\\s*\\\\(\\\\s*[\\'\"]?(.+?)[\\'\"]?\\\\s*\\\\)$', '^(none)$', '^(.+)$'];\n\n // each visual style property has a type and needs to be validated according to it\n styfn$2.types = {\n time: {\n number: true,\n min: 0,\n units: 's|ms',\n implicitUnits: 'ms'\n },\n percent: {\n number: true,\n min: 0,\n max: 100,\n units: '%',\n implicitUnits: '%'\n },\n percentages: {\n number: true,\n min: 0,\n max: 100,\n units: '%',\n implicitUnits: '%',\n multiple: true\n },\n zeroOneNumber: {\n number: true,\n min: 0,\n max: 1,\n unitless: true\n },\n zeroOneNumbers: {\n number: true,\n min: 0,\n max: 1,\n unitless: true,\n multiple: true\n },\n nOneOneNumber: {\n number: true,\n min: -1,\n max: 1,\n unitless: true\n },\n nonNegativeInt: {\n number: true,\n min: 0,\n integer: true,\n unitless: true\n },\n nonNegativeNumber: {\n number: true,\n min: 0,\n unitless: true\n },\n position: {\n enums: ['parent', 'origin']\n },\n nodeSize: {\n number: true,\n min: 0,\n enums: ['label']\n },\n number: {\n number: true,\n unitless: true\n },\n numbers: {\n number: true,\n unitless: true,\n multiple: true\n },\n positiveNumber: {\n number: true,\n unitless: true,\n min: 0,\n strictMin: true\n },\n size: {\n number: true,\n min: 0\n },\n bidirectionalSize: {\n number: true\n },\n // allows negative\n bidirectionalSizeMaybePercent: {\n number: true,\n allowPercent: true\n },\n // allows negative\n bidirectionalSizes: {\n number: true,\n multiple: true\n },\n // allows negative\n sizeMaybePercent: {\n number: true,\n min: 0,\n allowPercent: true\n },\n axisDirection: {\n enums: ['horizontal', 'leftward', 'rightward', 'vertical', 'upward', 'downward', 'auto']\n },\n axisDirectionExplicit: {\n enums: ['leftward', 'rightward', 'upward', 'downward']\n },\n axisDirectionPrimary: {\n enums: ['horizontal', 'vertical']\n },\n paddingRelativeTo: {\n enums: ['width', 'height', 'average', 'min', 'max']\n },\n bgWH: {\n number: true,\n min: 0,\n allowPercent: true,\n enums: ['auto'],\n multiple: true\n },\n bgPos: {\n number: true,\n allowPercent: true,\n multiple: true\n },\n bgRelativeTo: {\n enums: ['inner', 'include-padding'],\n multiple: true\n },\n bgRepeat: {\n enums: ['repeat', 'repeat-x', 'repeat-y', 'no-repeat'],\n multiple: true\n },\n bgFit: {\n enums: ['none', 'contain', 'cover'],\n multiple: true\n },\n bgCrossOrigin: {\n enums: ['anonymous', 'use-credentials', 'null'],\n multiple: true\n },\n bgClip: {\n enums: ['none', 'node'],\n multiple: true\n },\n bgContainment: {\n enums: ['inside', 'over'],\n multiple: true\n },\n boxSelection: {\n enums: ['contain', 'overlap', 'none']\n },\n color: {\n color: true\n },\n colors: {\n color: true,\n multiple: true\n },\n fill: {\n enums: ['solid', 'linear-gradient', 'radial-gradient']\n },\n bool: {\n enums: ['yes', 'no']\n },\n bools: {\n enums: ['yes', 'no'],\n multiple: true\n },\n lineStyle: {\n enums: ['solid', 'dotted', 'dashed']\n },\n lineCap: {\n enums: ['butt', 'round', 'square']\n },\n linePosition: {\n enums: ['center', 'inside', 'outside']\n },\n lineJoin: {\n enums: ['round', 'bevel', 'miter']\n },\n borderStyle: {\n enums: ['solid', 'dotted', 'dashed', 'double']\n },\n curveStyle: {\n enums: ['bezier', 'unbundled-bezier', 'haystack', 'segments', 'straight', 'straight-triangle', 'taxi', 'round-segments', 'round-taxi']\n },\n radiusType: {\n enums: ['arc-radius', 'influence-radius'],\n multiple: true\n },\n fontFamily: {\n regex: '^([\\\\w- \\\\\"]+(?:\\\\s*,\\\\s*[\\\\w- \\\\\"]+)*)$'\n },\n fontStyle: {\n enums: ['italic', 'normal', 'oblique']\n },\n fontWeight: {\n enums: ['normal', 'bold', 'bolder', 'lighter', '100', '200', '300', '400', '500', '600', '800', '900', 100, 200, 300, 400, 500, 600, 700, 800, 900]\n },\n textDecoration: {\n enums: ['none', 'underline', 'overline', 'line-through']\n },\n textTransform: {\n enums: ['none', 'uppercase', 'lowercase']\n },\n textWrap: {\n enums: ['none', 'wrap', 'ellipsis']\n },\n textOverflowWrap: {\n enums: ['whitespace', 'anywhere']\n },\n textBackgroundShape: {\n enums: ['rectangle', 'roundrectangle', 'round-rectangle', 'circle']\n },\n nodeShape: {\n enums: ['rectangle', 'roundrectangle', 'round-rectangle', 'cutrectangle', 'cut-rectangle', 'bottomroundrectangle', 'bottom-round-rectangle', 'barrel', 'ellipse', 'triangle', 'round-triangle', 'square', 'pentagon', 'round-pentagon', 'hexagon', 'round-hexagon', 'concavehexagon', 'concave-hexagon', 'heptagon', 'round-heptagon', 'octagon', 'round-octagon', 'tag', 'round-tag', 'star', 'diamond', 'round-diamond', 'vee', 'rhomboid', 'right-rhomboid', 'polygon']\n },\n overlayShape: {\n enums: ['roundrectangle', 'round-rectangle', 'ellipse']\n },\n cornerRadius: {\n number: true,\n min: 0,\n units: 'px|em',\n implicitUnits: 'px',\n enums: ['auto']\n },\n compoundIncludeLabels: {\n enums: ['include', 'exclude']\n },\n arrowShape: {\n enums: ['tee', 'triangle', 'triangle-tee', 'circle-triangle', 'triangle-cross', 'triangle-backcurve', 'vee', 'square', 'circle', 'diamond', 'chevron', 'none']\n },\n arrowFill: {\n enums: ['filled', 'hollow']\n },\n arrowWidth: {\n number: true,\n units: '%|px|em',\n implicitUnits: 'px',\n enums: ['match-line']\n },\n display: {\n enums: ['element', 'none']\n },\n visibility: {\n enums: ['hidden', 'visible']\n },\n zCompoundDepth: {\n enums: ['bottom', 'orphan', 'auto', 'top']\n },\n zIndexCompare: {\n enums: ['auto', 'manual']\n },\n valign: {\n enums: ['top', 'center', 'bottom']\n },\n halign: {\n enums: ['left', 'center', 'right']\n },\n justification: {\n enums: ['left', 'center', 'right', 'auto']\n },\n text: {\n string: true\n },\n data: {\n mapping: true,\n regex: data('data')\n },\n layoutData: {\n mapping: true,\n regex: data('layoutData')\n },\n scratch: {\n mapping: true,\n regex: data('scratch')\n },\n mapData: {\n mapping: true,\n regex: mapData('mapData')\n },\n mapLayoutData: {\n mapping: true,\n regex: mapData('mapLayoutData')\n },\n mapScratch: {\n mapping: true,\n regex: mapData('mapScratch')\n },\n fn: {\n mapping: true,\n fn: true\n },\n url: {\n regexes: urlRegexes,\n singleRegexMatchValue: true\n },\n urls: {\n regexes: urlRegexes,\n singleRegexMatchValue: true,\n multiple: true\n },\n propList: {\n propList: true\n },\n angle: {\n number: true,\n units: 'deg|rad',\n implicitUnits: 'rad'\n },\n textRotation: {\n number: true,\n units: 'deg|rad',\n implicitUnits: 'rad',\n enums: ['none', 'autorotate']\n },\n polygonPointList: {\n number: true,\n multiple: true,\n evenMultiple: true,\n min: -1,\n max: 1,\n unitless: true\n },\n edgeDistances: {\n enums: ['intersection', 'node-position', 'endpoints']\n },\n edgeEndpoint: {\n number: true,\n multiple: true,\n units: '%|px|em|deg|rad',\n implicitUnits: 'px',\n enums: ['inside-to-node', 'outside-to-node', 'outside-to-node-or-label', 'outside-to-line', 'outside-to-line-or-label'],\n singleEnum: true,\n validate: function validate(valArr, unitsArr) {\n switch (valArr.length) {\n case 2:\n // can be % or px only\n return unitsArr[0] !== 'deg' && unitsArr[0] !== 'rad' && unitsArr[1] !== 'deg' && unitsArr[1] !== 'rad';\n case 1:\n // can be enum, deg, or rad only\n return string(valArr[0]) || unitsArr[0] === 'deg' || unitsArr[0] === 'rad';\n default:\n return false;\n }\n }\n },\n easing: {\n regexes: ['^(spring)\\\\s*\\\\(\\\\s*(' + number$1 + ')\\\\s*,\\\\s*(' + number$1 + ')\\\\s*\\\\)$', '^(cubic-bezier)\\\\s*\\\\(\\\\s*(' + number$1 + ')\\\\s*,\\\\s*(' + number$1 + ')\\\\s*,\\\\s*(' + number$1 + ')\\\\s*,\\\\s*(' + number$1 + ')\\\\s*\\\\)$'],\n enums: ['linear', 'ease', 'ease-in', 'ease-out', 'ease-in-out', 'ease-in-sine', 'ease-out-sine', 'ease-in-out-sine', 'ease-in-quad', 'ease-out-quad', 'ease-in-out-quad', 'ease-in-cubic', 'ease-out-cubic', 'ease-in-out-cubic', 'ease-in-quart', 'ease-out-quart', 'ease-in-out-quart', 'ease-in-quint', 'ease-out-quint', 'ease-in-out-quint', 'ease-in-expo', 'ease-out-expo', 'ease-in-out-expo', 'ease-in-circ', 'ease-out-circ', 'ease-in-out-circ']\n },\n gradientDirection: {\n enums: ['to-bottom', 'to-top', 'to-left', 'to-right', 'to-bottom-right', 'to-bottom-left', 'to-top-right', 'to-top-left', 'to-right-bottom', 'to-left-bottom', 'to-right-top', 'to-left-top' // different order\n ]\n },\n boundsExpansion: {\n number: true,\n multiple: true,\n min: 0,\n validate: function validate(valArr) {\n var length = valArr.length;\n return length === 1 || length === 2 || length === 4;\n }\n }\n };\n var diff = {\n zeroNonZero: function zeroNonZero(val1, val2) {\n if ((val1 == null || val2 == null) && val1 !== val2) {\n return true; // null cases could represent any value\n }\n if (val1 == 0 && val2 != 0) {\n return true;\n } else if (val1 != 0 && val2 == 0) {\n return true;\n } else {\n return false;\n }\n },\n any: function any(val1, val2) {\n return val1 != val2;\n },\n emptyNonEmpty: function emptyNonEmpty(str1, str2) {\n var empty1 = emptyString(str1);\n var empty2 = emptyString(str2);\n return empty1 && !empty2 || !empty1 && empty2;\n }\n };\n\n // define visual style properties\n //\n // - n.b. adding a new group of props may require updates to updateStyleHints()\n // - adding new props to an existing group gets handled automatically\n\n var t = styfn$2.types;\n var mainLabel = [{\n name: 'label',\n type: t.text,\n triggersBounds: diff.any,\n triggersZOrder: diff.emptyNonEmpty\n }, {\n name: 'text-rotation',\n type: t.textRotation,\n triggersBounds: diff.any\n }, {\n name: 'text-margin-x',\n type: t.bidirectionalSize,\n triggersBounds: diff.any\n }, {\n name: 'text-margin-y',\n type: t.bidirectionalSize,\n triggersBounds: diff.any\n }];\n var sourceLabel = [{\n name: 'source-label',\n type: t.text,\n triggersBounds: diff.any\n }, {\n name: 'source-text-rotation',\n type: t.textRotation,\n triggersBounds: diff.any\n }, {\n name: 'source-text-margin-x',\n type: t.bidirectionalSize,\n triggersBounds: diff.any\n }, {\n name: 'source-text-margin-y',\n type: t.bidirectionalSize,\n triggersBounds: diff.any\n }, {\n name: 'source-text-offset',\n type: t.size,\n triggersBounds: diff.any\n }];\n var targetLabel = [{\n name: 'target-label',\n type: t.text,\n triggersBounds: diff.any\n }, {\n name: 'target-text-rotation',\n type: t.textRotation,\n triggersBounds: diff.any\n }, {\n name: 'target-text-margin-x',\n type: t.bidirectionalSize,\n triggersBounds: diff.any\n }, {\n name: 'target-text-margin-y',\n type: t.bidirectionalSize,\n triggersBounds: diff.any\n }, {\n name: 'target-text-offset',\n type: t.size,\n triggersBounds: diff.any\n }];\n var labelDimensions = [{\n name: 'font-family',\n type: t.fontFamily,\n triggersBounds: diff.any\n }, {\n name: 'font-style',\n type: t.fontStyle,\n triggersBounds: diff.any\n }, {\n name: 'font-weight',\n type: t.fontWeight,\n triggersBounds: diff.any\n }, {\n name: 'font-size',\n type: t.size,\n triggersBounds: diff.any\n }, {\n name: 'text-transform',\n type: t.textTransform,\n triggersBounds: diff.any\n }, {\n name: 'text-wrap',\n type: t.textWrap,\n triggersBounds: diff.any\n }, {\n name: 'text-overflow-wrap',\n type: t.textOverflowWrap,\n triggersBounds: diff.any\n }, {\n name: 'text-max-width',\n type: t.size,\n triggersBounds: diff.any\n }, {\n name: 'text-outline-width',\n type: t.size,\n triggersBounds: diff.any\n }, {\n name: 'line-height',\n type: t.positiveNumber,\n triggersBounds: diff.any\n }];\n var commonLabel = [{\n name: 'text-valign',\n type: t.valign,\n triggersBounds: diff.any\n }, {\n name: 'text-halign',\n type: t.halign,\n triggersBounds: diff.any\n }, {\n name: 'color',\n type: t.color\n }, {\n name: 'text-outline-color',\n type: t.color\n }, {\n name: 'text-outline-opacity',\n type: t.zeroOneNumber\n }, {\n name: 'text-background-color',\n type: t.color\n }, {\n name: 'text-background-opacity',\n type: t.zeroOneNumber\n }, {\n name: 'text-background-padding',\n type: t.size,\n triggersBounds: diff.any\n }, {\n name: 'text-border-opacity',\n type: t.zeroOneNumber\n }, {\n name: 'text-border-color',\n type: t.color\n }, {\n name: 'text-border-width',\n type: t.size,\n triggersBounds: diff.any\n }, {\n name: 'text-border-style',\n type: t.borderStyle,\n triggersBounds: diff.any\n }, {\n name: 'text-background-shape',\n type: t.textBackgroundShape,\n triggersBounds: diff.any\n }, {\n name: 'text-justification',\n type: t.justification\n }, {\n name: 'box-select-labels',\n type: t.bool,\n triggersBounds: diff.any\n }];\n var behavior = [{\n name: 'events',\n type: t.bool,\n triggersZOrder: diff.any\n }, {\n name: 'text-events',\n type: t.bool,\n triggersZOrder: diff.any\n }, {\n name: 'box-selection',\n type: t.boxSelection,\n triggersZOrder: diff.any\n }];\n var visibility = [{\n name: 'display',\n type: t.display,\n triggersZOrder: diff.any,\n triggersBounds: diff.any,\n triggersBoundsOfConnectedEdges: diff.any,\n triggersBoundsOfParallelEdges: function triggersBoundsOfParallelEdges(fromValue, toValue, ele) {\n if (fromValue === toValue) {\n return false;\n }\n\n // only if edge is bundled bezier (so as not to affect performance of other edges)\n return ele.pstyle('curve-style').value === 'bezier';\n }\n }, {\n name: 'visibility',\n type: t.visibility,\n triggersZOrder: diff.any\n }, {\n name: 'opacity',\n type: t.zeroOneNumber,\n triggersZOrder: diff.zeroNonZero\n }, {\n name: 'text-opacity',\n type: t.zeroOneNumber\n }, {\n name: 'min-zoomed-font-size',\n type: t.size\n }, {\n name: 'z-compound-depth',\n type: t.zCompoundDepth,\n triggersZOrder: diff.any\n }, {\n name: 'z-index-compare',\n type: t.zIndexCompare,\n triggersZOrder: diff.any\n }, {\n name: 'z-index',\n type: t.number,\n triggersZOrder: diff.any\n }];\n var overlay = [{\n name: 'overlay-padding',\n type: t.size,\n triggersBounds: diff.any\n }, {\n name: 'overlay-color',\n type: t.color\n }, {\n name: 'overlay-opacity',\n type: t.zeroOneNumber,\n triggersBounds: diff.zeroNonZero\n }, {\n name: 'overlay-shape',\n type: t.overlayShape,\n triggersBounds: diff.any\n }, {\n name: 'overlay-corner-radius',\n type: t.cornerRadius\n }];\n var underlay = [{\n name: 'underlay-padding',\n type: t.size,\n triggersBounds: diff.any\n }, {\n name: 'underlay-color',\n type: t.color\n }, {\n name: 'underlay-opacity',\n type: t.zeroOneNumber,\n triggersBounds: diff.zeroNonZero\n }, {\n name: 'underlay-shape',\n type: t.overlayShape,\n triggersBounds: diff.any\n }, {\n name: 'underlay-corner-radius',\n type: t.cornerRadius\n }];\n var transition = [{\n name: 'transition-property',\n type: t.propList\n }, {\n name: 'transition-duration',\n type: t.time\n }, {\n name: 'transition-delay',\n type: t.time\n }, {\n name: 'transition-timing-function',\n type: t.easing\n }];\n var nodeSizeHashOverride = function nodeSizeHashOverride(ele, parsedProp) {\n if (parsedProp.value === 'label') {\n return -ele.poolIndex(); // no hash key hits is using label size (hitrate for perf probably low anyway)\n } else {\n return parsedProp.pfValue;\n }\n };\n var nodeBody = [{\n name: 'height',\n type: t.nodeSize,\n triggersBounds: diff.any,\n hashOverride: nodeSizeHashOverride\n }, {\n name: 'width',\n type: t.nodeSize,\n triggersBounds: diff.any,\n hashOverride: nodeSizeHashOverride\n }, {\n name: 'shape',\n type: t.nodeShape,\n triggersBounds: diff.any\n }, {\n name: 'shape-polygon-points',\n type: t.polygonPointList,\n triggersBounds: diff.any\n }, {\n name: 'corner-radius',\n type: t.cornerRadius\n }, {\n name: 'background-color',\n type: t.color\n }, {\n name: 'background-fill',\n type: t.fill\n }, {\n name: 'background-opacity',\n type: t.zeroOneNumber\n }, {\n name: 'background-blacken',\n type: t.nOneOneNumber\n }, {\n name: 'background-gradient-stop-colors',\n type: t.colors\n }, {\n name: 'background-gradient-stop-positions',\n type: t.percentages\n }, {\n name: 'background-gradient-direction',\n type: t.gradientDirection\n }, {\n name: 'padding',\n type: t.sizeMaybePercent,\n triggersBounds: diff.any\n }, {\n name: 'padding-relative-to',\n type: t.paddingRelativeTo,\n triggersBounds: diff.any\n }, {\n name: 'bounds-expansion',\n type: t.boundsExpansion,\n triggersBounds: diff.any\n }];\n var nodeBorder = [{\n name: 'border-color',\n type: t.color\n }, {\n name: 'border-opacity',\n type: t.zeroOneNumber\n }, {\n name: 'border-width',\n type: t.size,\n triggersBounds: diff.any\n }, {\n name: 'border-style',\n type: t.borderStyle\n }, {\n name: 'border-cap',\n type: t.lineCap\n }, {\n name: 'border-join',\n type: t.lineJoin\n }, {\n name: 'border-dash-pattern',\n type: t.numbers\n }, {\n name: 'border-dash-offset',\n type: t.number\n }, {\n name: 'border-position',\n type: t.linePosition\n }];\n var nodeOutline = [{\n name: 'outline-color',\n type: t.color\n }, {\n name: 'outline-opacity',\n type: t.zeroOneNumber\n }, {\n name: 'outline-width',\n type: t.size,\n triggersBounds: diff.any\n }, {\n name: 'outline-style',\n type: t.borderStyle\n }, {\n name: 'outline-offset',\n type: t.size,\n triggersBounds: diff.any\n }];\n var backgroundImage = [{\n name: 'background-image',\n type: t.urls\n }, {\n name: 'background-image-crossorigin',\n type: t.bgCrossOrigin\n }, {\n name: 'background-image-opacity',\n type: t.zeroOneNumbers\n }, {\n name: 'background-image-containment',\n type: t.bgContainment\n }, {\n name: 'background-image-smoothing',\n type: t.bools\n }, {\n name: 'background-position-x',\n type: t.bgPos\n }, {\n name: 'background-position-y',\n type: t.bgPos\n }, {\n name: 'background-width-relative-to',\n type: t.bgRelativeTo\n }, {\n name: 'background-height-relative-to',\n type: t.bgRelativeTo\n }, {\n name: 'background-repeat',\n type: t.bgRepeat\n }, {\n name: 'background-fit',\n type: t.bgFit\n }, {\n name: 'background-clip',\n type: t.bgClip\n }, {\n name: 'background-width',\n type: t.bgWH\n }, {\n name: 'background-height',\n type: t.bgWH\n }, {\n name: 'background-offset-x',\n type: t.bgPos\n }, {\n name: 'background-offset-y',\n type: t.bgPos\n }];\n var compound = [{\n name: 'position',\n type: t.position,\n triggersBounds: diff.any\n }, {\n name: 'compound-sizing-wrt-labels',\n type: t.compoundIncludeLabels,\n triggersBounds: diff.any\n }, {\n name: 'min-width',\n type: t.size,\n triggersBounds: diff.any\n }, {\n name: 'min-width-bias-left',\n type: t.sizeMaybePercent,\n triggersBounds: diff.any\n }, {\n name: 'min-width-bias-right',\n type: t.sizeMaybePercent,\n triggersBounds: diff.any\n }, {\n name: 'min-height',\n type: t.size,\n triggersBounds: diff.any\n }, {\n name: 'min-height-bias-top',\n type: t.sizeMaybePercent,\n triggersBounds: diff.any\n }, {\n name: 'min-height-bias-bottom',\n type: t.sizeMaybePercent,\n triggersBounds: diff.any\n }];\n var edgeLine = [{\n name: 'line-style',\n type: t.lineStyle\n }, {\n name: 'line-color',\n type: t.color\n }, {\n name: 'line-fill',\n type: t.fill\n }, {\n name: 'line-cap',\n type: t.lineCap\n }, {\n name: 'line-opacity',\n type: t.zeroOneNumber\n }, {\n name: 'line-dash-pattern',\n type: t.numbers\n }, {\n name: 'line-dash-offset',\n type: t.number\n }, {\n name: 'line-outline-width',\n type: t.size\n }, {\n name: 'line-outline-color',\n type: t.color\n }, {\n name: 'line-gradient-stop-colors',\n type: t.colors\n }, {\n name: 'line-gradient-stop-positions',\n type: t.percentages\n }, {\n name: 'curve-style',\n type: t.curveStyle,\n triggersBounds: diff.any,\n triggersBoundsOfParallelEdges: function triggersBoundsOfParallelEdges(fromValue, toValue) {\n if (fromValue === toValue) {\n return false;\n } // must have diff\n\n return fromValue === 'bezier' ||\n // remove from bundle\n toValue === 'bezier'; // add to bundle\n }\n }, {\n name: 'haystack-radius',\n type: t.zeroOneNumber,\n triggersBounds: diff.any\n }, {\n name: 'source-endpoint',\n type: t.edgeEndpoint,\n triggersBounds: diff.any\n }, {\n name: 'target-endpoint',\n type: t.edgeEndpoint,\n triggersBounds: diff.any\n }, {\n name: 'control-point-step-size',\n type: t.size,\n triggersBounds: diff.any\n }, {\n name: 'control-point-distances',\n type: t.bidirectionalSizes,\n triggersBounds: diff.any\n }, {\n name: 'control-point-weights',\n type: t.numbers,\n triggersBounds: diff.any\n }, {\n name: 'segment-distances',\n type: t.bidirectionalSizes,\n triggersBounds: diff.any\n }, {\n name: 'segment-weights',\n type: t.numbers,\n triggersBounds: diff.any\n }, {\n name: 'segment-radii',\n type: t.numbers,\n triggersBounds: diff.any\n }, {\n name: 'radius-type',\n type: t.radiusType,\n triggersBounds: diff.any\n }, {\n name: 'taxi-turn',\n type: t.bidirectionalSizeMaybePercent,\n triggersBounds: diff.any\n }, {\n name: 'taxi-turn-min-distance',\n type: t.size,\n triggersBounds: diff.any\n }, {\n name: 'taxi-direction',\n type: t.axisDirection,\n triggersBounds: diff.any\n }, {\n name: 'taxi-radius',\n type: t.number,\n triggersBounds: diff.any\n }, {\n name: 'edge-distances',\n type: t.edgeDistances,\n triggersBounds: diff.any\n }, {\n name: 'arrow-scale',\n type: t.positiveNumber,\n triggersBounds: diff.any\n }, {\n name: 'loop-direction',\n type: t.angle,\n triggersBounds: diff.any\n }, {\n name: 'loop-sweep',\n type: t.angle,\n triggersBounds: diff.any\n }, {\n name: 'source-distance-from-node',\n type: t.size,\n triggersBounds: diff.any\n }, {\n name: 'target-distance-from-node',\n type: t.size,\n triggersBounds: diff.any\n }];\n var ghost = [{\n name: 'ghost',\n type: t.bool,\n triggersBounds: diff.any\n }, {\n name: 'ghost-offset-x',\n type: t.bidirectionalSize,\n triggersBounds: diff.any\n }, {\n name: 'ghost-offset-y',\n type: t.bidirectionalSize,\n triggersBounds: diff.any\n }, {\n name: 'ghost-opacity',\n type: t.zeroOneNumber\n }];\n var core = [{\n name: 'selection-box-color',\n type: t.color\n }, {\n name: 'selection-box-opacity',\n type: t.zeroOneNumber\n }, {\n name: 'selection-box-border-color',\n type: t.color\n }, {\n name: 'selection-box-border-width',\n type: t.size\n }, {\n name: 'active-bg-color',\n type: t.color\n }, {\n name: 'active-bg-opacity',\n type: t.zeroOneNumber\n }, {\n name: 'active-bg-size',\n type: t.size\n }, {\n name: 'outside-texture-bg-color',\n type: t.color\n }, {\n name: 'outside-texture-bg-opacity',\n type: t.zeroOneNumber\n }];\n\n // pie backgrounds for nodes\n var pie = [];\n styfn$2.pieBackgroundN = 16; // because the pie properties are numbered, give access to a constant N (for renderer use)\n pie.push({\n name: 'pie-size',\n type: t.sizeMaybePercent\n });\n pie.push({\n name: 'pie-hole',\n type: t.sizeMaybePercent\n });\n pie.push({\n name: 'pie-start-angle',\n type: t.angle\n });\n for (var i = 1; i <= styfn$2.pieBackgroundN; i++) {\n pie.push({\n name: 'pie-' + i + '-background-color',\n type: t.color\n });\n pie.push({\n name: 'pie-' + i + '-background-size',\n type: t.percent\n });\n pie.push({\n name: 'pie-' + i + '-background-opacity',\n type: t.zeroOneNumber\n });\n }\n\n // stripe backgrounds for nodes\n var stripe = [];\n styfn$2.stripeBackgroundN = 16; // because the stripe properties are numbered, give access to a constant N (for renderer use)\n stripe.push({\n name: 'stripe-size',\n type: t.sizeMaybePercent\n });\n stripe.push({\n name: 'stripe-direction',\n type: t.axisDirectionPrimary\n });\n for (var _i = 1; _i <= styfn$2.stripeBackgroundN; _i++) {\n stripe.push({\n name: 'stripe-' + _i + '-background-color',\n type: t.color\n });\n stripe.push({\n name: 'stripe-' + _i + '-background-size',\n type: t.percent\n });\n stripe.push({\n name: 'stripe-' + _i + '-background-opacity',\n type: t.zeroOneNumber\n });\n }\n\n // edge arrows\n var edgeArrow = [];\n var arrowPrefixes = styfn$2.arrowPrefixes = ['source', 'mid-source', 'target', 'mid-target'];\n [{\n name: 'arrow-shape',\n type: t.arrowShape,\n triggersBounds: diff.any\n }, {\n name: 'arrow-color',\n type: t.color\n }, {\n name: 'arrow-fill',\n type: t.arrowFill\n }, {\n name: 'arrow-width',\n type: t.arrowWidth\n }].forEach(function (prop) {\n arrowPrefixes.forEach(function (prefix) {\n var name = prefix + '-' + prop.name;\n var type = prop.type,\n triggersBounds = prop.triggersBounds;\n edgeArrow.push({\n name: name,\n type: type,\n triggersBounds: triggersBounds\n });\n });\n }, {});\n var props = styfn$2.properties = [].concat(behavior, transition, visibility, overlay, underlay, ghost, commonLabel, labelDimensions, mainLabel, sourceLabel, targetLabel, nodeBody, nodeBorder, nodeOutline, backgroundImage, pie, stripe, compound, edgeLine, edgeArrow, core);\n var propGroups = styfn$2.propertyGroups = {\n // common to all eles\n behavior: behavior,\n transition: transition,\n visibility: visibility,\n overlay: overlay,\n underlay: underlay,\n ghost: ghost,\n // labels\n commonLabel: commonLabel,\n labelDimensions: labelDimensions,\n mainLabel: mainLabel,\n sourceLabel: sourceLabel,\n targetLabel: targetLabel,\n // node props\n nodeBody: nodeBody,\n nodeBorder: nodeBorder,\n nodeOutline: nodeOutline,\n backgroundImage: backgroundImage,\n pie: pie,\n stripe: stripe,\n compound: compound,\n // edge props\n edgeLine: edgeLine,\n edgeArrow: edgeArrow,\n core: core\n };\n var propGroupNames = styfn$2.propertyGroupNames = {};\n var propGroupKeys = styfn$2.propertyGroupKeys = Object.keys(propGroups);\n propGroupKeys.forEach(function (key) {\n propGroupNames[key] = propGroups[key].map(function (prop) {\n return prop.name;\n });\n propGroups[key].forEach(function (prop) {\n return prop.groupKey = key;\n });\n });\n\n // define aliases\n var aliases = styfn$2.aliases = [{\n name: 'content',\n pointsTo: 'label'\n }, {\n name: 'control-point-distance',\n pointsTo: 'control-point-distances'\n }, {\n name: 'control-point-weight',\n pointsTo: 'control-point-weights'\n }, {\n name: 'segment-distance',\n pointsTo: 'segment-distances'\n }, {\n name: 'segment-weight',\n pointsTo: 'segment-weights'\n }, {\n name: 'segment-radius',\n pointsTo: 'segment-radii'\n }, {\n name: 'edge-text-rotation',\n pointsTo: 'text-rotation'\n }, {\n name: 'padding-left',\n pointsTo: 'padding'\n }, {\n name: 'padding-right',\n pointsTo: 'padding'\n }, {\n name: 'padding-top',\n pointsTo: 'padding'\n }, {\n name: 'padding-bottom',\n pointsTo: 'padding'\n }];\n\n // list of property names\n styfn$2.propertyNames = props.map(function (p) {\n return p.name;\n });\n\n // allow access of properties by name ( e.g. style.properties.height )\n for (var _i2 = 0; _i2 < props.length; _i2++) {\n var prop = props[_i2];\n props[prop.name] = prop; // allow lookup by name\n }\n\n // map aliases\n for (var _i3 = 0; _i3 < aliases.length; _i3++) {\n var alias = aliases[_i3];\n var pointsToProp = props[alias.pointsTo];\n var aliasProp = {\n name: alias.name,\n alias: true,\n pointsTo: pointsToProp\n };\n\n // add alias prop for parsing\n props.push(aliasProp);\n props[alias.name] = aliasProp; // allow lookup by name\n }\n})();\nstyfn$2.getDefaultProperty = function (name) {\n return this.getDefaultProperties()[name];\n};\nstyfn$2.getDefaultProperties = function () {\n var _p = this._private;\n if (_p.defaultProperties != null) {\n return _p.defaultProperties;\n }\n var rawProps = extend({\n // core props\n 'selection-box-color': '#ddd',\n 'selection-box-opacity': 0.65,\n 'selection-box-border-color': '#aaa',\n 'selection-box-border-width': 1,\n 'active-bg-color': 'black',\n 'active-bg-opacity': 0.15,\n 'active-bg-size': 30,\n 'outside-texture-bg-color': '#000',\n 'outside-texture-bg-opacity': 0.125,\n // common node/edge props\n 'events': 'yes',\n 'text-events': 'no',\n 'text-valign': 'top',\n 'text-halign': 'center',\n 'text-justification': 'auto',\n 'line-height': 1,\n 'color': '#000',\n 'box-selection': 'contain',\n 'text-outline-color': '#000',\n 'text-outline-width': 0,\n 'text-outline-opacity': 1,\n 'text-opacity': 1,\n 'text-decoration': 'none',\n 'text-transform': 'none',\n 'text-wrap': 'none',\n 'text-overflow-wrap': 'whitespace',\n 'text-max-width': 9999,\n 'text-background-color': '#000',\n 'text-background-opacity': 0,\n 'text-background-shape': 'rectangle',\n 'text-background-padding': 0,\n 'text-border-opacity': 0,\n 'text-border-width': 0,\n 'text-border-style': 'solid',\n 'text-border-color': '#000',\n 'font-family': 'Helvetica Neue, Helvetica, sans-serif',\n 'font-style': 'normal',\n 'font-weight': 'normal',\n 'font-size': 16,\n 'min-zoomed-font-size': 0,\n 'text-rotation': 'none',\n 'source-text-rotation': 'none',\n 'target-text-rotation': 'none',\n 'visibility': 'visible',\n 'display': 'element',\n 'opacity': 1,\n 'z-compound-depth': 'auto',\n 'z-index-compare': 'auto',\n 'z-index': 0,\n 'label': '',\n 'text-margin-x': 0,\n 'text-margin-y': 0,\n 'source-label': '',\n 'source-text-offset': 0,\n 'source-text-margin-x': 0,\n 'source-text-margin-y': 0,\n 'target-label': '',\n 'target-text-offset': 0,\n 'target-text-margin-x': 0,\n 'target-text-margin-y': 0,\n 'overlay-opacity': 0,\n 'overlay-color': '#000',\n 'overlay-padding': 10,\n 'overlay-shape': 'round-rectangle',\n 'overlay-corner-radius': 'auto',\n 'underlay-opacity': 0,\n 'underlay-color': '#000',\n 'underlay-padding': 10,\n 'underlay-shape': 'round-rectangle',\n 'underlay-corner-radius': 'auto',\n 'transition-property': 'none',\n 'transition-duration': 0,\n 'transition-delay': 0,\n 'transition-timing-function': 'linear',\n 'box-select-labels': 'no',\n // node props\n 'background-blacken': 0,\n 'background-color': '#999',\n 'background-fill': 'solid',\n 'background-opacity': 1,\n 'background-image': 'none',\n 'background-image-crossorigin': 'anonymous',\n 'background-image-opacity': 1,\n 'background-image-containment': 'inside',\n 'background-image-smoothing': 'yes',\n 'background-position-x': '50%',\n 'background-position-y': '50%',\n 'background-offset-x': 0,\n 'background-offset-y': 0,\n 'background-width-relative-to': 'include-padding',\n 'background-height-relative-to': 'include-padding',\n 'background-repeat': 'no-repeat',\n 'background-fit': 'none',\n 'background-clip': 'node',\n 'background-width': 'auto',\n 'background-height': 'auto',\n 'border-color': '#000',\n 'border-opacity': 1,\n 'border-width': 0,\n 'border-style': 'solid',\n 'border-dash-pattern': [4, 2],\n 'border-dash-offset': 0,\n 'border-cap': 'butt',\n 'border-join': 'miter',\n 'border-position': 'center',\n 'outline-color': '#999',\n 'outline-opacity': 1,\n 'outline-width': 0,\n 'outline-offset': 0,\n 'outline-style': 'solid',\n 'height': 30,\n 'width': 30,\n 'shape': 'ellipse',\n 'shape-polygon-points': '-1, -1, 1, -1, 1, 1, -1, 1',\n 'corner-radius': 'auto',\n 'bounds-expansion': 0,\n // node gradient\n 'background-gradient-direction': 'to-bottom',\n 'background-gradient-stop-colors': '#999',\n 'background-gradient-stop-positions': '0%',\n // ghost props\n 'ghost': 'no',\n 'ghost-offset-y': 0,\n 'ghost-offset-x': 0,\n 'ghost-opacity': 0,\n // compound props\n 'padding': 0,\n 'padding-relative-to': 'width',\n 'position': 'origin',\n 'compound-sizing-wrt-labels': 'include',\n 'min-width': 0,\n 'min-width-bias-left': 0,\n 'min-width-bias-right': 0,\n 'min-height': 0,\n 'min-height-bias-top': 0,\n 'min-height-bias-bottom': 0\n }, {\n // node pie bg\n 'pie-size': '100%',\n 'pie-hole': 0,\n 'pie-start-angle': '0deg'\n }, [{\n name: 'pie-{{i}}-background-color',\n value: 'black'\n }, {\n name: 'pie-{{i}}-background-size',\n value: '0%'\n }, {\n name: 'pie-{{i}}-background-opacity',\n value: 1\n }].reduce(function (css, prop) {\n for (var i = 1; i <= styfn$2.pieBackgroundN; i++) {\n var name = prop.name.replace('{{i}}', i);\n var val = prop.value;\n css[name] = val;\n }\n return css;\n }, {}), {\n // node stripes bg\n 'stripe-size': '100%',\n 'stripe-direction': 'horizontal'\n }, [{\n name: 'stripe-{{i}}-background-color',\n value: 'black'\n }, {\n name: 'stripe-{{i}}-background-size',\n value: '0%'\n }, {\n name: 'stripe-{{i}}-background-opacity',\n value: 1\n }].reduce(function (css, prop) {\n for (var i = 1; i <= styfn$2.stripeBackgroundN; i++) {\n var name = prop.name.replace('{{i}}', i);\n var val = prop.value;\n css[name] = val;\n }\n return css;\n }, {}), {\n // edge props\n 'line-style': 'solid',\n 'line-color': '#999',\n 'line-fill': 'solid',\n 'line-cap': 'butt',\n 'line-opacity': 1,\n 'line-outline-width': 0,\n 'line-outline-color': '#000',\n 'line-gradient-stop-colors': '#999',\n 'line-gradient-stop-positions': '0%',\n 'control-point-step-size': 40,\n 'control-point-weights': 0.5,\n 'segment-weights': 0.5,\n 'segment-distances': 20,\n 'segment-radii': 15,\n 'radius-type': 'arc-radius',\n 'taxi-turn': '50%',\n 'taxi-radius': 15,\n 'taxi-turn-min-distance': 10,\n 'taxi-direction': 'auto',\n 'edge-distances': 'intersection',\n 'curve-style': 'haystack',\n 'haystack-radius': 0,\n 'arrow-scale': 1,\n 'loop-direction': '-45deg',\n 'loop-sweep': '-90deg',\n 'source-distance-from-node': 0,\n 'target-distance-from-node': 0,\n 'source-endpoint': 'outside-to-node',\n 'target-endpoint': 'outside-to-node',\n 'line-dash-pattern': [6, 3],\n 'line-dash-offset': 0\n }, [{\n name: 'arrow-shape',\n value: 'none'\n }, {\n name: 'arrow-color',\n value: '#999'\n }, {\n name: 'arrow-fill',\n value: 'filled'\n }, {\n name: 'arrow-width',\n value: 1\n }].reduce(function (css, prop) {\n styfn$2.arrowPrefixes.forEach(function (prefix) {\n var name = prefix + '-' + prop.name;\n var val = prop.value;\n css[name] = val;\n });\n return css;\n }, {}));\n var parsedProps = {};\n for (var i = 0; i < this.properties.length; i++) {\n var prop = this.properties[i];\n if (prop.pointsTo) {\n continue;\n }\n var name = prop.name;\n var val = rawProps[name];\n var parsedProp = this.parse(name, val);\n parsedProps[name] = parsedProp;\n }\n _p.defaultProperties = parsedProps;\n return _p.defaultProperties;\n};\nstyfn$2.addDefaultStylesheet = function () {\n this.selector(':parent').css({\n 'shape': 'rectangle',\n 'padding': 10,\n 'background-color': '#eee',\n 'border-color': '#ccc',\n 'border-width': 1\n }).selector('edge').css({\n 'width': 3\n }).selector(':loop').css({\n 'curve-style': 'bezier'\n }).selector('edge:compound').css({\n 'curve-style': 'bezier',\n 'source-endpoint': 'outside-to-line',\n 'target-endpoint': 'outside-to-line'\n }).selector(':selected').css({\n 'background-color': '#0169D9',\n 'line-color': '#0169D9',\n 'source-arrow-color': '#0169D9',\n 'target-arrow-color': '#0169D9',\n 'mid-source-arrow-color': '#0169D9',\n 'mid-target-arrow-color': '#0169D9'\n }).selector(':parent:selected').css({\n 'background-color': '#CCE1F9',\n 'border-color': '#aec8e5'\n }).selector(':active').css({\n 'overlay-color': 'black',\n 'overlay-padding': 10,\n 'overlay-opacity': 0.25\n });\n this.defaultLength = this.length;\n};\n\nvar styfn$1 = {};\n\n// a caching layer for property parsing\nstyfn$1.parse = function (name, value, propIsBypass, propIsFlat) {\n var self = this;\n\n // function values can't be cached in all cases, and there isn't much benefit of caching them anyway\n if (fn$6(value)) {\n return self.parseImplWarn(name, value, propIsBypass, propIsFlat);\n }\n var flatKey = propIsFlat === 'mapping' || propIsFlat === true || propIsFlat === false || propIsFlat == null ? 'dontcare' : propIsFlat;\n var bypassKey = propIsBypass ? 't' : 'f';\n var valueKey = '' + value;\n var argHash = hashStrings(name, valueKey, bypassKey, flatKey);\n var propCache = self.propCache = self.propCache || [];\n var ret;\n if (!(ret = propCache[argHash])) {\n ret = propCache[argHash] = self.parseImplWarn(name, value, propIsBypass, propIsFlat);\n }\n\n // - bypasses can't be shared b/c the value can be changed by animations or otherwise overridden\n // - mappings can't be shared b/c mappings are per-element\n if (propIsBypass || propIsFlat === 'mapping') {\n // need a copy since props are mutated later in their lifecycles\n ret = copy(ret);\n if (ret) {\n ret.value = copy(ret.value); // because it could be an array, e.g. colour\n }\n }\n return ret;\n};\nstyfn$1.parseImplWarn = function (name, value, propIsBypass, propIsFlat) {\n var prop = this.parseImpl(name, value, propIsBypass, propIsFlat);\n if (!prop && value != null) {\n warn(\"The style property `\".concat(name, \": \").concat(value, \"` is invalid\"));\n }\n if (prop && (prop.name === 'width' || prop.name === 'height') && value === 'label') {\n warn('The style value of `label` is deprecated for `' + prop.name + '`');\n }\n return prop;\n};\n\n// parse a property; return null on invalid; return parsed property otherwise\n// fields :\n// - name : the name of the property\n// - value : the parsed, native-typed value of the property\n// - strValue : a string value that represents the property value in valid css\n// - bypass : true iff the property is a bypass property\nstyfn$1.parseImpl = function (name, value, propIsBypass, propIsFlat) {\n var self = this;\n name = camel2dash(name); // make sure the property name is in dash form (e.g. 'property-name' not 'propertyName')\n\n var property = self.properties[name];\n var passedValue = value;\n var types = self.types;\n if (!property) {\n return null;\n } // return null on property of unknown name\n if (value === undefined) {\n return null;\n } // can't assign undefined\n\n // the property may be an alias\n if (property.alias) {\n property = property.pointsTo;\n name = property.name;\n }\n var valueIsString = string(value);\n if (valueIsString) {\n // trim the value to make parsing easier\n value = value.trim();\n }\n var type = property.type;\n if (!type) {\n return null;\n } // no type, no luck\n\n // check if bypass is null or empty string (i.e. indication to delete bypass property)\n if (propIsBypass && (value === '' || value === null)) {\n return {\n name: name,\n value: value,\n bypass: true,\n deleteBypass: true\n };\n }\n\n // check if value is a function used as a mapper\n if (fn$6(value)) {\n return {\n name: name,\n value: value,\n strValue: 'fn',\n mapped: types.fn,\n bypass: propIsBypass\n };\n }\n\n // check if value is mapped\n var data, mapData;\n if (!valueIsString || propIsFlat || value.length < 7 || value[1] !== 'a') ; else if (value.length >= 7 && value[0] === 'd' && (data = new RegExp(types.data.regex).exec(value))) {\n if (propIsBypass) {\n return false;\n } // mappers not allowed in bypass\n\n var mapped = types.data;\n return {\n name: name,\n value: data,\n strValue: '' + value,\n mapped: mapped,\n field: data[1],\n bypass: propIsBypass\n };\n } else if (value.length >= 10 && value[0] === 'm' && (mapData = new RegExp(types.mapData.regex).exec(value))) {\n if (propIsBypass) {\n return false;\n } // mappers not allowed in bypass\n if (type.multiple) {\n return false;\n } // impossible to map to num\n\n var _mapped = types.mapData;\n\n // we can map only if the type is a colour or a number\n if (!(type.color || type.number)) {\n return false;\n }\n var valueMin = this.parse(name, mapData[4]); // parse to validate\n if (!valueMin || valueMin.mapped) {\n return false;\n } // can't be invalid or mapped\n\n var valueMax = this.parse(name, mapData[5]); // parse to validate\n if (!valueMax || valueMax.mapped) {\n return false;\n } // can't be invalid or mapped\n\n // check if valueMin and valueMax are the same\n if (valueMin.pfValue === valueMax.pfValue || valueMin.strValue === valueMax.strValue) {\n warn('`' + name + ': ' + value + '` is not a valid mapper because the output range is zero; converting to `' + name + ': ' + valueMin.strValue + '`');\n return this.parse(name, valueMin.strValue); // can't make much of a mapper without a range\n } else if (type.color) {\n var c1 = valueMin.value;\n var c2 = valueMax.value;\n var same = c1[0] === c2[0] // red\n && c1[1] === c2[1] // green\n && c1[2] === c2[2] // blue\n && (\n // optional alpha\n c1[3] === c2[3] // same alpha outright\n || (c1[3] == null || c1[3] === 1 // full opacity for colour 1?\n ) && (c2[3] == null || c2[3] === 1) // full opacity for colour 2?\n );\n if (same) {\n return false;\n } // can't make a mapper without a range\n }\n return {\n name: name,\n value: mapData,\n strValue: '' + value,\n mapped: _mapped,\n field: mapData[1],\n fieldMin: parseFloat(mapData[2]),\n // min & max are numeric\n fieldMax: parseFloat(mapData[3]),\n valueMin: valueMin.value,\n valueMax: valueMax.value,\n bypass: propIsBypass\n };\n }\n if (type.multiple && propIsFlat !== 'multiple') {\n var vals;\n if (valueIsString) {\n vals = value.split(/\\s+/);\n } else if (array(value)) {\n vals = value;\n } else {\n vals = [value];\n }\n if (type.evenMultiple && vals.length % 2 !== 0) {\n return null;\n }\n var valArr = [];\n var unitsArr = [];\n var pfValArr = [];\n var strVal = '';\n var hasEnum = false;\n for (var i = 0; i < vals.length; i++) {\n var p = self.parse(name, vals[i], propIsBypass, 'multiple');\n hasEnum = hasEnum || string(p.value);\n valArr.push(p.value);\n pfValArr.push(p.pfValue != null ? p.pfValue : p.value);\n unitsArr.push(p.units);\n strVal += (i > 0 ? ' ' : '') + p.strValue;\n }\n if (type.validate && !type.validate(valArr, unitsArr)) {\n return null;\n }\n if (type.singleEnum && hasEnum) {\n if (valArr.length === 1 && string(valArr[0])) {\n return {\n name: name,\n value: valArr[0],\n strValue: valArr[0],\n bypass: propIsBypass\n };\n } else {\n return null;\n }\n }\n return {\n name: name,\n value: valArr,\n pfValue: pfValArr,\n strValue: strVal,\n bypass: propIsBypass,\n units: unitsArr\n };\n }\n\n // several types also allow enums\n var checkEnums = function checkEnums() {\n for (var _i = 0; _i < type.enums.length; _i++) {\n var en = type.enums[_i];\n if (en === value) {\n return {\n name: name,\n value: value,\n strValue: '' + value,\n bypass: propIsBypass\n };\n }\n }\n return null;\n };\n\n // check the type and return the appropriate object\n if (type.number) {\n var units;\n var implicitUnits = 'px'; // not set => px\n\n if (type.units) {\n // use specified units if set\n units = type.units;\n }\n if (type.implicitUnits) {\n implicitUnits = type.implicitUnits;\n }\n if (!type.unitless) {\n if (valueIsString) {\n var unitsRegex = 'px|em' + (type.allowPercent ? '|\\\\%' : '');\n if (units) {\n unitsRegex = units;\n } // only allow explicit units if so set\n var match = value.match('^(' + number + ')(' + unitsRegex + ')?' + '$');\n if (match) {\n value = match[1];\n units = match[2] || implicitUnits;\n }\n } else if (!units || type.implicitUnits) {\n units = implicitUnits; // implicitly px if unspecified\n }\n }\n value = parseFloat(value);\n\n // if not a number and enums not allowed, then the value is invalid\n if (isNaN(value) && type.enums === undefined) {\n return null;\n }\n\n // check if this number type also accepts special keywords in place of numbers\n // (i.e. `left`, `auto`, etc)\n if (isNaN(value) && type.enums !== undefined) {\n value = passedValue;\n return checkEnums();\n }\n\n // check if value must be an integer\n if (type.integer && !integer(value)) {\n return null;\n }\n\n // check value is within range\n if (type.min !== undefined && (value < type.min || type.strictMin && value === type.min) || type.max !== undefined && (value > type.max || type.strictMax && value === type.max)) {\n return null;\n }\n var ret = {\n name: name,\n value: value,\n strValue: '' + value + (units ? units : ''),\n units: units,\n bypass: propIsBypass\n };\n\n // normalise value in pixels\n if (type.unitless || units !== 'px' && units !== 'em') {\n ret.pfValue = value;\n } else {\n ret.pfValue = units === 'px' || !units ? value : this.getEmSizeInPixels() * value;\n }\n\n // normalise value in ms\n if (units === 'ms' || units === 's') {\n ret.pfValue = units === 'ms' ? value : 1000 * value;\n }\n\n // normalise value in rad\n if (units === 'deg' || units === 'rad') {\n ret.pfValue = units === 'rad' ? value : deg2rad(value);\n }\n\n // normalize value in %\n if (units === '%') {\n ret.pfValue = value / 100;\n }\n return ret;\n } else if (type.propList) {\n var props = [];\n var propsStr = '' + value;\n if (propsStr === 'none') ; else {\n // go over each prop\n\n var propsSplit = propsStr.split(/\\s*,\\s*|\\s+/);\n for (var _i2 = 0; _i2 < propsSplit.length; _i2++) {\n var propName = propsSplit[_i2].trim();\n if (self.properties[propName]) {\n props.push(propName);\n } else {\n warn('`' + propName + '` is not a valid property name');\n }\n }\n if (props.length === 0) {\n return null;\n }\n }\n return {\n name: name,\n value: props,\n strValue: props.length === 0 ? 'none' : props.join(' '),\n bypass: propIsBypass\n };\n } else if (type.color) {\n var tuple = color2tuple(value);\n if (!tuple) {\n return null;\n }\n return {\n name: name,\n value: tuple,\n pfValue: tuple,\n strValue: 'rgb(' + tuple[0] + ',' + tuple[1] + ',' + tuple[2] + ')',\n // n.b. no spaces b/c of multiple support\n bypass: propIsBypass\n };\n } else if (type.regex || type.regexes) {\n // first check enums\n if (type.enums) {\n var enumProp = checkEnums();\n if (enumProp) {\n return enumProp;\n }\n }\n var regexes = type.regexes ? type.regexes : [type.regex];\n for (var _i3 = 0; _i3 < regexes.length; _i3++) {\n var regex = new RegExp(regexes[_i3]); // make a regex from the type string\n var m = regex.exec(value);\n if (m) {\n // regex matches\n return {\n name: name,\n value: type.singleRegexMatchValue ? m[1] : m,\n strValue: '' + value,\n bypass: propIsBypass\n };\n }\n }\n return null; // didn't match any\n } else if (type.string) {\n // just return\n return {\n name: name,\n value: '' + value,\n strValue: '' + value,\n bypass: propIsBypass\n };\n } else if (type.enums) {\n // check enums last because it's a combo type in others\n return checkEnums();\n } else {\n return null; // not a type we can handle\n }\n};\n\nvar _Style = function Style(cy) {\n if (!(this instanceof _Style)) {\n return new _Style(cy);\n }\n if (!core(cy)) {\n error('A style must have a core reference');\n return;\n }\n this._private = {\n cy: cy,\n coreStyle: {}\n };\n this.length = 0;\n this.resetToDefault();\n};\nvar styfn = _Style.prototype;\nstyfn.instanceString = function () {\n return 'style';\n};\n\n// remove all contexts\nstyfn.clear = function () {\n var _p = this._private;\n var cy = _p.cy;\n var eles = cy.elements();\n for (var i = 0; i < this.length; i++) {\n this[i] = undefined;\n }\n this.length = 0;\n _p.contextStyles = {};\n _p.propDiffs = {};\n this.cleanElements(eles, true);\n eles.forEach(function (ele) {\n var ele_p = ele[0]._private;\n ele_p.styleDirty = true;\n ele_p.appliedInitStyle = false;\n });\n return this; // chaining\n};\nstyfn.resetToDefault = function () {\n this.clear();\n this.addDefaultStylesheet();\n return this;\n};\n\n// builds a style object for the 'core' selector\nstyfn.core = function (propName) {\n return this._private.coreStyle[propName] || this.getDefaultProperty(propName);\n};\n\n// create a new context from the specified selector string and switch to that context\nstyfn.selector = function (selectorStr) {\n // 'core' is a special case and does not need a selector\n var selector = selectorStr === 'core' ? null : new Selector(selectorStr);\n var i = this.length++; // new context means new index\n this[i] = {\n selector: selector,\n properties: [],\n mappedProperties: [],\n index: i\n };\n return this; // chaining\n};\n\n// add one or many css rules to the current context\nstyfn.css = function () {\n var self = this;\n var args = arguments;\n if (args.length === 1) {\n var map = args[0];\n for (var i = 0; i < self.properties.length; i++) {\n var prop = self.properties[i];\n var mapVal = map[prop.name];\n if (mapVal === undefined) {\n mapVal = map[dash2camel(prop.name)];\n }\n if (mapVal !== undefined) {\n this.cssRule(prop.name, mapVal);\n }\n }\n } else if (args.length === 2) {\n this.cssRule(args[0], args[1]);\n }\n\n // do nothing if args are invalid\n\n return this; // chaining\n};\nstyfn.style = styfn.css;\n\n// add a single css rule to the current context\nstyfn.cssRule = function (name, value) {\n // name-value pair\n var property = this.parse(name, value);\n\n // add property to current context if valid\n if (property) {\n var i = this.length - 1;\n this[i].properties.push(property);\n this[i].properties[property.name] = property; // allow access by name as well\n\n if (property.name.match(/pie-(\\d+)-background-size/) && property.value) {\n this._private.hasPie = true;\n }\n if (property.name.match(/stripe-(\\d+)-background-size/) && property.value) {\n this._private.hasStripe = true;\n }\n if (property.mapped) {\n this[i].mappedProperties.push(property);\n }\n\n // add to core style if necessary\n var currentSelectorIsCore = !this[i].selector;\n if (currentSelectorIsCore) {\n this._private.coreStyle[property.name] = property;\n }\n }\n return this; // chaining\n};\nstyfn.append = function (style) {\n if (stylesheet(style)) {\n style.appendToStyle(this);\n } else if (array(style)) {\n this.appendFromJson(style);\n } else if (string(style)) {\n this.appendFromString(style);\n } // you probably wouldn't want to append a Style, since you'd duplicate the default parts\n\n return this;\n};\n\n// static function\n_Style.fromJson = function (cy, json) {\n var style = new _Style(cy);\n style.fromJson(json);\n return style;\n};\n_Style.fromString = function (cy, string) {\n return new _Style(cy).fromString(string);\n};\n[styfn$8, styfn$7, styfn$6, styfn$5, styfn$4, styfn$3, styfn$2, styfn$1].forEach(function (props) {\n extend(styfn, props);\n});\n_Style.types = styfn.types;\n_Style.properties = styfn.properties;\n_Style.propertyGroups = styfn.propertyGroups;\n_Style.propertyGroupNames = styfn.propertyGroupNames;\n_Style.propertyGroupKeys = styfn.propertyGroupKeys;\n\nvar corefn$2 = {\n style: function style(newStyle) {\n if (newStyle) {\n var s = this.setStyle(newStyle);\n s.update();\n }\n return this._private.style;\n },\n setStyle: function setStyle(style) {\n var _p = this._private;\n if (stylesheet(style)) {\n _p.style = style.generateStyle(this);\n } else if (array(style)) {\n _p.style = _Style.fromJson(this, style);\n } else if (string(style)) {\n _p.style = _Style.fromString(this, style);\n } else {\n _p.style = _Style(this);\n }\n return _p.style;\n },\n // e.g. cy.data() changed => recalc ele mappers\n updateStyle: function updateStyle() {\n this.mutableElements().updateStyle(); // just send to all eles\n }\n};\n\nvar defaultSelectionType = 'single';\nvar corefn$1 = {\n autolock: function autolock(bool) {\n if (bool !== undefined) {\n this._private.autolock = bool ? true : false;\n } else {\n return this._private.autolock;\n }\n return this; // chaining\n },\n autoungrabify: function autoungrabify(bool) {\n if (bool !== undefined) {\n this._private.autoungrabify = bool ? true : false;\n } else {\n return this._private.autoungrabify;\n }\n return this; // chaining\n },\n autounselectify: function autounselectify(bool) {\n if (bool !== undefined) {\n this._private.autounselectify = bool ? true : false;\n } else {\n return this._private.autounselectify;\n }\n return this; // chaining\n },\n selectionType: function selectionType(selType) {\n var _p = this._private;\n if (_p.selectionType == null) {\n _p.selectionType = defaultSelectionType;\n }\n if (selType !== undefined) {\n if (selType === 'additive' || selType === 'single') {\n _p.selectionType = selType;\n }\n } else {\n return _p.selectionType;\n }\n return this;\n },\n panningEnabled: function panningEnabled(bool) {\n if (bool !== undefined) {\n this._private.panningEnabled = bool ? true : false;\n } else {\n return this._private.panningEnabled;\n }\n return this; // chaining\n },\n userPanningEnabled: function userPanningEnabled(bool) {\n if (bool !== undefined) {\n this._private.userPanningEnabled = bool ? true : false;\n } else {\n return this._private.userPanningEnabled;\n }\n return this; // chaining\n },\n zoomingEnabled: function zoomingEnabled(bool) {\n if (bool !== undefined) {\n this._private.zoomingEnabled = bool ? true : false;\n } else {\n return this._private.zoomingEnabled;\n }\n return this; // chaining\n },\n userZoomingEnabled: function userZoomingEnabled(bool) {\n if (bool !== undefined) {\n this._private.userZoomingEnabled = bool ? true : false;\n } else {\n return this._private.userZoomingEnabled;\n }\n return this; // chaining\n },\n boxSelectionEnabled: function boxSelectionEnabled(bool) {\n if (bool !== undefined) {\n this._private.boxSelectionEnabled = bool ? true : false;\n } else {\n return this._private.boxSelectionEnabled;\n }\n return this; // chaining\n },\n pan: function pan() {\n var args = arguments;\n var pan = this._private.pan;\n var dim, val, dims, x, y;\n switch (args.length) {\n case 0:\n // .pan()\n return pan;\n case 1:\n if (string(args[0])) {\n // .pan('x')\n dim = args[0];\n return pan[dim];\n } else if (plainObject(args[0])) {\n // .pan({ x: 0, y: 100 })\n if (!this._private.panningEnabled) {\n return this;\n }\n dims = args[0];\n x = dims.x;\n y = dims.y;\n if (number$1(x)) {\n pan.x = x;\n }\n if (number$1(y)) {\n pan.y = y;\n }\n this.emit('pan viewport');\n }\n break;\n case 2:\n // .pan('x', 100)\n if (!this._private.panningEnabled) {\n return this;\n }\n dim = args[0];\n val = args[1];\n if ((dim === 'x' || dim === 'y') && number$1(val)) {\n pan[dim] = val;\n }\n this.emit('pan viewport');\n break;\n // invalid\n }\n this.notify('viewport');\n return this; // chaining\n },\n panBy: function panBy(arg0, arg1) {\n var args = arguments;\n var pan = this._private.pan;\n var dim, val, dims, x, y;\n if (!this._private.panningEnabled) {\n return this;\n }\n switch (args.length) {\n case 1:\n if (plainObject(arg0)) {\n // .panBy({ x: 0, y: 100 })\n dims = args[0];\n x = dims.x;\n y = dims.y;\n if (number$1(x)) {\n pan.x += x;\n }\n if (number$1(y)) {\n pan.y += y;\n }\n this.emit('pan viewport');\n }\n break;\n case 2:\n // .panBy('x', 100)\n dim = arg0;\n val = arg1;\n if ((dim === 'x' || dim === 'y') && number$1(val)) {\n pan[dim] += val;\n }\n this.emit('pan viewport');\n break;\n // invalid\n }\n this.notify('viewport');\n return this; // chaining\n },\n gc: function gc() {\n this.notify('gc');\n },\n fit: function fit(elements, padding) {\n var viewportState = this.getFitViewport(elements, padding);\n if (viewportState) {\n var _p = this._private;\n _p.zoom = viewportState.zoom;\n _p.pan = viewportState.pan;\n this.emit('pan zoom viewport');\n this.notify('viewport');\n }\n return this; // chaining\n },\n getFitViewport: function getFitViewport(elements, padding) {\n if (number$1(elements) && padding === undefined) {\n // elements is optional\n padding = elements;\n elements = undefined;\n }\n if (!this._private.panningEnabled || !this._private.zoomingEnabled) {\n return;\n }\n var bb;\n if (string(elements)) {\n var sel = elements;\n elements = this.$(sel);\n } else if (boundingBox(elements)) {\n // assume bb\n var bbe = elements;\n bb = {\n x1: bbe.x1,\n y1: bbe.y1,\n x2: bbe.x2,\n y2: bbe.y2\n };\n bb.w = bb.x2 - bb.x1;\n bb.h = bb.y2 - bb.y1;\n } else if (!elementOrCollection(elements)) {\n elements = this.mutableElements();\n }\n if (elementOrCollection(elements) && elements.empty()) {\n return;\n } // can't fit to nothing\n\n bb = bb || elements.boundingBox();\n var w = this.width();\n var h = this.height();\n var zoom;\n padding = number$1(padding) ? padding : 0;\n if (!isNaN(w) && !isNaN(h) && w > 0 && h > 0 && !isNaN(bb.w) && !isNaN(bb.h) && bb.w > 0 && bb.h > 0) {\n zoom = Math.min((w - 2 * padding) / bb.w, (h - 2 * padding) / bb.h);\n\n // crop zoom\n zoom = zoom > this._private.maxZoom ? this._private.maxZoom : zoom;\n zoom = zoom < this._private.minZoom ? this._private.minZoom : zoom;\n var pan = {\n // now pan to middle\n x: (w - zoom * (bb.x1 + bb.x2)) / 2,\n y: (h - zoom * (bb.y1 + bb.y2)) / 2\n };\n return {\n zoom: zoom,\n pan: pan\n };\n }\n return;\n },\n zoomRange: function zoomRange(min, max) {\n var _p = this._private;\n if (max == null) {\n var opts = min;\n min = opts.min;\n max = opts.max;\n }\n if (number$1(min) && number$1(max) && min <= max) {\n _p.minZoom = min;\n _p.maxZoom = max;\n } else if (number$1(min) && max === undefined && min <= _p.maxZoom) {\n _p.minZoom = min;\n } else if (number$1(max) && min === undefined && max >= _p.minZoom) {\n _p.maxZoom = max;\n }\n return this;\n },\n minZoom: function minZoom(zoom) {\n if (zoom === undefined) {\n return this._private.minZoom;\n } else {\n return this.zoomRange({\n min: zoom\n });\n }\n },\n maxZoom: function maxZoom(zoom) {\n if (zoom === undefined) {\n return this._private.maxZoom;\n } else {\n return this.zoomRange({\n max: zoom\n });\n }\n },\n getZoomedViewport: function getZoomedViewport(params) {\n var _p = this._private;\n var currentPan = _p.pan;\n var currentZoom = _p.zoom;\n var pos; // in rendered px\n var zoom;\n var bail = false;\n if (!_p.zoomingEnabled) {\n // zooming disabled\n bail = true;\n }\n if (number$1(params)) {\n // then set the zoom\n zoom = params;\n } else if (plainObject(params)) {\n // then zoom about a point\n zoom = params.level;\n if (params.position != null) {\n pos = modelToRenderedPosition$1(params.position, currentZoom, currentPan);\n } else if (params.renderedPosition != null) {\n pos = params.renderedPosition;\n }\n if (pos != null && !_p.panningEnabled) {\n // panning disabled\n bail = true;\n }\n }\n\n // crop zoom\n zoom = zoom > _p.maxZoom ? _p.maxZoom : zoom;\n zoom = zoom < _p.minZoom ? _p.minZoom : zoom;\n\n // can't zoom with invalid params\n if (bail || !number$1(zoom) || zoom === currentZoom || pos != null && (!number$1(pos.x) || !number$1(pos.y))) {\n return null;\n }\n if (pos != null) {\n // set zoom about position\n var pan1 = currentPan;\n var zoom1 = currentZoom;\n var zoom2 = zoom;\n var pan2 = {\n x: -zoom2 / zoom1 * (pos.x - pan1.x) + pos.x,\n y: -zoom2 / zoom1 * (pos.y - pan1.y) + pos.y\n };\n return {\n zoomed: true,\n panned: true,\n zoom: zoom2,\n pan: pan2\n };\n } else {\n // just set the zoom\n return {\n zoomed: true,\n panned: false,\n zoom: zoom,\n pan: currentPan\n };\n }\n },\n zoom: function zoom(params) {\n if (params === undefined) {\n // get\n return this._private.zoom;\n } else {\n // set\n var vp = this.getZoomedViewport(params);\n var _p = this._private;\n if (vp == null || !vp.zoomed) {\n return this;\n }\n _p.zoom = vp.zoom;\n if (vp.panned) {\n _p.pan.x = vp.pan.x;\n _p.pan.y = vp.pan.y;\n }\n this.emit('zoom' + (vp.panned ? ' pan' : '') + ' viewport');\n this.notify('viewport');\n return this; // chaining\n }\n },\n viewport: function viewport(opts) {\n var _p = this._private;\n var zoomDefd = true;\n var panDefd = true;\n var events = []; // to trigger\n var zoomFailed = false;\n var panFailed = false;\n if (!opts) {\n return this;\n }\n if (!number$1(opts.zoom)) {\n zoomDefd = false;\n }\n if (!plainObject(opts.pan)) {\n panDefd = false;\n }\n if (!zoomDefd && !panDefd) {\n return this;\n }\n if (zoomDefd) {\n var z = opts.zoom;\n if (z < _p.minZoom || z > _p.maxZoom || !_p.zoomingEnabled) {\n zoomFailed = true;\n } else {\n _p.zoom = z;\n events.push('zoom');\n }\n }\n if (panDefd && (!zoomFailed || !opts.cancelOnFailedZoom) && _p.panningEnabled) {\n var p = opts.pan;\n if (number$1(p.x)) {\n _p.pan.x = p.x;\n panFailed = false;\n }\n if (number$1(p.y)) {\n _p.pan.y = p.y;\n panFailed = false;\n }\n if (!panFailed) {\n events.push('pan');\n }\n }\n if (events.length > 0) {\n events.push('viewport');\n this.emit(events.join(' '));\n this.notify('viewport');\n }\n return this; // chaining\n },\n center: function center(elements) {\n var pan = this.getCenterPan(elements);\n if (pan) {\n this._private.pan = pan;\n this.emit('pan viewport');\n this.notify('viewport');\n }\n return this; // chaining\n },\n getCenterPan: function getCenterPan(elements, zoom) {\n if (!this._private.panningEnabled) {\n return;\n }\n if (string(elements)) {\n var selector = elements;\n elements = this.mutableElements().filter(selector);\n } else if (!elementOrCollection(elements)) {\n elements = this.mutableElements();\n }\n if (elements.length === 0) {\n return;\n } // can't centre pan to nothing\n\n var bb = elements.boundingBox();\n var w = this.width();\n var h = this.height();\n zoom = zoom === undefined ? this._private.zoom : zoom;\n var pan = {\n // middle\n x: (w - zoom * (bb.x1 + bb.x2)) / 2,\n y: (h - zoom * (bb.y1 + bb.y2)) / 2\n };\n return pan;\n },\n reset: function reset() {\n if (!this._private.panningEnabled || !this._private.zoomingEnabled) {\n return this;\n }\n this.viewport({\n pan: {\n x: 0,\n y: 0\n },\n zoom: 1\n });\n return this; // chaining\n },\n invalidateSize: function invalidateSize() {\n this._private.sizeCache = null;\n },\n size: function size() {\n var _p = this._private;\n var container = _p.container;\n var cy = this;\n return _p.sizeCache = _p.sizeCache || (container ? function () {\n var style = cy.window().getComputedStyle(container);\n var val = function val(name) {\n return parseFloat(style.getPropertyValue(name));\n };\n return {\n width: container.clientWidth - val('padding-left') - val('padding-right'),\n height: container.clientHeight - val('padding-top') - val('padding-bottom')\n };\n }() : {\n // fallback if no container (not 0 b/c can be used for dividing etc)\n width: 1,\n height: 1\n });\n },\n width: function width() {\n return this.size().width;\n },\n height: function height() {\n return this.size().height;\n },\n extent: function extent() {\n var pan = this._private.pan;\n var zoom = this._private.zoom;\n var rb = this.renderedExtent();\n var b = {\n x1: (rb.x1 - pan.x) / zoom,\n x2: (rb.x2 - pan.x) / zoom,\n y1: (rb.y1 - pan.y) / zoom,\n y2: (rb.y2 - pan.y) / zoom\n };\n b.w = b.x2 - b.x1;\n b.h = b.y2 - b.y1;\n return b;\n },\n renderedExtent: function renderedExtent() {\n var width = this.width();\n var height = this.height();\n return {\n x1: 0,\n y1: 0,\n x2: width,\n y2: height,\n w: width,\n h: height\n };\n },\n multiClickDebounceTime: function multiClickDebounceTime(_int) {\n if (_int) this._private.multiClickDebounceTime = _int;else return this._private.multiClickDebounceTime;\n return this; // chaining\n }\n};\n\n// aliases\ncorefn$1.centre = corefn$1.center;\n\n// backwards compatibility\ncorefn$1.autolockNodes = corefn$1.autolock;\ncorefn$1.autoungrabifyNodes = corefn$1.autoungrabify;\n\nvar fn = {\n data: define.data({\n field: 'data',\n bindingEvent: 'data',\n allowBinding: true,\n allowSetting: true,\n settingEvent: 'data',\n settingTriggersEvent: true,\n triggerFnName: 'trigger',\n allowGetting: true,\n updateStyle: true\n }),\n removeData: define.removeData({\n field: 'data',\n event: 'data',\n triggerFnName: 'trigger',\n triggerEvent: true,\n updateStyle: true\n }),\n scratch: define.data({\n field: 'scratch',\n bindingEvent: 'scratch',\n allowBinding: true,\n allowSetting: true,\n settingEvent: 'scratch',\n settingTriggersEvent: true,\n triggerFnName: 'trigger',\n allowGetting: true,\n updateStyle: true\n }),\n removeScratch: define.removeData({\n field: 'scratch',\n event: 'scratch',\n triggerFnName: 'trigger',\n triggerEvent: true,\n updateStyle: true\n })\n};\n\n// aliases\nfn.attr = fn.data;\nfn.removeAttr = fn.removeData;\n\nvar Core = function Core(opts) {\n var cy = this;\n opts = extend({}, opts);\n var container = opts.container;\n\n // allow for passing a wrapped jquery object\n // e.g. cytoscape({ container: $('#cy') })\n if (container && !htmlElement(container) && htmlElement(container[0])) {\n container = container[0];\n }\n var reg = container ? container._cyreg : null; // e.g. already registered some info (e.g. readies) via jquery\n reg = reg || {};\n if (reg && reg.cy) {\n reg.cy.destroy();\n reg = {}; // old instance => replace reg completely\n }\n var readies = reg.readies = reg.readies || [];\n if (container) {\n container._cyreg = reg;\n } // make sure container assoc'd reg points to this cy\n reg.cy = cy;\n var head = _window !== undefined && container !== undefined && !opts.headless;\n var options = opts;\n options.layout = extend({\n name: head ? 'grid' : 'null'\n }, options.layout);\n options.renderer = extend({\n name: head ? 'canvas' : 'null'\n }, options.renderer);\n var defVal = function defVal(def, val, altVal) {\n if (val !== undefined) {\n return val;\n } else if (altVal !== undefined) {\n return altVal;\n } else {\n return def;\n }\n };\n var _p = this._private = {\n container: container,\n // html dom ele container\n ready: false,\n // whether ready has been triggered\n options: options,\n // cached options\n elements: new Collection(this),\n // elements in the graph\n listeners: [],\n // list of listeners\n aniEles: new Collection(this),\n // elements being animated\n data: options.data || {},\n // data for the core\n scratch: {},\n // scratch object for core\n layout: null,\n renderer: null,\n destroyed: false,\n // whether destroy was called\n notificationsEnabled: true,\n // whether notifications are sent to the renderer\n minZoom: 1e-50,\n maxZoom: 1e50,\n zoomingEnabled: defVal(true, options.zoomingEnabled),\n userZoomingEnabled: defVal(true, options.userZoomingEnabled),\n panningEnabled: defVal(true, options.panningEnabled),\n userPanningEnabled: defVal(true, options.userPanningEnabled),\n boxSelectionEnabled: defVal(true, options.boxSelectionEnabled),\n autolock: defVal(false, options.autolock, options.autolockNodes),\n autoungrabify: defVal(false, options.autoungrabify, options.autoungrabifyNodes),\n autounselectify: defVal(false, options.autounselectify),\n styleEnabled: options.styleEnabled === undefined ? head : options.styleEnabled,\n zoom: number$1(options.zoom) ? options.zoom : 1,\n pan: {\n x: plainObject(options.pan) && number$1(options.pan.x) ? options.pan.x : 0,\n y: plainObject(options.pan) && number$1(options.pan.y) ? options.pan.y : 0\n },\n animation: {\n // object for currently-running animations\n current: [],\n queue: []\n },\n hasCompoundNodes: false,\n multiClickDebounceTime: defVal(250, options.multiClickDebounceTime)\n };\n this.createEmitter();\n\n // set selection type\n this.selectionType(options.selectionType);\n\n // init zoom bounds\n this.zoomRange({\n min: options.minZoom,\n max: options.maxZoom\n });\n var loadExtData = function loadExtData(extData, next) {\n var anyIsPromise = extData.some(promise);\n if (anyIsPromise) {\n return Promise$1.all(extData).then(next); // load all data asynchronously, then exec rest of init\n } else {\n next(extData); // exec synchronously for convenience\n }\n };\n\n // start with the default stylesheet so we have something before loading an external stylesheet\n if (_p.styleEnabled) {\n cy.setStyle([]);\n }\n\n // create the renderer\n var rendererOptions = extend({}, options, options.renderer); // allow rendering hints in top level options\n cy.initRenderer(rendererOptions);\n var setElesAndLayout = function setElesAndLayout(elements, onload, ondone) {\n cy.notifications(false);\n\n // remove old elements\n var oldEles = cy.mutableElements();\n if (oldEles.length > 0) {\n oldEles.remove();\n }\n if (elements != null) {\n if (plainObject(elements) || array(elements)) {\n cy.add(elements);\n }\n }\n cy.one('layoutready', function (e) {\n cy.notifications(true);\n cy.emit(e); // we missed this event by turning notifications off, so pass it on\n\n cy.one('load', onload);\n cy.emitAndNotify('load');\n }).one('layoutstop', function () {\n cy.one('done', ondone);\n cy.emit('done');\n });\n var layoutOpts = extend({}, cy._private.options.layout);\n layoutOpts.eles = cy.elements();\n cy.layout(layoutOpts).run();\n };\n loadExtData([options.style, options.elements], function (thens) {\n var initStyle = thens[0];\n var initEles = thens[1];\n\n // init style\n if (_p.styleEnabled) {\n cy.style().append(initStyle);\n }\n\n // initial load\n setElesAndLayout(initEles, function () {\n // onready\n cy.startAnimationLoop();\n _p.ready = true;\n\n // if a ready callback is specified as an option, the bind it\n if (fn$6(options.ready)) {\n cy.on('ready', options.ready);\n }\n\n // bind all the ready handlers registered before creating this instance\n for (var i = 0; i < readies.length; i++) {\n var fn = readies[i];\n cy.on('ready', fn);\n }\n if (reg) {\n reg.readies = [];\n } // clear b/c we've bound them all and don't want to keep it around in case a new core uses the same div etc\n\n cy.emit('ready');\n }, options.done);\n });\n};\nvar corefn = Core.prototype; // short alias\n\nextend(corefn, {\n instanceString: function instanceString() {\n return 'core';\n },\n isReady: function isReady() {\n return this._private.ready;\n },\n destroyed: function destroyed() {\n return this._private.destroyed;\n },\n ready: function ready(fn) {\n if (this.isReady()) {\n this.emitter().emit('ready', [], fn); // just calls fn as though triggered via ready event\n } else {\n this.on('ready', fn);\n }\n return this;\n },\n destroy: function destroy() {\n var cy = this;\n if (cy.destroyed()) return;\n cy.stopAnimationLoop();\n cy.destroyRenderer();\n this.emit('destroy');\n cy._private.destroyed = true;\n return cy;\n },\n hasElementWithId: function hasElementWithId(id) {\n return this._private.elements.hasElementWithId(id);\n },\n getElementById: function getElementById(id) {\n return this._private.elements.getElementById(id);\n },\n hasCompoundNodes: function hasCompoundNodes() {\n return this._private.hasCompoundNodes;\n },\n headless: function headless() {\n return this._private.renderer.isHeadless();\n },\n styleEnabled: function styleEnabled() {\n return this._private.styleEnabled;\n },\n addToPool: function addToPool(eles) {\n this._private.elements.merge(eles);\n return this; // chaining\n },\n removeFromPool: function removeFromPool(eles) {\n this._private.elements.unmerge(eles);\n return this;\n },\n container: function container() {\n return this._private.container || null;\n },\n window: function window() {\n var container = this._private.container;\n if (container == null) return _window;\n var ownerDocument = this._private.container.ownerDocument;\n if (ownerDocument === undefined || ownerDocument == null) {\n return _window;\n }\n return ownerDocument.defaultView || _window;\n },\n mount: function mount(container) {\n if (container == null) {\n return;\n }\n var cy = this;\n var _p = cy._private;\n var options = _p.options;\n if (!htmlElement(container) && htmlElement(container[0])) {\n container = container[0];\n }\n cy.stopAnimationLoop();\n cy.destroyRenderer();\n _p.container = container;\n _p.styleEnabled = true;\n cy.invalidateSize();\n cy.initRenderer(extend({}, options, options.renderer, {\n // allow custom renderer name to be re-used, otherwise use canvas\n name: options.renderer.name === 'null' ? 'canvas' : options.renderer.name\n }));\n cy.startAnimationLoop();\n cy.style(options.style);\n cy.emit('mount');\n return cy;\n },\n unmount: function unmount() {\n var cy = this;\n cy.stopAnimationLoop();\n cy.destroyRenderer();\n cy.initRenderer({\n name: 'null'\n });\n cy.emit('unmount');\n return cy;\n },\n options: function options() {\n return copy(this._private.options);\n },\n json: function json(obj) {\n var cy = this;\n var _p = cy._private;\n var eles = cy.mutableElements();\n var getFreshRef = function getFreshRef(ele) {\n return cy.getElementById(ele.id());\n };\n if (plainObject(obj)) {\n // set\n\n cy.startBatch();\n if (obj.elements) {\n var idInJson = {};\n var updateEles = function updateEles(jsons, gr) {\n var toAdd = [];\n var toMod = [];\n for (var i = 0; i < jsons.length; i++) {\n var json = jsons[i];\n if (!json.data.id) {\n warn('cy.json() cannot handle elements without an ID attribute');\n continue;\n }\n var id = '' + json.data.id; // id must be string\n var ele = cy.getElementById(id);\n idInJson[id] = true;\n if (ele.length !== 0) {\n // existing element should be updated\n toMod.push({\n ele: ele,\n json: json\n });\n } else {\n // otherwise should be added\n if (gr) {\n json.group = gr;\n toAdd.push(json);\n } else {\n toAdd.push(json);\n }\n }\n }\n cy.add(toAdd);\n for (var _i = 0; _i < toMod.length; _i++) {\n var _toMod$_i = toMod[_i],\n _ele = _toMod$_i.ele,\n _json = _toMod$_i.json;\n _ele.json(_json);\n }\n };\n if (array(obj.elements)) {\n // elements: []\n updateEles(obj.elements);\n } else {\n // elements: { nodes: [], edges: [] }\n var grs = ['nodes', 'edges'];\n for (var i = 0; i < grs.length; i++) {\n var gr = grs[i];\n var elements = obj.elements[gr];\n if (array(elements)) {\n updateEles(elements, gr);\n }\n }\n }\n var parentsToRemove = cy.collection();\n eles.filter(function (ele) {\n return !idInJson[ele.id()];\n }).forEach(function (ele) {\n if (ele.isParent()) {\n parentsToRemove.merge(ele);\n } else {\n ele.remove();\n }\n });\n\n // so that children are not removed w/parent\n parentsToRemove.forEach(function (ele) {\n return ele.children().move({\n parent: null\n });\n });\n\n // intermediate parents may be moved by prior line, so make sure we remove by fresh refs\n parentsToRemove.forEach(function (ele) {\n return getFreshRef(ele).remove();\n });\n }\n if (obj.style) {\n cy.style(obj.style);\n }\n if (obj.zoom != null && obj.zoom !== _p.zoom) {\n cy.zoom(obj.zoom);\n }\n if (obj.pan) {\n if (obj.pan.x !== _p.pan.x || obj.pan.y !== _p.pan.y) {\n cy.pan(obj.pan);\n }\n }\n if (obj.data) {\n cy.data(obj.data);\n }\n var fields = ['minZoom', 'maxZoom', 'zoomingEnabled', 'userZoomingEnabled', 'panningEnabled', 'userPanningEnabled', 'boxSelectionEnabled', 'autolock', 'autoungrabify', 'autounselectify', 'multiClickDebounceTime'];\n for (var _i2 = 0; _i2 < fields.length; _i2++) {\n var f = fields[_i2];\n if (obj[f] != null) {\n cy[f](obj[f]);\n }\n }\n cy.endBatch();\n return this; // chaining\n } else {\n // get\n var flat = !!obj;\n var json = {};\n if (flat) {\n json.elements = this.elements().map(function (ele) {\n return ele.json();\n });\n } else {\n json.elements = {};\n eles.forEach(function (ele) {\n var group = ele.group();\n if (!json.elements[group]) {\n json.elements[group] = [];\n }\n json.elements[group].push(ele.json());\n });\n }\n if (this._private.styleEnabled) {\n json.style = cy.style().json();\n }\n json.data = copy(cy.data());\n var options = _p.options;\n json.zoomingEnabled = _p.zoomingEnabled;\n json.userZoomingEnabled = _p.userZoomingEnabled;\n json.zoom = _p.zoom;\n json.minZoom = _p.minZoom;\n json.maxZoom = _p.maxZoom;\n json.panningEnabled = _p.panningEnabled;\n json.userPanningEnabled = _p.userPanningEnabled;\n json.pan = copy(_p.pan);\n json.boxSelectionEnabled = _p.boxSelectionEnabled;\n json.renderer = copy(options.renderer);\n json.hideEdgesOnViewport = options.hideEdgesOnViewport;\n json.textureOnViewport = options.textureOnViewport;\n json.wheelSensitivity = options.wheelSensitivity;\n json.motionBlur = options.motionBlur;\n json.multiClickDebounceTime = options.multiClickDebounceTime;\n return json;\n }\n }\n});\ncorefn.$id = corefn.getElementById;\n[corefn$9, corefn$8, elesfn, corefn$7, corefn$6, corefn$5, corefn$4, corefn$3, corefn$2, corefn$1, fn].forEach(function (props) {\n extend(corefn, props);\n});\n\n/* eslint-disable no-unused-vars */\nvar defaults$7 = {\n fit: true,\n // whether to fit the viewport to the graph\n directed: false,\n // whether the tree is directed downwards (or edges can point in any direction if false)\n direction: 'downward',\n // determines the direction in which the tree structure is drawn. The possible values are 'downward', 'upward', 'rightward', or 'leftward'.\n padding: 30,\n // padding on fit\n circle: false,\n // put depths in concentric circles if true, put depths top down if false\n grid: false,\n // whether to create an even grid into which the DAG is placed (circle:false only)\n spacingFactor: 1.75,\n // positive spacing factor, larger => more space between nodes (N.B. n/a if causes overlap)\n boundingBox: undefined,\n // constrain layout bounds; { x1, y1, x2, y2 } or { x1, y1, w, h }\n avoidOverlap: true,\n // prevents node overlap, may overflow boundingBox if not enough space\n nodeDimensionsIncludeLabels: false,\n // Excludes the label when calculating node bounding boxes for the layout algorithm\n roots: undefined,\n // the roots of the trees\n depthSort: undefined,\n // a sorting function to order nodes at equal depth. e.g. function(a, b){ return a.data('weight') - b.data('weight') }\n animate: false,\n // whether to transition the node positions\n animationDuration: 500,\n // duration of animation in ms if enabled\n animationEasing: undefined,\n // easing of animation if enabled,\n animateFilter: function animateFilter(node, i) {\n return true;\n },\n // a function that determines whether the node should be animated. All nodes animated by default on animate enabled. Non-animated nodes are positioned immediately when the layout starts\n ready: undefined,\n // callback on layoutready\n stop: undefined,\n // callback on layoutstop\n transform: function transform(node, position) {\n return position;\n } // transform a given node position. Useful for changing flow direction in discrete layouts\n};\nvar deprecatedOptionDefaults = {\n maximal: false,\n // whether to shift nodes down their natural BFS depths in order to avoid upwards edges (DAGS only); setting acyclic to true sets maximal to true also\n acyclic: false // whether the tree is acyclic and thus a node could be shifted (due to the maximal option) multiple times without causing an infinite loop; setting to true sets maximal to true also; if you are uncertain whether a tree is acyclic, set to false to avoid potential infinite loops\n};\n\n/* eslint-enable */\n\nvar getInfo = function getInfo(ele) {\n return ele.scratch('breadthfirst');\n};\nvar setInfo = function setInfo(ele, obj) {\n return ele.scratch('breadthfirst', obj);\n};\nfunction BreadthFirstLayout(options) {\n this.options = extend({}, defaults$7, deprecatedOptionDefaults, options);\n}\nBreadthFirstLayout.prototype.run = function () {\n var options = this.options;\n var cy = options.cy;\n var eles = options.eles;\n var nodes = eles.nodes().filter(function (n) {\n return n.isChildless();\n });\n var graph = eles;\n var directed = options.directed;\n var maximal = options.acyclic || options.maximal || options.maximalAdjustments > 0; // maximalAdjustments for compat. w/ old code; also, setting acyclic to true sets maximal to true\n\n var hasBoundingBox = !!options.boundingBox;\n var bb = makeBoundingBox(hasBoundingBox ? options.boundingBox : structuredClone(cy.extent()));\n var roots;\n if (elementOrCollection(options.roots)) {\n roots = options.roots;\n } else if (array(options.roots)) {\n var rootsArray = [];\n for (var i = 0; i < options.roots.length; i++) {\n var id = options.roots[i];\n var ele = cy.getElementById(id);\n rootsArray.push(ele);\n }\n roots = cy.collection(rootsArray);\n } else if (string(options.roots)) {\n roots = cy.$(options.roots);\n } else {\n if (directed) {\n roots = nodes.roots();\n } else {\n var components = eles.components();\n roots = cy.collection();\n var _loop = function _loop() {\n var comp = components[_i];\n var maxDegree = comp.maxDegree(false);\n var compRoots = comp.filter(function (ele) {\n return ele.degree(false) === maxDegree;\n });\n roots = roots.add(compRoots);\n };\n for (var _i = 0; _i < components.length; _i++) {\n _loop();\n }\n }\n }\n var depths = [];\n var foundByBfs = {};\n var addToDepth = function addToDepth(ele, d) {\n if (depths[d] == null) {\n depths[d] = [];\n }\n var i = depths[d].length;\n depths[d].push(ele);\n setInfo(ele, {\n index: i,\n depth: d\n });\n };\n var changeDepth = function changeDepth(ele, newDepth) {\n var _getInfo = getInfo(ele),\n depth = _getInfo.depth,\n index = _getInfo.index;\n depths[depth][index] = null;\n\n // add only childless nodes\n if (ele.isChildless()) addToDepth(ele, newDepth);\n };\n\n // find the depths of the nodes\n graph.bfs({\n roots: roots,\n directed: options.directed,\n visit: function visit(node, edge, pNode, i, depth) {\n var ele = node[0];\n var id = ele.id();\n\n // add only childless nodes\n if (ele.isChildless()) addToDepth(ele, depth);\n foundByBfs[id] = true;\n }\n });\n\n // check for nodes not found by bfs\n var orphanNodes = [];\n for (var _i2 = 0; _i2 < nodes.length; _i2++) {\n var _ele = nodes[_i2];\n if (foundByBfs[_ele.id()]) {\n continue;\n } else {\n orphanNodes.push(_ele);\n }\n }\n\n // assign the nodes a depth and index\n var assignDepthsAt = function assignDepthsAt(i) {\n var eles = depths[i];\n for (var j = 0; j < eles.length; j++) {\n var _ele2 = eles[j];\n if (_ele2 == null) {\n eles.splice(j, 1);\n j--;\n continue;\n }\n setInfo(_ele2, {\n depth: i,\n index: j\n });\n }\n };\n var adjustMaximally = function adjustMaximally(ele, shifted) {\n var eInfo = getInfo(ele);\n var incomers = ele.incomers().filter(function (el) {\n return el.isNode() && eles.has(el);\n });\n var maxDepth = -1;\n var id = ele.id();\n for (var k = 0; k < incomers.length; k++) {\n var incmr = incomers[k];\n var iInfo = getInfo(incmr);\n maxDepth = Math.max(maxDepth, iInfo.depth);\n }\n if (eInfo.depth <= maxDepth) {\n if (!options.acyclic && shifted[id]) {\n return null;\n }\n var newDepth = maxDepth + 1;\n changeDepth(ele, newDepth);\n shifted[id] = newDepth;\n return true;\n }\n return false;\n };\n\n // for the directed case, try to make the edges all go down (i.e. depth i => depth i + 1)\n if (directed && maximal) {\n var Q = [];\n var shifted = {};\n var enqueue = function enqueue(n) {\n return Q.push(n);\n };\n var dequeue = function dequeue() {\n return Q.shift();\n };\n nodes.forEach(function (n) {\n return Q.push(n);\n });\n while (Q.length > 0) {\n var _ele3 = dequeue();\n var didShift = adjustMaximally(_ele3, shifted);\n if (didShift) {\n _ele3.outgoers().filter(function (el) {\n return el.isNode() && eles.has(el);\n }).forEach(enqueue);\n } else if (didShift === null) {\n warn('Detected double maximal shift for node `' + _ele3.id() + '`. Bailing maximal adjustment due to cycle. Use `options.maximal: true` only on DAGs.');\n break; // exit on failure\n }\n }\n }\n\n // find min distance we need to leave between nodes\n var minDistance = 0;\n if (options.avoidOverlap) {\n for (var _i3 = 0; _i3 < nodes.length; _i3++) {\n var n = nodes[_i3];\n var nbb = n.layoutDimensions(options);\n var w = nbb.w;\n var h = nbb.h;\n minDistance = Math.max(minDistance, w, h);\n }\n }\n\n // get the weighted percent for an element based on its connectivity to other levels\n var cachedWeightedPercent = {};\n var getWeightedPercent = function getWeightedPercent(ele) {\n if (cachedWeightedPercent[ele.id()]) {\n return cachedWeightedPercent[ele.id()];\n }\n var eleDepth = getInfo(ele).depth;\n var neighbors = ele.neighborhood();\n var percent = 0;\n var samples = 0;\n for (var _i4 = 0; _i4 < neighbors.length; _i4++) {\n var neighbor = neighbors[_i4];\n if (neighbor.isEdge() || neighbor.isParent() || !nodes.has(neighbor)) {\n continue;\n }\n var bf = getInfo(neighbor);\n if (bf == null) {\n continue;\n }\n var index = bf.index;\n var depth = bf.depth;\n\n // unassigned neighbours shouldn't affect the ordering\n if (index == null || depth == null) {\n continue;\n }\n var nDepth = depths[depth].length;\n if (depth < eleDepth) {\n // only get influenced by elements above\n percent += index / nDepth;\n samples++;\n }\n }\n samples = Math.max(1, samples);\n percent = percent / samples;\n if (samples === 0) {\n // put lone nodes at the start\n percent = 0;\n }\n cachedWeightedPercent[ele.id()] = percent;\n return percent;\n };\n\n // rearrange the indices in each depth level based on connectivity\n var sortFn = function sortFn(a, b) {\n var apct = getWeightedPercent(a);\n var bpct = getWeightedPercent(b);\n var diff = apct - bpct;\n if (diff === 0) {\n return ascending(a.id(), b.id()); // make sure sort doesn't have don't-care comparisons\n } else {\n return diff;\n }\n };\n if (options.depthSort !== undefined) {\n sortFn = options.depthSort;\n }\n var depthsLen = depths.length;\n\n // sort each level to make connected nodes closer\n for (var _i5 = 0; _i5 < depthsLen; _i5++) {\n depths[_i5].sort(sortFn);\n assignDepthsAt(_i5);\n }\n\n // assign orphan nodes to a new top-level depth\n var orphanDepth = [];\n for (var _i6 = 0; _i6 < orphanNodes.length; _i6++) {\n orphanDepth.push(orphanNodes[_i6]);\n }\n var assignDepths = function assignDepths() {\n for (var _i7 = 0; _i7 < depthsLen; _i7++) {\n assignDepthsAt(_i7);\n }\n };\n\n // add a new top-level depth only when there are orphan nodes\n if (orphanDepth.length) {\n depths.unshift(orphanDepth);\n depthsLen = depths.length;\n assignDepths();\n }\n var biggestDepthSize = 0;\n for (var _i8 = 0; _i8 < depthsLen; _i8++) {\n biggestDepthSize = Math.max(depths[_i8].length, biggestDepthSize);\n }\n var center = {\n x: bb.x1 + bb.w / 2,\n y: bb.y1 + bb.h / 2\n };\n\n // average node size\n var aveNodeSize = nodes.reduce(function (acc, node) {\n return function (box) {\n return {\n w: acc.w === -1 ? box.w : (acc.w + box.w) / 2,\n h: acc.h === -1 ? box.h : (acc.h + box.h) / 2\n };\n }(node.boundingBox({\n includeLabels: options.nodeDimensionsIncludeLabels\n }));\n }, {\n w: -1,\n h: -1\n });\n var distanceY = Math.max(\n // only one depth\n depthsLen === 1 ? 0 :\n // inside a bounding box, no need for top & bottom padding\n hasBoundingBox ? (bb.h - options.padding * 2 - aveNodeSize.h) / (depthsLen - 1) : (bb.h - options.padding * 2 - aveNodeSize.h) / (depthsLen + 1), minDistance);\n var maxDepthSize = depths.reduce(function (max, eles) {\n return Math.max(max, eles.length);\n }, 0);\n var getPositionTopBottom = function getPositionTopBottom(ele) {\n var _getInfo2 = getInfo(ele),\n depth = _getInfo2.depth,\n index = _getInfo2.index;\n if (options.circle) {\n var radiusStepSize = Math.min(bb.w / 2 / depthsLen, bb.h / 2 / depthsLen);\n radiusStepSize = Math.max(radiusStepSize, minDistance);\n var radius = radiusStepSize * depth + radiusStepSize - (depthsLen > 0 && depths[0].length <= 3 ? radiusStepSize / 2 : 0);\n var theta = 2 * Math.PI / depths[depth].length * index;\n if (depth === 0 && depths[0].length === 1) {\n radius = 1;\n }\n return {\n x: center.x + radius * Math.cos(theta),\n y: center.y + radius * Math.sin(theta)\n };\n } else {\n var depthSize = depths[depth].length;\n var distanceX = Math.max(\n // only one depth\n depthSize === 1 ? 0 :\n // inside a bounding box, no need for left & right padding\n hasBoundingBox ? (bb.w - options.padding * 2 - aveNodeSize.w) / ((options.grid ? maxDepthSize : depthSize) - 1) : (bb.w - options.padding * 2 - aveNodeSize.w) / ((options.grid ? maxDepthSize : depthSize) + 1), minDistance);\n var epos = {\n x: center.x + (index + 1 - (depthSize + 1) / 2) * distanceX,\n y: center.y + (depth + 1 - (depthsLen + 1) / 2) * distanceY\n };\n return epos;\n }\n };\n var rotateDegrees = {\n 'downward': 0,\n 'leftward': 90,\n 'upward': 180,\n 'rightward': -90\n };\n if (Object.keys(rotateDegrees).indexOf(options.direction) === -1) {\n error(\"Invalid direction '\".concat(options.direction, \"' specified for breadthfirst layout. Valid values are: \").concat(Object.keys(rotateDegrees).join(', ')));\n }\n var getPosition = function getPosition(ele) {\n return rotatePosAndSkewByBox(getPositionTopBottom(ele), bb, rotateDegrees[options.direction]);\n };\n eles.nodes().layoutPositions(this, options, getPosition);\n return this; // chaining\n};\n\nvar defaults$6 = {\n fit: true,\n // whether to fit the viewport to the graph\n padding: 30,\n // the padding on fit\n boundingBox: undefined,\n // constrain layout bounds; { x1, y1, x2, y2 } or { x1, y1, w, h }\n avoidOverlap: true,\n // prevents node overlap, may overflow boundingBox and radius if not enough space\n nodeDimensionsIncludeLabels: false,\n // Excludes the label when calculating node bounding boxes for the layout algorithm\n spacingFactor: undefined,\n // Applies a multiplicative factor (>0) to expand or compress the overall area that the nodes take up\n radius: undefined,\n // the radius of the circle\n startAngle: 3 / 2 * Math.PI,\n // where nodes start in radians\n sweep: undefined,\n // how many radians should be between the first and last node (defaults to full circle)\n clockwise: true,\n // whether the layout should go clockwise (true) or counterclockwise/anticlockwise (false)\n sort: undefined,\n // a sorting function to order the nodes; e.g. function(a, b){ return a.data('weight') - b.data('weight') }\n animate: false,\n // whether to transition the node positions\n animationDuration: 500,\n // duration of animation in ms if enabled\n animationEasing: undefined,\n // easing of animation if enabled\n animateFilter: function animateFilter(node, i) {\n return true;\n },\n // a function that determines whether the node should be animated. All nodes animated by default on animate enabled. Non-animated nodes are positioned immediately when the layout starts\n ready: undefined,\n // callback on layoutready\n stop: undefined,\n // callback on layoutstop\n transform: function transform(node, position) {\n return position;\n } // transform a given node position. Useful for changing flow direction in discrete layouts \n};\nfunction CircleLayout(options) {\n this.options = extend({}, defaults$6, options);\n}\nCircleLayout.prototype.run = function () {\n var params = this.options;\n var options = params;\n var cy = params.cy;\n var eles = options.eles;\n var clockwise = options.counterclockwise !== undefined ? !options.counterclockwise : options.clockwise;\n var nodes = eles.nodes().not(':parent');\n if (options.sort) {\n nodes = nodes.sort(options.sort);\n }\n var bb = makeBoundingBox(options.boundingBox ? options.boundingBox : {\n x1: 0,\n y1: 0,\n w: cy.width(),\n h: cy.height()\n });\n var center = {\n x: bb.x1 + bb.w / 2,\n y: bb.y1 + bb.h / 2\n };\n var sweep = options.sweep === undefined ? 2 * Math.PI - 2 * Math.PI / nodes.length : options.sweep;\n var dTheta = sweep / Math.max(1, nodes.length - 1);\n var r;\n var minDistance = 0;\n for (var i = 0; i < nodes.length; i++) {\n var n = nodes[i];\n var nbb = n.layoutDimensions(options);\n var w = nbb.w;\n var h = nbb.h;\n minDistance = Math.max(minDistance, w, h);\n }\n if (number$1(options.radius)) {\n r = options.radius;\n } else if (nodes.length <= 1) {\n r = 0;\n } else {\n r = Math.min(bb.h, bb.w) / 2 - minDistance;\n }\n\n // calculate the radius\n if (nodes.length > 1 && options.avoidOverlap) {\n // but only if more than one node (can't overlap)\n minDistance *= 1.75; // just to have some nice spacing\n\n var dcos = Math.cos(dTheta) - Math.cos(0);\n var dsin = Math.sin(dTheta) - Math.sin(0);\n var rMin = Math.sqrt(minDistance * minDistance / (dcos * dcos + dsin * dsin)); // s.t. no nodes overlapping\n r = Math.max(rMin, r);\n }\n var getPos = function getPos(ele, i) {\n var theta = options.startAngle + i * dTheta * (clockwise ? 1 : -1);\n var rx = r * Math.cos(theta);\n var ry = r * Math.sin(theta);\n var pos = {\n x: center.x + rx,\n y: center.y + ry\n };\n return pos;\n };\n eles.nodes().layoutPositions(this, options, getPos);\n return this; // chaining\n};\n\nvar defaults$5 = {\n fit: true,\n // whether to fit the viewport to the graph\n padding: 30,\n // the padding on fit\n startAngle: 3 / 2 * Math.PI,\n // where nodes start in radians\n sweep: undefined,\n // how many radians should be between the first and last node (defaults to full circle)\n clockwise: true,\n // whether the layout should go clockwise (true) or counterclockwise/anticlockwise (false)\n equidistant: false,\n // whether levels have an equal radial distance betwen them, may cause bounding box overflow\n minNodeSpacing: 10,\n // min spacing between outside of nodes (used for radius adjustment)\n boundingBox: undefined,\n // constrain layout bounds; { x1, y1, x2, y2 } or { x1, y1, w, h }\n avoidOverlap: true,\n // prevents node overlap, may overflow boundingBox if not enough space\n nodeDimensionsIncludeLabels: false,\n // Excludes the label when calculating node bounding boxes for the layout algorithm\n height: undefined,\n // height of layout area (overrides container height)\n width: undefined,\n // width of layout area (overrides container width)\n spacingFactor: undefined,\n // Applies a multiplicative factor (>0) to expand or compress the overall area that the nodes take up\n concentric: function concentric(node) {\n // returns numeric value for each node, placing higher nodes in levels towards the centre\n return node.degree();\n },\n levelWidth: function levelWidth(nodes) {\n // the variation of concentric values in each level\n return nodes.maxDegree() / 4;\n },\n animate: false,\n // whether to transition the node positions\n animationDuration: 500,\n // duration of animation in ms if enabled\n animationEasing: undefined,\n // easing of animation if enabled\n animateFilter: function animateFilter(node, i) {\n return true;\n },\n // a function that determines whether the node should be animated. All nodes animated by default on animate enabled. Non-animated nodes are positioned immediately when the layout starts\n ready: undefined,\n // callback on layoutready\n stop: undefined,\n // callback on layoutstop\n transform: function transform(node, position) {\n return position;\n } // transform a given node position. Useful for changing flow direction in discrete layouts\n};\nfunction ConcentricLayout(options) {\n this.options = extend({}, defaults$5, options);\n}\nConcentricLayout.prototype.run = function () {\n var params = this.options;\n var options = params;\n var clockwise = options.counterclockwise !== undefined ? !options.counterclockwise : options.clockwise;\n var cy = params.cy;\n var eles = options.eles;\n var nodes = eles.nodes().not(':parent');\n var bb = makeBoundingBox(options.boundingBox ? options.boundingBox : {\n x1: 0,\n y1: 0,\n w: cy.width(),\n h: cy.height()\n });\n var center = {\n x: bb.x1 + bb.w / 2,\n y: bb.y1 + bb.h / 2\n };\n var nodeValues = []; // { node, value }\n var maxNodeSize = 0;\n for (var i = 0; i < nodes.length; i++) {\n var node = nodes[i];\n var value = undefined;\n\n // calculate the node value\n value = options.concentric(node);\n nodeValues.push({\n value: value,\n node: node\n });\n\n // for style mapping\n node._private.scratch.concentric = value;\n }\n\n // in case we used the `concentric` in style\n nodes.updateStyle();\n\n // calculate max size now based on potentially updated mappers\n for (var _i = 0; _i < nodes.length; _i++) {\n var _node = nodes[_i];\n var nbb = _node.layoutDimensions(options);\n maxNodeSize = Math.max(maxNodeSize, nbb.w, nbb.h);\n }\n\n // sort node values in descreasing order\n nodeValues.sort(function (a, b) {\n return b.value - a.value;\n });\n var levelWidth = options.levelWidth(nodes);\n\n // put the values into levels\n var levels = [[]];\n var currentLevel = levels[0];\n for (var _i2 = 0; _i2 < nodeValues.length; _i2++) {\n var val = nodeValues[_i2];\n if (currentLevel.length > 0) {\n var diff = Math.abs(currentLevel[0].value - val.value);\n if (diff >= levelWidth) {\n currentLevel = [];\n levels.push(currentLevel);\n }\n }\n currentLevel.push(val);\n }\n\n // create positions from levels\n\n var minDist = maxNodeSize + options.minNodeSpacing; // min dist between nodes\n\n if (!options.avoidOverlap) {\n // then strictly constrain to bb\n var firstLvlHasMulti = levels.length > 0 && levels[0].length > 1;\n var maxR = Math.min(bb.w, bb.h) / 2 - minDist;\n var rStep = maxR / (levels.length + firstLvlHasMulti ? 1 : 0);\n minDist = Math.min(minDist, rStep);\n }\n\n // find the metrics for each level\n var r = 0;\n for (var _i3 = 0; _i3 < levels.length; _i3++) {\n var level = levels[_i3];\n var sweep = options.sweep === undefined ? 2 * Math.PI - 2 * Math.PI / level.length : options.sweep;\n var dTheta = level.dTheta = sweep / Math.max(1, level.length - 1);\n\n // calculate the radius\n if (level.length > 1 && options.avoidOverlap) {\n // but only if more than one node (can't overlap)\n var dcos = Math.cos(dTheta) - Math.cos(0);\n var dsin = Math.sin(dTheta) - Math.sin(0);\n var rMin = Math.sqrt(minDist * minDist / (dcos * dcos + dsin * dsin)); // s.t. no nodes overlapping\n\n r = Math.max(rMin, r);\n }\n level.r = r;\n r += minDist;\n }\n if (options.equidistant) {\n var rDeltaMax = 0;\n var _r = 0;\n for (var _i4 = 0; _i4 < levels.length; _i4++) {\n var _level = levels[_i4];\n var rDelta = _level.r - _r;\n rDeltaMax = Math.max(rDeltaMax, rDelta);\n }\n _r = 0;\n for (var _i5 = 0; _i5 < levels.length; _i5++) {\n var _level2 = levels[_i5];\n if (_i5 === 0) {\n _r = _level2.r;\n }\n _level2.r = _r;\n _r += rDeltaMax;\n }\n }\n\n // calculate the node positions\n var pos = {}; // id => position\n for (var _i6 = 0; _i6 < levels.length; _i6++) {\n var _level3 = levels[_i6];\n var _dTheta = _level3.dTheta;\n var _r2 = _level3.r;\n for (var j = 0; j < _level3.length; j++) {\n var _val = _level3[j];\n var theta = options.startAngle + (clockwise ? 1 : -1) * _dTheta * j;\n var p = {\n x: center.x + _r2 * Math.cos(theta),\n y: center.y + _r2 * Math.sin(theta)\n };\n pos[_val.node.id()] = p;\n }\n }\n\n // position the nodes\n eles.nodes().layoutPositions(this, options, function (ele) {\n var id = ele.id();\n return pos[id];\n });\n return this; // chaining\n};\n\n/*\nThe CoSE layout was written by Gerardo Huck.\nhttps://www.linkedin.com/in/gerardohuck/\n\nBased on the following article:\nhttp://dl.acm.org/citation.cfm?id=1498047\n\nModifications tracked on Github.\n*/\n\nvar DEBUG;\n\n/**\n * @brief : default layout options\n */\nvar defaults$4 = {\n // Called on `layoutready`\n ready: function ready() {},\n // Called on `layoutstop`\n stop: function stop() {},\n // Whether to animate while running the layout\n // true : Animate continuously as the layout is running\n // false : Just show the end result\n // 'end' : Animate with the end result, from the initial positions to the end positions\n animate: true,\n // Easing of the animation for animate:'end'\n animationEasing: undefined,\n // The duration of the animation for animate:'end'\n animationDuration: undefined,\n // A function that determines whether the node should be animated\n // All nodes animated by default on animate enabled\n // Non-animated nodes are positioned immediately when the layout starts\n animateFilter: function animateFilter(node, i) {\n return true;\n },\n // The layout animates only after this many milliseconds for animate:true\n // (prevents flashing on fast runs)\n animationThreshold: 250,\n // Number of iterations between consecutive screen positions update\n refresh: 20,\n // Whether to fit the network view after when done\n fit: true,\n // Padding on fit\n padding: 30,\n // Constrain layout bounds; { x1, y1, x2, y2 } or { x1, y1, w, h }\n boundingBox: undefined,\n // Excludes the label when calculating node bounding boxes for the layout algorithm\n nodeDimensionsIncludeLabels: false,\n // Randomize the initial positions of the nodes (true) or use existing positions (false)\n randomize: false,\n // Extra spacing between components in non-compound graphs\n componentSpacing: 40,\n // Node repulsion (non overlapping) multiplier\n nodeRepulsion: function nodeRepulsion(node) {\n return 2048;\n },\n // Node repulsion (overlapping) multiplier\n nodeOverlap: 4,\n // Ideal edge (non nested) length\n idealEdgeLength: function idealEdgeLength(edge) {\n return 32;\n },\n // Divisor to compute edge forces\n edgeElasticity: function edgeElasticity(edge) {\n return 32;\n },\n // Nesting factor (multiplier) to compute ideal edge length for nested edges\n nestingFactor: 1.2,\n // Gravity force (constant)\n gravity: 1,\n // Maximum number of iterations to perform\n numIter: 1000,\n // Initial temperature (maximum node displacement)\n initialTemp: 1000,\n // Cooling factor (how the temperature is reduced between consecutive iterations\n coolingFactor: 0.99,\n // Lower temperature threshold (below this point the layout will end)\n minTemp: 1.0\n};\n\n/**\n * @brief : constructor\n * @arg options : object containing layout options\n */\nfunction CoseLayout(options) {\n this.options = extend({}, defaults$4, options);\n this.options.layout = this;\n\n // Exclude any edge that has a source or target node that is not in the set of passed-in nodes\n var nodes = this.options.eles.nodes();\n var edges = this.options.eles.edges();\n var notEdges = edges.filter(function (e) {\n var sourceId = e.source().data('id');\n var targetId = e.target().data('id');\n var hasSource = nodes.some(function (n) {\n return n.data('id') === sourceId;\n });\n var hasTarget = nodes.some(function (n) {\n return n.data('id') === targetId;\n });\n return !hasSource || !hasTarget;\n });\n this.options.eles = this.options.eles.not(notEdges);\n}\n\n/**\n * @brief : runs the layout\n */\nCoseLayout.prototype.run = function () {\n var options = this.options;\n var cy = options.cy;\n var layout = this;\n layout.stopped = false;\n if (options.animate === true || options.animate === false) {\n layout.emit({\n type: 'layoutstart',\n layout: layout\n });\n }\n\n // Set DEBUG - Global variable\n if (true === options.debug) {\n DEBUG = true;\n } else {\n DEBUG = false;\n }\n\n // Initialize layout info\n var layoutInfo = createLayoutInfo(cy, layout, options);\n\n // Show LayoutInfo contents if debugging\n if (DEBUG) {\n printLayoutInfo(layoutInfo);\n }\n\n // If required, randomize node positions\n if (options.randomize) {\n randomizePositions(layoutInfo);\n }\n var startTime = performanceNow();\n var refresh = function refresh() {\n refreshPositions(layoutInfo, cy, options);\n\n // Fit the graph if necessary\n if (true === options.fit) {\n cy.fit(options.padding);\n }\n };\n var mainLoop = function mainLoop(i) {\n if (layout.stopped || i >= options.numIter) {\n // logDebug(\"Layout manually stopped. Stopping computation in step \" + i);\n return false;\n }\n\n // Do one step in the phisical simulation\n step(layoutInfo, options);\n\n // Update temperature\n layoutInfo.temperature = layoutInfo.temperature * options.coolingFactor;\n // logDebug(\"New temperature: \" + layoutInfo.temperature);\n\n if (layoutInfo.temperature < options.minTemp) {\n // logDebug(\"Temperature drop below minimum threshold. Stopping computation in step \" + i);\n return false;\n }\n return true;\n };\n var done = function done() {\n if (options.animate === true || options.animate === false) {\n refresh();\n\n // Layout has finished\n layout.one('layoutstop', options.stop);\n layout.emit({\n type: 'layoutstop',\n layout: layout\n });\n } else {\n var nodes = options.eles.nodes();\n var getScaledPos = getScaleInBoundsFn(layoutInfo, options, nodes);\n nodes.layoutPositions(layout, options, getScaledPos);\n }\n };\n var i = 0;\n var loopRet = true;\n if (options.animate === true) {\n var _frame = function frame() {\n var f = 0;\n while (loopRet && f < options.refresh) {\n loopRet = mainLoop(i);\n i++;\n f++;\n }\n if (!loopRet) {\n // it's done\n separateComponents(layoutInfo, options);\n done();\n } else {\n var now = performanceNow();\n if (now - startTime >= options.animationThreshold) {\n refresh();\n }\n requestAnimationFrame(_frame);\n }\n };\n _frame();\n } else {\n while (loopRet) {\n loopRet = mainLoop(i);\n i++;\n }\n separateComponents(layoutInfo, options);\n done();\n }\n return this; // chaining\n};\n\n/**\n * @brief : called on continuous layouts to stop them before they finish\n */\nCoseLayout.prototype.stop = function () {\n this.stopped = true;\n if (this.thread) {\n this.thread.stop();\n }\n this.emit('layoutstop');\n return this; // chaining\n};\nCoseLayout.prototype.destroy = function () {\n if (this.thread) {\n this.thread.stop();\n }\n return this; // chaining\n};\n\n/**\n * @brief : Creates an object which is contains all the data\n * used in the layout process\n * @arg cy : cytoscape.js object\n * @return : layoutInfo object initialized\n */\nvar createLayoutInfo = function createLayoutInfo(cy, layout, options) {\n // Shortcut\n var edges = options.eles.edges();\n var nodes = options.eles.nodes();\n var bb = makeBoundingBox(options.boundingBox ? options.boundingBox : {\n x1: 0,\n y1: 0,\n w: cy.width(),\n h: cy.height()\n });\n var layoutInfo = {\n isCompound: cy.hasCompoundNodes(),\n layoutNodes: [],\n idToIndex: {},\n nodeSize: nodes.size(),\n graphSet: [],\n indexToGraph: [],\n layoutEdges: [],\n edgeSize: edges.size(),\n temperature: options.initialTemp,\n clientWidth: bb.w,\n clientHeight: bb.h,\n boundingBox: bb\n };\n var components = options.eles.components();\n var id2cmptId = {};\n for (var i = 0; i < components.length; i++) {\n var component = components[i];\n for (var j = 0; j < component.length; j++) {\n var node = component[j];\n id2cmptId[node.id()] = i;\n }\n }\n\n // Iterate over all nodes, creating layout nodes\n for (var i = 0; i < layoutInfo.nodeSize; i++) {\n var n = nodes[i];\n var nbb = n.layoutDimensions(options);\n var tempNode = {};\n tempNode.isLocked = n.locked();\n tempNode.id = n.data('id');\n tempNode.parentId = n.data('parent');\n tempNode.cmptId = id2cmptId[n.id()];\n tempNode.children = [];\n tempNode.positionX = n.position('x');\n tempNode.positionY = n.position('y');\n tempNode.offsetX = 0;\n tempNode.offsetY = 0;\n tempNode.height = nbb.w;\n tempNode.width = nbb.h;\n tempNode.maxX = tempNode.positionX + tempNode.width / 2;\n tempNode.minX = tempNode.positionX - tempNode.width / 2;\n tempNode.maxY = tempNode.positionY + tempNode.height / 2;\n tempNode.minY = tempNode.positionY - tempNode.height / 2;\n tempNode.padLeft = parseFloat(n.style('padding'));\n tempNode.padRight = parseFloat(n.style('padding'));\n tempNode.padTop = parseFloat(n.style('padding'));\n tempNode.padBottom = parseFloat(n.style('padding'));\n\n // forces\n tempNode.nodeRepulsion = fn$6(options.nodeRepulsion) ? options.nodeRepulsion(n) : options.nodeRepulsion;\n\n // Add new node\n layoutInfo.layoutNodes.push(tempNode);\n // Add entry to id-index map\n layoutInfo.idToIndex[tempNode.id] = i;\n }\n\n // Inline implementation of a queue, used for traversing the graph in BFS order\n var queue = [];\n var start = 0; // Points to the start the queue\n var end = -1; // Points to the end of the queue\n\n var tempGraph = [];\n\n // Second pass to add child information and\n // initialize queue for hierarchical traversal\n for (var i = 0; i < layoutInfo.nodeSize; i++) {\n var n = layoutInfo.layoutNodes[i];\n var p_id = n.parentId;\n // Check if node n has a parent node\n if (null != p_id) {\n // Add node Id to parent's list of children\n layoutInfo.layoutNodes[layoutInfo.idToIndex[p_id]].children.push(n.id);\n } else {\n // If a node doesn't have a parent, then it's in the root graph\n queue[++end] = n.id;\n tempGraph.push(n.id);\n }\n }\n\n // Add root graph to graphSet\n layoutInfo.graphSet.push(tempGraph);\n\n // Traverse the graph, level by level,\n while (start <= end) {\n // Get the node to visit and remove it from queue\n var node_id = queue[start++];\n var node_ix = layoutInfo.idToIndex[node_id];\n var node = layoutInfo.layoutNodes[node_ix];\n var children = node.children;\n if (children.length > 0) {\n // Add children nodes as a new graph to graph set\n layoutInfo.graphSet.push(children);\n // Add children to que queue to be visited\n for (var i = 0; i < children.length; i++) {\n queue[++end] = children[i];\n }\n }\n }\n\n // Create indexToGraph map\n for (var i = 0; i < layoutInfo.graphSet.length; i++) {\n var graph = layoutInfo.graphSet[i];\n for (var j = 0; j < graph.length; j++) {\n var index = layoutInfo.idToIndex[graph[j]];\n layoutInfo.indexToGraph[index] = i;\n }\n }\n\n // Iterate over all edges, creating Layout Edges\n for (var i = 0; i < layoutInfo.edgeSize; i++) {\n var e = edges[i];\n var tempEdge = {};\n tempEdge.id = e.data('id');\n tempEdge.sourceId = e.data('source');\n tempEdge.targetId = e.data('target');\n\n // Compute ideal length\n var idealLength = fn$6(options.idealEdgeLength) ? options.idealEdgeLength(e) : options.idealEdgeLength;\n var elasticity = fn$6(options.edgeElasticity) ? options.edgeElasticity(e) : options.edgeElasticity;\n\n // Check if it's an inter graph edge\n var sourceIx = layoutInfo.idToIndex[tempEdge.sourceId];\n var targetIx = layoutInfo.idToIndex[tempEdge.targetId];\n var sourceGraph = layoutInfo.indexToGraph[sourceIx];\n var targetGraph = layoutInfo.indexToGraph[targetIx];\n if (sourceGraph != targetGraph) {\n // Find lowest common graph ancestor\n var lca = findLCA(tempEdge.sourceId, tempEdge.targetId, layoutInfo);\n\n // Compute sum of node depths, relative to lca graph\n var lcaGraph = layoutInfo.graphSet[lca];\n var depth = 0;\n\n // Source depth\n var tempNode = layoutInfo.layoutNodes[sourceIx];\n while (-1 === lcaGraph.indexOf(tempNode.id)) {\n tempNode = layoutInfo.layoutNodes[layoutInfo.idToIndex[tempNode.parentId]];\n depth++;\n }\n\n // Target depth\n tempNode = layoutInfo.layoutNodes[targetIx];\n while (-1 === lcaGraph.indexOf(tempNode.id)) {\n tempNode = layoutInfo.layoutNodes[layoutInfo.idToIndex[tempNode.parentId]];\n depth++;\n }\n\n // logDebug('LCA of nodes ' + tempEdge.sourceId + ' and ' + tempEdge.targetId +\n // \". Index: \" + lca + \" Contents: \" + lcaGraph.toString() +\n // \". Depth: \" + depth);\n\n // Update idealLength\n idealLength *= depth * options.nestingFactor;\n }\n tempEdge.idealLength = idealLength;\n tempEdge.elasticity = elasticity;\n layoutInfo.layoutEdges.push(tempEdge);\n }\n\n // Finally, return layoutInfo object\n return layoutInfo;\n};\n\n/**\n * @brief : This function finds the index of the lowest common\n * graph ancestor between 2 nodes in the subtree\n * (from the graph hierarchy induced tree) whose\n * root is graphIx\n *\n * @arg node1: node1's ID\n * @arg node2: node2's ID\n * @arg layoutInfo: layoutInfo object\n *\n */\nvar findLCA = function findLCA(node1, node2, layoutInfo) {\n // Find their common ancester, starting from the root graph\n var res = _findLCA_aux(node1, node2, 0, layoutInfo);\n if (2 > res.count) {\n // If aux function couldn't find the common ancester,\n // then it is the root graph\n return 0;\n } else {\n return res.graph;\n }\n};\n\n/**\n * @brief : Auxiliary function used for LCA computation\n *\n * @arg node1 : node1's ID\n * @arg node2 : node2's ID\n * @arg graphIx : subgraph index\n * @arg layoutInfo : layoutInfo object\n *\n * @return : object of the form {count: X, graph: Y}, where:\n * X is the number of ancestors (max: 2) found in\n * graphIx (and it's subgraphs),\n * Y is the graph index of the lowest graph containing\n * all X nodes\n */\nvar _findLCA_aux = function findLCA_aux(node1, node2, graphIx, layoutInfo) {\n var graph = layoutInfo.graphSet[graphIx];\n // If both nodes belongs to graphIx\n if (-1 < graph.indexOf(node1) && -1 < graph.indexOf(node2)) {\n return {\n count: 2,\n graph: graphIx\n };\n }\n\n // Make recursive calls for all subgraphs\n var c = 0;\n for (var i = 0; i < graph.length; i++) {\n var nodeId = graph[i];\n var nodeIx = layoutInfo.idToIndex[nodeId];\n var children = layoutInfo.layoutNodes[nodeIx].children;\n\n // If the node has no child, skip it\n if (0 === children.length) {\n continue;\n }\n var childGraphIx = layoutInfo.indexToGraph[layoutInfo.idToIndex[children[0]]];\n var result = _findLCA_aux(node1, node2, childGraphIx, layoutInfo);\n if (0 === result.count) {\n // Neither node1 nor node2 are present in this subgraph\n continue;\n } else if (1 === result.count) {\n // One of (node1, node2) is present in this subgraph\n c++;\n if (2 === c) {\n // We've already found both nodes, no need to keep searching\n break;\n }\n } else {\n // Both nodes are present in this subgraph\n return result;\n }\n }\n return {\n count: c,\n graph: graphIx\n };\n};\n\n/**\n * @brief: printsLayoutInfo into js console\n * Only used for debbuging\n */\nvar printLayoutInfo; \n\n/**\n * @brief : Randomizes the position of all nodes\n */\nvar randomizePositions = function randomizePositions(layoutInfo, cy) {\n var width = layoutInfo.clientWidth;\n var height = layoutInfo.clientHeight;\n for (var i = 0; i < layoutInfo.nodeSize; i++) {\n var n = layoutInfo.layoutNodes[i];\n\n // No need to randomize compound nodes or locked nodes\n if (0 === n.children.length && !n.isLocked) {\n n.positionX = Math.random() * width;\n n.positionY = Math.random() * height;\n }\n }\n};\nvar getScaleInBoundsFn = function getScaleInBoundsFn(layoutInfo, options, nodes) {\n var bb = layoutInfo.boundingBox;\n var coseBB = {\n x1: Infinity,\n x2: -Infinity,\n y1: Infinity,\n y2: -Infinity\n };\n if (options.boundingBox) {\n nodes.forEach(function (node) {\n var lnode = layoutInfo.layoutNodes[layoutInfo.idToIndex[node.data('id')]];\n coseBB.x1 = Math.min(coseBB.x1, lnode.positionX);\n coseBB.x2 = Math.max(coseBB.x2, lnode.positionX);\n coseBB.y1 = Math.min(coseBB.y1, lnode.positionY);\n coseBB.y2 = Math.max(coseBB.y2, lnode.positionY);\n });\n coseBB.w = coseBB.x2 - coseBB.x1;\n coseBB.h = coseBB.y2 - coseBB.y1;\n }\n return function (ele, i) {\n var lnode = layoutInfo.layoutNodes[layoutInfo.idToIndex[ele.data('id')]];\n if (options.boundingBox) {\n // then add extra bounding box constraint\n // Handle single node case where coseBB.w or coseBB.h is 0\n var pctX = coseBB.w === 0 ? 0.5 : (lnode.positionX - coseBB.x1) / coseBB.w;\n var pctY = coseBB.h === 0 ? 0.5 : (lnode.positionY - coseBB.y1) / coseBB.h;\n return {\n x: bb.x1 + pctX * bb.w,\n y: bb.y1 + pctY * bb.h\n };\n } else {\n return {\n x: lnode.positionX,\n y: lnode.positionY\n };\n }\n };\n};\n\n/**\n * @brief : Updates the positions of nodes in the network\n * @arg layoutInfo : LayoutInfo object\n * @arg cy : Cytoscape object\n * @arg options : Layout options\n */\nvar refreshPositions = function refreshPositions(layoutInfo, cy, options) {\n // var s = 'Refreshing positions';\n // logDebug(s);\n\n var layout = options.layout;\n var nodes = options.eles.nodes();\n var getScaledPos = getScaleInBoundsFn(layoutInfo, options, nodes);\n nodes.positions(getScaledPos);\n\n // Trigger layoutReady only on first call\n if (true !== layoutInfo.ready) {\n // s = 'Triggering layoutready';\n // logDebug(s);\n layoutInfo.ready = true;\n layout.one('layoutready', options.ready);\n layout.emit({\n type: 'layoutready',\n layout: this\n });\n }\n};\n\n/**\n * @brief : Logs a debug message in JS console, if DEBUG is ON\n */\n// var logDebug = function(text) {\n// if (DEBUG) {\n// console.debug(text);\n// }\n// };\n\n/**\n * @brief : Performs one iteration of the physical simulation\n * @arg layoutInfo : LayoutInfo object already initialized\n * @arg cy : Cytoscape object\n * @arg options : Layout options\n */\nvar step = function step(layoutInfo, options, _step) {\n // var s = \"\\n\\n###############################\";\n // s += \"\\nSTEP: \" + step;\n // s += \"\\n###############################\\n\";\n // logDebug(s);\n\n // Calculate node repulsions\n calculateNodeForces(layoutInfo, options);\n // Calculate edge forces\n calculateEdgeForces(layoutInfo);\n // Calculate gravity forces\n calculateGravityForces(layoutInfo, options);\n // Propagate forces from parent to child\n propagateForces(layoutInfo);\n // Update positions based on calculated forces\n updatePositions(layoutInfo);\n};\n\n/**\n * @brief : Computes the node repulsion forces\n */\nvar calculateNodeForces = function calculateNodeForces(layoutInfo, options) {\n // Go through each of the graphs in graphSet\n // Nodes only repel each other if they belong to the same graph\n // var s = 'calculateNodeForces';\n // logDebug(s);\n for (var i = 0; i < layoutInfo.graphSet.length; i++) {\n var graph = layoutInfo.graphSet[i];\n var numNodes = graph.length;\n\n // s = \"Set: \" + graph.toString();\n // logDebug(s);\n\n // Now get all the pairs of nodes\n // Only get each pair once, (A, B) = (B, A)\n for (var j = 0; j < numNodes; j++) {\n var node1 = layoutInfo.layoutNodes[layoutInfo.idToIndex[graph[j]]];\n for (var k = j + 1; k < numNodes; k++) {\n var node2 = layoutInfo.layoutNodes[layoutInfo.idToIndex[graph[k]]];\n nodeRepulsion(node1, node2, layoutInfo, options);\n }\n }\n }\n};\nvar randomDistance = function randomDistance(max) {\n return -1 + 2 * max * Math.random();\n};\n\n/**\n * @brief : Compute the node repulsion forces between a pair of nodes\n */\nvar nodeRepulsion = function nodeRepulsion(node1, node2, layoutInfo, options) {\n // var s = \"Node repulsion. Node1: \" + node1.id + \" Node2: \" + node2.id;\n\n var cmptId1 = node1.cmptId;\n var cmptId2 = node2.cmptId;\n if (cmptId1 !== cmptId2 && !layoutInfo.isCompound) {\n return;\n }\n\n // Get direction of line connecting both node centers\n var directionX = node2.positionX - node1.positionX;\n var directionY = node2.positionY - node1.positionY;\n var maxRandDist = 1;\n // s += \"\\ndirectionX: \" + directionX + \", directionY: \" + directionY;\n\n // If both centers are the same, apply a random force\n if (0 === directionX && 0 === directionY) {\n directionX = randomDistance(maxRandDist);\n directionY = randomDistance(maxRandDist);\n }\n var overlap = nodesOverlap(node1, node2, directionX, directionY);\n if (overlap > 0) {\n // s += \"\\nNodes DO overlap.\";\n // s += \"\\nOverlap: \" + overlap;\n // If nodes overlap, repulsion force is proportional\n // to the overlap\n var force = options.nodeOverlap * overlap;\n\n // Compute the module and components of the force vector\n var distance = Math.sqrt(directionX * directionX + directionY * directionY);\n // s += \"\\nDistance: \" + distance;\n var forceX = force * directionX / distance;\n var forceY = force * directionY / distance;\n } else {\n // s += \"\\nNodes do NOT overlap.\";\n // If there's no overlap, force is inversely proportional\n // to squared distance\n\n // Get clipping points for both nodes\n var point1 = findClippingPoint(node1, directionX, directionY);\n var point2 = findClippingPoint(node2, -1 * directionX, -1 * directionY);\n\n // Use clipping points to compute distance\n var distanceX = point2.x - point1.x;\n var distanceY = point2.y - point1.y;\n var distanceSqr = distanceX * distanceX + distanceY * distanceY;\n var distance = Math.sqrt(distanceSqr);\n // s += \"\\nDistance: \" + distance;\n\n // Compute the module and components of the force vector\n var force = (node1.nodeRepulsion + node2.nodeRepulsion) / distanceSqr;\n var forceX = force * distanceX / distance;\n var forceY = force * distanceY / distance;\n }\n\n // Apply force\n if (!node1.isLocked) {\n node1.offsetX -= forceX;\n node1.offsetY -= forceY;\n }\n if (!node2.isLocked) {\n node2.offsetX += forceX;\n node2.offsetY += forceY;\n }\n\n // s += \"\\nForceX: \" + forceX + \" ForceY: \" + forceY;\n // logDebug(s);\n\n return;\n};\n\n/**\n * @brief : Determines whether two nodes overlap or not\n * @return : Amount of overlapping (0 => no overlap)\n */\nvar nodesOverlap = function nodesOverlap(node1, node2, dX, dY) {\n if (dX > 0) {\n var overlapX = node1.maxX - node2.minX;\n } else {\n var overlapX = node2.maxX - node1.minX;\n }\n if (dY > 0) {\n var overlapY = node1.maxY - node2.minY;\n } else {\n var overlapY = node2.maxY - node1.minY;\n }\n if (overlapX >= 0 && overlapY >= 0) {\n return Math.sqrt(overlapX * overlapX + overlapY * overlapY);\n } else {\n return 0;\n }\n};\n\n/**\n * @brief : Finds the point in which an edge (direction dX, dY) intersects\n * the rectangular bounding box of it's source/target node\n */\nvar findClippingPoint = function findClippingPoint(node, dX, dY) {\n // Shorcuts\n var X = node.positionX;\n var Y = node.positionY;\n var H = node.height || 1;\n var W = node.width || 1;\n var dirSlope = dY / dX;\n var nodeSlope = H / W;\n\n // var s = 'Computing clipping point of node ' + node.id +\n // \" . Height: \" + H + \", Width: \" + W +\n // \"\\nDirection \" + dX + \", \" + dY;\n //\n // Compute intersection\n var res = {};\n\n // Case: Vertical direction (up)\n if (0 === dX && 0 < dY) {\n res.x = X;\n // s += \"\\nUp direction\";\n res.y = Y + H / 2;\n return res;\n }\n\n // Case: Vertical direction (down)\n if (0 === dX && 0 > dY) {\n res.x = X;\n res.y = Y + H / 2;\n // s += \"\\nDown direction\";\n\n return res;\n }\n\n // Case: Intersects the right border\n if (0 < dX && -1 * nodeSlope <= dirSlope && dirSlope <= nodeSlope) {\n res.x = X + W / 2;\n res.y = Y + W * dY / 2 / dX;\n // s += \"\\nRightborder\";\n\n return res;\n }\n\n // Case: Intersects the left border\n if (0 > dX && -1 * nodeSlope <= dirSlope && dirSlope <= nodeSlope) {\n res.x = X - W / 2;\n res.y = Y - W * dY / 2 / dX;\n // s += \"\\nLeftborder\";\n\n return res;\n }\n\n // Case: Intersects the top border\n if (0 < dY && (dirSlope <= -1 * nodeSlope || dirSlope >= nodeSlope)) {\n res.x = X + H * dX / 2 / dY;\n res.y = Y + H / 2;\n // s += \"\\nTop border\";\n\n return res;\n }\n\n // Case: Intersects the bottom border\n if (0 > dY && (dirSlope <= -1 * nodeSlope || dirSlope >= nodeSlope)) {\n res.x = X - H * dX / 2 / dY;\n res.y = Y - H / 2;\n // s += \"\\nBottom border\";\n\n return res;\n }\n\n // s += \"\\nClipping point found at \" + res.x + \", \" + res.y;\n // logDebug(s);\n return res;\n};\n\n/**\n * @brief : Calculates all edge forces\n */\nvar calculateEdgeForces = function calculateEdgeForces(layoutInfo, options) {\n // Iterate over all edges\n for (var i = 0; i < layoutInfo.edgeSize; i++) {\n // Get edge, source & target nodes\n var edge = layoutInfo.layoutEdges[i];\n var sourceIx = layoutInfo.idToIndex[edge.sourceId];\n var source = layoutInfo.layoutNodes[sourceIx];\n var targetIx = layoutInfo.idToIndex[edge.targetId];\n var target = layoutInfo.layoutNodes[targetIx];\n\n // Get direction of line connecting both node centers\n var directionX = target.positionX - source.positionX;\n var directionY = target.positionY - source.positionY;\n\n // If both centers are the same, do nothing.\n // A random force has already been applied as node repulsion\n if (0 === directionX && 0 === directionY) {\n continue;\n }\n\n // Get clipping points for both nodes\n var point1 = findClippingPoint(source, directionX, directionY);\n var point2 = findClippingPoint(target, -1 * directionX, -1 * directionY);\n var lx = point2.x - point1.x;\n var ly = point2.y - point1.y;\n var l = Math.sqrt(lx * lx + ly * ly);\n var force = Math.pow(edge.idealLength - l, 2) / edge.elasticity;\n if (0 !== l) {\n var forceX = force * lx / l;\n var forceY = force * ly / l;\n } else {\n var forceX = 0;\n var forceY = 0;\n }\n\n // Add this force to target and source nodes\n if (!source.isLocked) {\n source.offsetX += forceX;\n source.offsetY += forceY;\n }\n if (!target.isLocked) {\n target.offsetX -= forceX;\n target.offsetY -= forceY;\n }\n\n // var s = 'Edge force between nodes ' + source.id + ' and ' + target.id;\n // s += \"\\nDistance: \" + l + \" Force: (\" + forceX + \", \" + forceY + \")\";\n // logDebug(s);\n }\n};\n\n/**\n * @brief : Computes gravity forces for all nodes\n */\nvar calculateGravityForces = function calculateGravityForces(layoutInfo, options) {\n if (options.gravity === 0) {\n return;\n }\n var distThreshold = 1;\n\n // var s = 'calculateGravityForces';\n // logDebug(s);\n for (var i = 0; i < layoutInfo.graphSet.length; i++) {\n var graph = layoutInfo.graphSet[i];\n var numNodes = graph.length;\n\n // s = \"Set: \" + graph.toString();\n // logDebug(s);\n\n // Compute graph center\n if (0 === i) {\n var centerX = layoutInfo.clientHeight / 2;\n var centerY = layoutInfo.clientWidth / 2;\n } else {\n // Get Parent node for this graph, and use its position as center\n var temp = layoutInfo.layoutNodes[layoutInfo.idToIndex[graph[0]]];\n var parent = layoutInfo.layoutNodes[layoutInfo.idToIndex[temp.parentId]];\n var centerX = parent.positionX;\n var centerY = parent.positionY;\n }\n // s = \"Center found at: \" + centerX + \", \" + centerY;\n // logDebug(s);\n\n // Apply force to all nodes in graph\n for (var j = 0; j < numNodes; j++) {\n var node = layoutInfo.layoutNodes[layoutInfo.idToIndex[graph[j]]];\n // s = \"Node: \" + node.id;\n\n if (node.isLocked) {\n continue;\n }\n var dx = centerX - node.positionX;\n var dy = centerY - node.positionY;\n var d = Math.sqrt(dx * dx + dy * dy);\n if (d > distThreshold) {\n var fx = options.gravity * dx / d;\n var fy = options.gravity * dy / d;\n node.offsetX += fx;\n node.offsetY += fy;\n // s += \": Applied force: \" + fx + \", \" + fy;\n }\n // logDebug(s);\n }\n }\n};\n\n/**\n * @brief : This function propagates the existing offsets from\n * parent nodes to its descendents.\n * @arg layoutInfo : layoutInfo Object\n * @arg cy : cytoscape Object\n * @arg options : Layout options\n */\nvar propagateForces = function propagateForces(layoutInfo, options) {\n // Inline implementation of a queue, used for traversing the graph in BFS order\n var queue = [];\n var start = 0; // Points to the start the queue\n var end = -1; // Points to the end of the queue\n\n // logDebug('propagateForces');\n\n // Start by visiting the nodes in the root graph\n queue.push.apply(queue, layoutInfo.graphSet[0]);\n end += layoutInfo.graphSet[0].length;\n\n // Traverse the graph, level by level,\n while (start <= end) {\n // Get the node to visit and remove it from queue\n var nodeId = queue[start++];\n var nodeIndex = layoutInfo.idToIndex[nodeId];\n var node = layoutInfo.layoutNodes[nodeIndex];\n var children = node.children;\n\n // We only need to process the node if it's compound\n if (0 < children.length && !node.isLocked) {\n var offX = node.offsetX;\n var offY = node.offsetY;\n\n // var s = \"Propagating offset from parent node : \" + node.id +\n // \". OffsetX: \" + offX + \". OffsetY: \" + offY;\n // s += \"\\n Children: \" + children.toString();\n // logDebug(s);\n\n for (var i = 0; i < children.length; i++) {\n var childNode = layoutInfo.layoutNodes[layoutInfo.idToIndex[children[i]]];\n // Propagate offset\n childNode.offsetX += offX;\n childNode.offsetY += offY;\n // Add children to queue to be visited\n queue[++end] = children[i];\n }\n\n // Reset parent offsets\n node.offsetX = 0;\n node.offsetY = 0;\n }\n }\n};\n\n/**\n * @brief : Updates the layout model positions, based on\n * the accumulated forces\n */\nvar updatePositions = function updatePositions(layoutInfo, options) {\n // var s = 'Updating positions';\n // logDebug(s);\n\n // Reset boundaries for compound nodes\n for (var i = 0; i < layoutInfo.nodeSize; i++) {\n var n = layoutInfo.layoutNodes[i];\n if (0 < n.children.length) {\n // logDebug(\"Resetting boundaries of compound node: \" + n.id);\n n.maxX = undefined;\n n.minX = undefined;\n n.maxY = undefined;\n n.minY = undefined;\n }\n }\n for (var i = 0; i < layoutInfo.nodeSize; i++) {\n var n = layoutInfo.layoutNodes[i];\n if (0 < n.children.length || n.isLocked) {\n // No need to set compound or locked node position\n // logDebug(\"Skipping position update of node: \" + n.id);\n continue;\n }\n // s = \"Node: \" + n.id + \" Previous position: (\" +\n // n.positionX + \", \" + n.positionY + \").\";\n\n // Limit displacement in order to improve stability\n var tempForce = limitForce(n.offsetX, n.offsetY, layoutInfo.temperature);\n n.positionX += tempForce.x;\n n.positionY += tempForce.y;\n n.offsetX = 0;\n n.offsetY = 0;\n n.minX = n.positionX - n.width;\n n.maxX = n.positionX + n.width;\n n.minY = n.positionY - n.height;\n n.maxY = n.positionY + n.height;\n // s += \" New Position: (\" + n.positionX + \", \" + n.positionY + \").\";\n // logDebug(s);\n\n // Update ancestry boudaries\n _updateAncestryBoundaries(n, layoutInfo);\n }\n\n // Update size, position of compund nodes\n for (var i = 0; i < layoutInfo.nodeSize; i++) {\n var n = layoutInfo.layoutNodes[i];\n if (0 < n.children.length && !n.isLocked) {\n n.positionX = (n.maxX + n.minX) / 2;\n n.positionY = (n.maxY + n.minY) / 2;\n n.width = n.maxX - n.minX;\n n.height = n.maxY - n.minY;\n // s = \"Updating position, size of compound node \" + n.id;\n // s += \"\\nPositionX: \" + n.positionX + \", PositionY: \" + n.positionY;\n // s += \"\\nWidth: \" + n.width + \", Height: \" + n.height;\n // logDebug(s);\n }\n }\n};\n\n/**\n * @brief : Limits a force (forceX, forceY) to be not\n * greater (in modulo) than max.\n 8 Preserves force direction.\n */\nvar limitForce = function limitForce(forceX, forceY, max) {\n // var s = \"Limiting force: (\" + forceX + \", \" + forceY + \"). Max: \" + max;\n var force = Math.sqrt(forceX * forceX + forceY * forceY);\n if (force > max) {\n var res = {\n x: max * forceX / force,\n y: max * forceY / force\n };\n } else {\n var res = {\n x: forceX,\n y: forceY\n };\n }\n\n // s += \".\\nResult: (\" + res.x + \", \" + res.y + \")\";\n // logDebug(s);\n\n return res;\n};\n\n/**\n * @brief : Function used for keeping track of compound node\n * sizes, since they should bound all their subnodes.\n */\nvar _updateAncestryBoundaries = function updateAncestryBoundaries(node, layoutInfo) {\n // var s = \"Propagating new position/size of node \" + node.id;\n var parentId = node.parentId;\n if (null == parentId) {\n // If there's no parent, we are done\n // s += \". No parent node.\";\n // logDebug(s);\n return;\n }\n\n // Get Parent Node\n var p = layoutInfo.layoutNodes[layoutInfo.idToIndex[parentId]];\n var flag = false;\n\n // MaxX\n if (null == p.maxX || node.maxX + p.padRight > p.maxX) {\n p.maxX = node.maxX + p.padRight;\n flag = true;\n // s += \"\\nNew maxX for parent node \" + p.id + \": \" + p.maxX;\n }\n\n // MinX\n if (null == p.minX || node.minX - p.padLeft < p.minX) {\n p.minX = node.minX - p.padLeft;\n flag = true;\n // s += \"\\nNew minX for parent node \" + p.id + \": \" + p.minX;\n }\n\n // MaxY\n if (null == p.maxY || node.maxY + p.padBottom > p.maxY) {\n p.maxY = node.maxY + p.padBottom;\n flag = true;\n // s += \"\\nNew maxY for parent node \" + p.id + \": \" + p.maxY;\n }\n\n // MinY\n if (null == p.minY || node.minY - p.padTop < p.minY) {\n p.minY = node.minY - p.padTop;\n flag = true;\n // s += \"\\nNew minY for parent node \" + p.id + \": \" + p.minY;\n }\n\n // If updated boundaries, propagate changes upward\n if (flag) {\n // logDebug(s);\n return _updateAncestryBoundaries(p, layoutInfo);\n }\n\n // s += \". No changes in boundaries/position of parent node \" + p.id;\n // logDebug(s);\n return;\n};\nvar separateComponents = function separateComponents(layoutInfo, options) {\n var nodes = layoutInfo.layoutNodes;\n var components = [];\n for (var i = 0; i < nodes.length; i++) {\n var node = nodes[i];\n var cid = node.cmptId;\n var component = components[cid] = components[cid] || [];\n component.push(node);\n }\n var totalA = 0;\n for (var i = 0; i < components.length; i++) {\n var c = components[i];\n if (!c) {\n continue;\n }\n c.x1 = Infinity;\n c.x2 = -Infinity;\n c.y1 = Infinity;\n c.y2 = -Infinity;\n for (var j = 0; j < c.length; j++) {\n var n = c[j];\n c.x1 = Math.min(c.x1, n.positionX - n.width / 2);\n c.x2 = Math.max(c.x2, n.positionX + n.width / 2);\n c.y1 = Math.min(c.y1, n.positionY - n.height / 2);\n c.y2 = Math.max(c.y2, n.positionY + n.height / 2);\n }\n c.w = c.x2 - c.x1;\n c.h = c.y2 - c.y1;\n totalA += c.w * c.h;\n }\n components.sort(function (c1, c2) {\n return c2.w * c2.h - c1.w * c1.h;\n });\n var x = 0;\n var y = 0;\n var usedW = 0;\n var rowH = 0;\n var maxRowW = Math.sqrt(totalA) * layoutInfo.clientWidth / layoutInfo.clientHeight;\n for (var i = 0; i < components.length; i++) {\n var c = components[i];\n if (!c) {\n continue;\n }\n for (var j = 0; j < c.length; j++) {\n var n = c[j];\n if (!n.isLocked) {\n n.positionX += x - c.x1;\n n.positionY += y - c.y1;\n }\n }\n x += c.w + options.componentSpacing;\n usedW += c.w + options.componentSpacing;\n rowH = Math.max(rowH, c.h);\n if (usedW > maxRowW) {\n y += rowH + options.componentSpacing;\n x = 0;\n usedW = 0;\n rowH = 0;\n }\n }\n};\n\nvar defaults$3 = {\n fit: true,\n // whether to fit the viewport to the graph\n padding: 30,\n // padding used on fit\n boundingBox: undefined,\n // constrain layout bounds; { x1, y1, x2, y2 } or { x1, y1, w, h }\n avoidOverlap: true,\n // prevents node overlap, may overflow boundingBox if not enough space\n avoidOverlapPadding: 10,\n // extra spacing around nodes when avoidOverlap: true\n nodeDimensionsIncludeLabels: false,\n // Excludes the label when calculating node bounding boxes for the layout algorithm\n spacingFactor: undefined,\n // Applies a multiplicative factor (>0) to expand or compress the overall area that the nodes take up\n condense: false,\n // uses all available space on false, uses minimal space on true\n rows: undefined,\n // force num of rows in the grid\n cols: undefined,\n // force num of columns in the grid\n position: function position(node) {},\n // returns { row, col } for element\n sort: undefined,\n // a sorting function to order the nodes; e.g. function(a, b){ return a.data('weight') - b.data('weight') }\n animate: false,\n // whether to transition the node positions\n animationDuration: 500,\n // duration of animation in ms if enabled\n animationEasing: undefined,\n // easing of animation if enabled\n animateFilter: function animateFilter(node, i) {\n return true;\n },\n // a function that determines whether the node should be animated. All nodes animated by default on animate enabled. Non-animated nodes are positioned immediately when the layout starts\n ready: undefined,\n // callback on layoutready\n stop: undefined,\n // callback on layoutstop\n transform: function transform(node, position) {\n return position;\n } // transform a given node position. Useful for changing flow direction in discrete layouts \n};\nfunction GridLayout(options) {\n this.options = extend({}, defaults$3, options);\n}\nGridLayout.prototype.run = function () {\n var params = this.options;\n var options = params;\n var cy = params.cy;\n var eles = options.eles;\n var nodes = eles.nodes().not(':parent');\n if (options.sort) {\n nodes = nodes.sort(options.sort);\n }\n var bb = makeBoundingBox(options.boundingBox ? options.boundingBox : {\n x1: 0,\n y1: 0,\n w: cy.width(),\n h: cy.height()\n });\n if (bb.h === 0 || bb.w === 0) {\n eles.nodes().layoutPositions(this, options, function (ele) {\n return {\n x: bb.x1,\n y: bb.y1\n };\n });\n } else {\n // width/height * splits^2 = cells where splits is number of times to split width\n var cells = nodes.size();\n var splits = Math.sqrt(cells * bb.h / bb.w);\n var rows = Math.round(splits);\n var cols = Math.round(bb.w / bb.h * splits);\n var small = function small(val) {\n if (val == null) {\n return Math.min(rows, cols);\n } else {\n var min = Math.min(rows, cols);\n if (min == rows) {\n rows = val;\n } else {\n cols = val;\n }\n }\n };\n var large = function large(val) {\n if (val == null) {\n return Math.max(rows, cols);\n } else {\n var max = Math.max(rows, cols);\n if (max == rows) {\n rows = val;\n } else {\n cols = val;\n }\n }\n };\n var oRows = options.rows;\n var oCols = options.cols != null ? options.cols : options.columns;\n\n // if rows or columns were set in options, use those values\n if (oRows != null && oCols != null) {\n rows = oRows;\n cols = oCols;\n } else if (oRows != null && oCols == null) {\n rows = oRows;\n cols = Math.ceil(cells / rows);\n } else if (oRows == null && oCols != null) {\n cols = oCols;\n rows = Math.ceil(cells / cols);\n }\n\n // otherwise use the automatic values and adjust accordingly\n\n // if rounding was up, see if we can reduce rows or columns\n else if (cols * rows > cells) {\n var sm = small();\n var lg = large();\n\n // reducing the small side takes away the most cells, so try it first\n if ((sm - 1) * lg >= cells) {\n small(sm - 1);\n } else if ((lg - 1) * sm >= cells) {\n large(lg - 1);\n }\n } else {\n // if rounding was too low, add rows or columns\n while (cols * rows < cells) {\n var _sm = small();\n var _lg = large();\n\n // try to add to larger side first (adds less in multiplication)\n if ((_lg + 1) * _sm >= cells) {\n large(_lg + 1);\n } else {\n small(_sm + 1);\n }\n }\n }\n var cellWidth = bb.w / cols;\n var cellHeight = bb.h / rows;\n if (options.condense) {\n cellWidth = 0;\n cellHeight = 0;\n }\n if (options.avoidOverlap) {\n for (var i = 0; i < nodes.length; i++) {\n var node = nodes[i];\n var pos = node._private.position;\n if (pos.x == null || pos.y == null) {\n // for bb\n pos.x = 0;\n pos.y = 0;\n }\n var nbb = node.layoutDimensions(options);\n var p = options.avoidOverlapPadding;\n var w = nbb.w + p;\n var h = nbb.h + p;\n cellWidth = Math.max(cellWidth, w);\n cellHeight = Math.max(cellHeight, h);\n }\n }\n var cellUsed = {}; // e.g. 'c-0-2' => true\n\n var used = function used(row, col) {\n return cellUsed['c-' + row + '-' + col] ? true : false;\n };\n var use = function use(row, col) {\n cellUsed['c-' + row + '-' + col] = true;\n };\n\n // to keep track of current cell position\n var row = 0;\n var col = 0;\n var moveToNextCell = function moveToNextCell() {\n col++;\n if (col >= cols) {\n col = 0;\n row++;\n }\n };\n\n // get a cache of all the manual positions\n var id2manPos = {};\n for (var _i = 0; _i < nodes.length; _i++) {\n var _node = nodes[_i];\n var rcPos = options.position(_node);\n if (rcPos && (rcPos.row !== undefined || rcPos.col !== undefined)) {\n // must have at least row or col def'd\n var _pos = {\n row: rcPos.row,\n col: rcPos.col\n };\n if (_pos.col === undefined) {\n // find unused col\n _pos.col = 0;\n while (used(_pos.row, _pos.col)) {\n _pos.col++;\n }\n } else if (_pos.row === undefined) {\n // find unused row\n _pos.row = 0;\n while (used(_pos.row, _pos.col)) {\n _pos.row++;\n }\n }\n id2manPos[_node.id()] = _pos;\n use(_pos.row, _pos.col);\n }\n }\n var getPos = function getPos(element, i) {\n var x, y;\n if (element.locked() || element.isParent()) {\n return false;\n }\n\n // see if we have a manual position set\n var rcPos = id2manPos[element.id()];\n if (rcPos) {\n x = rcPos.col * cellWidth + cellWidth / 2 + bb.x1;\n y = rcPos.row * cellHeight + cellHeight / 2 + bb.y1;\n } else {\n // otherwise set automatically\n\n while (used(row, col)) {\n moveToNextCell();\n }\n x = col * cellWidth + cellWidth / 2 + bb.x1;\n y = row * cellHeight + cellHeight / 2 + bb.y1;\n use(row, col);\n moveToNextCell();\n }\n return {\n x: x,\n y: y\n };\n };\n nodes.layoutPositions(this, options, getPos);\n }\n return this; // chaining\n};\n\n// default layout options\nvar defaults$2 = {\n ready: function ready() {},\n // on layoutready\n stop: function stop() {} // on layoutstop\n};\n\n// constructor\n// options : object containing layout options\nfunction NullLayout(options) {\n this.options = extend({}, defaults$2, options);\n}\n\n// runs the layout\nNullLayout.prototype.run = function () {\n var options = this.options;\n var eles = options.eles; // elements to consider in the layout\n var layout = this;\n\n // cy is automatically populated for us in the constructor\n // (disable eslint for next line as this serves as example layout code to external developers)\n // eslint-disable-next-line no-unused-vars\n options.cy;\n layout.emit('layoutstart');\n\n // puts all nodes at (0, 0)\n // n.b. most layouts would use layoutPositions(), instead of positions() and manual events\n eles.nodes().positions(function () {\n return {\n x: 0,\n y: 0\n };\n });\n\n // trigger layoutready when each node has had its position set at least once\n layout.one('layoutready', options.ready);\n layout.emit('layoutready');\n\n // trigger layoutstop when the layout stops (e.g. finishes)\n layout.one('layoutstop', options.stop);\n layout.emit('layoutstop');\n return this; // chaining\n};\n\n// called on continuous layouts to stop them before they finish\nNullLayout.prototype.stop = function () {\n return this; // chaining\n};\n\nvar defaults$1 = {\n positions: undefined,\n // map of (node id) => (position obj); or function(node){ return somPos; }\n zoom: undefined,\n // the zoom level to set (prob want fit = false if set)\n pan: undefined,\n // the pan level to set (prob want fit = false if set)\n fit: true,\n // whether to fit to viewport\n padding: 30,\n // padding on fit\n spacingFactor: undefined,\n // Applies a multiplicative factor (>0) to expand or compress the overall area that the nodes take up\n animate: false,\n // whether to transition the node positions\n animationDuration: 500,\n // duration of animation in ms if enabled\n animationEasing: undefined,\n // easing of animation if enabled\n animateFilter: function animateFilter(node, i) {\n return true;\n },\n // a function that determines whether the node should be animated. All nodes animated by default on animate enabled. Non-animated nodes are positioned immediately when the layout starts\n ready: undefined,\n // callback on layoutready\n stop: undefined,\n // callback on layoutstop\n transform: function transform(node, position) {\n return position;\n } // transform a given node position. Useful for changing flow direction in discrete layouts\n};\nfunction PresetLayout(options) {\n this.options = extend({}, defaults$1, options);\n}\nPresetLayout.prototype.run = function () {\n var options = this.options;\n var eles = options.eles;\n var nodes = eles.nodes();\n var posIsFn = fn$6(options.positions);\n function getPosition(node) {\n if (options.positions == null) {\n return copyPosition(node.position());\n }\n if (posIsFn) {\n return options.positions(node);\n }\n var pos = options.positions[node._private.data.id];\n if (pos == null) {\n return null;\n }\n return pos;\n }\n nodes.layoutPositions(this, options, function (node, i) {\n var position = getPosition(node);\n if (node.locked() || position == null) {\n return false;\n }\n return position;\n });\n return this; // chaining\n};\n\nvar defaults = {\n fit: true,\n // whether to fit to viewport\n padding: 30,\n // fit padding\n boundingBox: undefined,\n // constrain layout bounds; { x1, y1, x2, y2 } or { x1, y1, w, h }\n animate: false,\n // whether to transition the node positions\n animationDuration: 500,\n // duration of animation in ms if enabled\n animationEasing: undefined,\n // easing of animation if enabled\n animateFilter: function animateFilter(node, i) {\n return true;\n },\n // a function that determines whether the node should be animated. All nodes animated by default on animate enabled. Non-animated nodes are positioned immediately when the layout starts\n ready: undefined,\n // callback on layoutready\n stop: undefined,\n // callback on layoutstop\n transform: function transform(node, position) {\n return position;\n } // transform a given node position. Useful for changing flow direction in discrete layouts \n};\nfunction RandomLayout(options) {\n this.options = extend({}, defaults, options);\n}\nRandomLayout.prototype.run = function () {\n var options = this.options;\n var cy = options.cy;\n var eles = options.eles;\n var bb = makeBoundingBox(options.boundingBox ? options.boundingBox : {\n x1: 0,\n y1: 0,\n w: cy.width(),\n h: cy.height()\n });\n var getPos = function getPos(node, i) {\n return {\n x: bb.x1 + Math.round(Math.random() * bb.w),\n y: bb.y1 + Math.round(Math.random() * bb.h)\n };\n };\n eles.nodes().layoutPositions(this, options, getPos);\n return this; // chaining\n};\n\nvar layout = [{\n name: 'breadthfirst',\n impl: BreadthFirstLayout\n}, {\n name: 'circle',\n impl: CircleLayout\n}, {\n name: 'concentric',\n impl: ConcentricLayout\n}, {\n name: 'cose',\n impl: CoseLayout\n}, {\n name: 'grid',\n impl: GridLayout\n}, {\n name: 'null',\n impl: NullLayout\n}, {\n name: 'preset',\n impl: PresetLayout\n}, {\n name: 'random',\n impl: RandomLayout\n}];\n\nfunction NullRenderer(options) {\n this.options = options;\n this.notifications = 0; // for testing\n}\nvar noop = function noop() {};\nvar throwImgErr = function throwImgErr() {\n throw new Error('A headless instance can not render images');\n};\nNullRenderer.prototype = {\n recalculateRenderedStyle: noop,\n notify: function notify() {\n this.notifications++;\n },\n init: noop,\n isHeadless: function isHeadless() {\n return true;\n },\n png: throwImgErr,\n jpg: throwImgErr\n};\n\nvar BRp$f = {};\nBRp$f.arrowShapeWidth = 0.3;\nBRp$f.registerArrowShapes = function () {\n var arrowShapes = this.arrowShapes = {};\n var renderer = this;\n\n // Contract for arrow shapes:\n // 0, 0 is arrow tip\n // (0, 1) is direction towards node\n // (1, 0) is right\n //\n // functional api:\n // collide: check x, y in shape\n // roughCollide: called before collide, no false negatives\n // draw: draw\n // spacing: dist(arrowTip, nodeBoundary)\n // gap: dist(edgeTip, nodeBoundary), edgeTip may != arrowTip\n\n var bbCollide = function bbCollide(x, y, size, angle, translation, edgeWidth, padding) {\n var x1 = translation.x - size / 2 - padding;\n var x2 = translation.x + size / 2 + padding;\n var y1 = translation.y - size / 2 - padding;\n var y2 = translation.y + size / 2 + padding;\n var inside = x1 <= x && x <= x2 && y1 <= y && y <= y2;\n return inside;\n };\n var transform = function transform(x, y, size, angle, translation) {\n var xRotated = x * Math.cos(angle) - y * Math.sin(angle);\n var yRotated = x * Math.sin(angle) + y * Math.cos(angle);\n var xScaled = xRotated * size;\n var yScaled = yRotated * size;\n var xTranslated = xScaled + translation.x;\n var yTranslated = yScaled + translation.y;\n return {\n x: xTranslated,\n y: yTranslated\n };\n };\n var transformPoints = function transformPoints(pts, size, angle, translation) {\n var retPts = [];\n for (var i = 0; i < pts.length; i += 2) {\n var x = pts[i];\n var y = pts[i + 1];\n retPts.push(transform(x, y, size, angle, translation));\n }\n return retPts;\n };\n var pointsToArr = function pointsToArr(pts) {\n var ret = [];\n for (var i = 0; i < pts.length; i++) {\n var p = pts[i];\n ret.push(p.x, p.y);\n }\n return ret;\n };\n var standardGap = function standardGap(edge) {\n return edge.pstyle('width').pfValue * edge.pstyle('arrow-scale').pfValue * 2;\n };\n var defineArrowShape = function defineArrowShape(name, defn) {\n if (string(defn)) {\n defn = arrowShapes[defn];\n }\n arrowShapes[name] = extend({\n name: name,\n points: [-0.15, -0.3, 0.15, -0.3, 0.15, 0.3, -0.15, 0.3],\n collide: function collide(x, y, size, angle, translation, padding) {\n var points = pointsToArr(transformPoints(this.points, size + 2 * padding, angle, translation));\n var inside = pointInsidePolygonPoints(x, y, points);\n return inside;\n },\n roughCollide: bbCollide,\n draw: function draw(context, size, angle, translation) {\n var points = transformPoints(this.points, size, angle, translation);\n renderer.arrowShapeImpl('polygon')(context, points);\n },\n spacing: function spacing(edge) {\n return 0;\n },\n gap: standardGap\n }, defn);\n };\n defineArrowShape('none', {\n collide: falsify,\n roughCollide: falsify,\n draw: noop$1,\n spacing: zeroify,\n gap: zeroify\n });\n defineArrowShape('triangle', {\n points: [-0.15, -0.3, 0, 0, 0.15, -0.3]\n });\n defineArrowShape('arrow', 'triangle');\n defineArrowShape('triangle-backcurve', {\n points: arrowShapes['triangle'].points,\n controlPoint: [0, -0.15],\n roughCollide: bbCollide,\n draw: function draw(context, size, angle, translation, edgeWidth) {\n var ptsTrans = transformPoints(this.points, size, angle, translation);\n var ctrlPt = this.controlPoint;\n var ctrlPtTrans = transform(ctrlPt[0], ctrlPt[1], size, angle, translation);\n renderer.arrowShapeImpl(this.name)(context, ptsTrans, ctrlPtTrans);\n },\n gap: function gap(edge) {\n return standardGap(edge) * 0.8;\n }\n });\n defineArrowShape('triangle-tee', {\n points: [0, 0, 0.15, -0.3, -0.15, -0.3, 0, 0],\n pointsTee: [-0.15, -0.4, -0.15, -0.5, 0.15, -0.5, 0.15, -0.4],\n collide: function collide(x, y, size, angle, translation, edgeWidth, padding) {\n var triPts = pointsToArr(transformPoints(this.points, size + 2 * padding, angle, translation));\n var teePts = pointsToArr(transformPoints(this.pointsTee, size + 2 * padding, angle, translation));\n var inside = pointInsidePolygonPoints(x, y, triPts) || pointInsidePolygonPoints(x, y, teePts);\n return inside;\n },\n draw: function draw(context, size, angle, translation, edgeWidth) {\n var triPts = transformPoints(this.points, size, angle, translation);\n var teePts = transformPoints(this.pointsTee, size, angle, translation);\n renderer.arrowShapeImpl(this.name)(context, triPts, teePts);\n }\n });\n defineArrowShape('circle-triangle', {\n radius: 0.15,\n pointsTr: [0, -0.15, 0.15, -0.45, -0.15, -0.45, 0, -0.15],\n collide: function collide(x, y, size, angle, translation, edgeWidth, padding) {\n var t = translation;\n var circleInside = Math.pow(t.x - x, 2) + Math.pow(t.y - y, 2) <= Math.pow((size + 2 * padding) * this.radius, 2);\n var triPts = pointsToArr(transformPoints(this.points, size + 2 * padding, angle, translation));\n return pointInsidePolygonPoints(x, y, triPts) || circleInside;\n },\n draw: function draw(context, size, angle, translation, edgeWidth) {\n var triPts = transformPoints(this.pointsTr, size, angle, translation);\n renderer.arrowShapeImpl(this.name)(context, triPts, translation.x, translation.y, this.radius * size);\n },\n spacing: function spacing(edge) {\n return renderer.getArrowWidth(edge.pstyle('width').pfValue, edge.pstyle('arrow-scale').value) * this.radius;\n }\n });\n defineArrowShape('triangle-cross', {\n points: [0, 0, 0.15, -0.3, -0.15, -0.3, 0, 0],\n baseCrossLinePts: [-0.15, -0.4,\n // first half of the rectangle\n -0.15, -0.4, 0.15, -0.4,\n // second half of the rectangle\n 0.15, -0.4],\n crossLinePts: function crossLinePts(size, edgeWidth) {\n // shift points so that the distance between the cross points matches edge width\n var p = this.baseCrossLinePts.slice();\n var shiftFactor = edgeWidth / size;\n var y0 = 3;\n var y1 = 5;\n p[y0] = p[y0] - shiftFactor;\n p[y1] = p[y1] - shiftFactor;\n return p;\n },\n collide: function collide(x, y, size, angle, translation, edgeWidth, padding) {\n var triPts = pointsToArr(transformPoints(this.points, size + 2 * padding, angle, translation));\n var teePts = pointsToArr(transformPoints(this.crossLinePts(size, edgeWidth), size + 2 * padding, angle, translation));\n var inside = pointInsidePolygonPoints(x, y, triPts) || pointInsidePolygonPoints(x, y, teePts);\n return inside;\n },\n draw: function draw(context, size, angle, translation, edgeWidth) {\n var triPts = transformPoints(this.points, size, angle, translation);\n var crossLinePts = transformPoints(this.crossLinePts(size, edgeWidth), size, angle, translation);\n renderer.arrowShapeImpl(this.name)(context, triPts, crossLinePts);\n }\n });\n defineArrowShape('vee', {\n points: [-0.15, -0.3, 0, 0, 0.15, -0.3, 0, -0.15],\n gap: function gap(edge) {\n return standardGap(edge) * 0.525;\n }\n });\n defineArrowShape('circle', {\n radius: 0.15,\n collide: function collide(x, y, size, angle, translation, edgeWidth, padding) {\n var t = translation;\n var inside = Math.pow(t.x - x, 2) + Math.pow(t.y - y, 2) <= Math.pow((size + 2 * padding) * this.radius, 2);\n return inside;\n },\n draw: function draw(context, size, angle, translation, edgeWidth) {\n renderer.arrowShapeImpl(this.name)(context, translation.x, translation.y, this.radius * size);\n },\n spacing: function spacing(edge) {\n return renderer.getArrowWidth(edge.pstyle('width').pfValue, edge.pstyle('arrow-scale').value) * this.radius;\n }\n });\n defineArrowShape('tee', {\n points: [-0.15, 0, -0.15, -0.1, 0.15, -0.1, 0.15, 0],\n spacing: function spacing(edge) {\n return 1;\n },\n gap: function gap(edge) {\n return 1;\n }\n });\n defineArrowShape('square', {\n points: [-0.15, 0.00, 0.15, 0.00, 0.15, -0.3, -0.15, -0.3]\n });\n defineArrowShape('diamond', {\n points: [-0.15, -0.15, 0, -0.3, 0.15, -0.15, 0, 0],\n gap: function gap(edge) {\n return edge.pstyle('width').pfValue * edge.pstyle('arrow-scale').value;\n }\n });\n defineArrowShape('chevron', {\n points: [0, 0, -0.15, -0.15, -0.1, -0.2, 0, -0.1, 0.1, -0.2, 0.15, -0.15],\n gap: function gap(edge) {\n return 0.95 * edge.pstyle('width').pfValue * edge.pstyle('arrow-scale').value;\n }\n });\n};\n\nvar BRp$e = {};\n\n// Project mouse\nBRp$e.projectIntoViewport = function (clientX, clientY) {\n var cy = this.cy;\n var offsets = this.findContainerClientCoords();\n var offsetLeft = offsets[0];\n var offsetTop = offsets[1];\n var scale = offsets[4];\n var pan = cy.pan();\n var zoom = cy.zoom();\n var x = ((clientX - offsetLeft) / scale - pan.x) / zoom;\n var y = ((clientY - offsetTop) / scale - pan.y) / zoom;\n return [x, y];\n};\nBRp$e.findContainerClientCoords = function () {\n if (this.containerBB) {\n return this.containerBB;\n }\n var container = this.container;\n var rect = container.getBoundingClientRect();\n var style = this.cy.window().getComputedStyle(container);\n var styleValue = function styleValue(name) {\n return parseFloat(style.getPropertyValue(name));\n };\n var padding = {\n left: styleValue('padding-left'),\n right: styleValue('padding-right'),\n top: styleValue('padding-top'),\n bottom: styleValue('padding-bottom')\n };\n var border = {\n left: styleValue('border-left-width'),\n right: styleValue('border-right-width'),\n top: styleValue('border-top-width'),\n bottom: styleValue('border-bottom-width')\n };\n var clientWidth = container.clientWidth;\n var clientHeight = container.clientHeight;\n var paddingHor = padding.left + padding.right;\n var paddingVer = padding.top + padding.bottom;\n var borderHor = border.left + border.right;\n var scale = rect.width / (clientWidth + borderHor);\n var unscaledW = clientWidth - paddingHor;\n var unscaledH = clientHeight - paddingVer;\n var left = rect.left + padding.left + border.left;\n var top = rect.top + padding.top + border.top;\n return this.containerBB = [left, top, unscaledW, unscaledH, scale];\n};\nBRp$e.invalidateContainerClientCoordsCache = function () {\n this.containerBB = null;\n};\nBRp$e.findNearestElement = function (x, y, interactiveElementsOnly, isTouch) {\n return this.findNearestElements(x, y, interactiveElementsOnly, isTouch)[0];\n};\nBRp$e.findNearestElements = function (x, y, interactiveElementsOnly, isTouch) {\n var self = this;\n var r = this;\n var eles = r.getCachedZSortedEles();\n var near = []; // 1 node max, 1 edge max\n var zoom = r.cy.zoom();\n var hasCompounds = r.cy.hasCompoundNodes();\n var edgeThreshold = (isTouch ? 24 : 8) / zoom;\n var nodeThreshold = (isTouch ? 8 : 2) / zoom;\n var labelThreshold = (isTouch ? 8 : 2) / zoom;\n var minSqDist = Infinity;\n var nearEdge;\n var nearNode;\n if (interactiveElementsOnly) {\n eles = eles.interactive;\n }\n function addEle(ele, sqDist) {\n if (ele.isNode()) {\n if (nearNode) {\n return; // can't replace node\n } else {\n nearNode = ele;\n near.push(ele);\n }\n }\n if (ele.isEdge() && (sqDist == null || sqDist < minSqDist)) {\n if (nearEdge) {\n // then replace existing edge\n // can replace only if same z-index\n if (nearEdge.pstyle('z-compound-depth').value === ele.pstyle('z-compound-depth').value && nearEdge.pstyle('z-compound-depth').value === ele.pstyle('z-compound-depth').value) {\n for (var i = 0; i < near.length; i++) {\n if (near[i].isEdge()) {\n near[i] = ele;\n nearEdge = ele;\n minSqDist = sqDist != null ? sqDist : minSqDist;\n break;\n }\n }\n }\n } else {\n near.push(ele);\n nearEdge = ele;\n minSqDist = sqDist != null ? sqDist : minSqDist;\n }\n }\n }\n function checkNode(node) {\n var width = node.outerWidth() + 2 * nodeThreshold;\n var height = node.outerHeight() + 2 * nodeThreshold;\n var hw = width / 2;\n var hh = height / 2;\n var pos = node.position();\n var cornerRadius = node.pstyle('corner-radius').value === 'auto' ? 'auto' : node.pstyle('corner-radius').pfValue;\n var rs = node._private.rscratch;\n if (pos.x - hw <= x && x <= pos.x + hw // bb check x\n && pos.y - hh <= y && y <= pos.y + hh // bb check y\n ) {\n var shape = r.nodeShapes[self.getNodeShape(node)];\n if (shape.checkPoint(x, y, 0, width, height, pos.x, pos.y, cornerRadius, rs)) {\n addEle(node, 0);\n return true;\n }\n }\n }\n function checkEdge(edge) {\n var _p = edge._private;\n var rs = _p.rscratch;\n var styleWidth = edge.pstyle('width').pfValue;\n var scale = edge.pstyle('arrow-scale').value;\n var width = styleWidth / 2 + edgeThreshold; // more like a distance radius from centre\n var widthSq = width * width;\n var width2 = width * 2;\n var src = _p.source;\n var tgt = _p.target;\n var sqDist;\n if (rs.edgeType === 'segments' || rs.edgeType === 'straight' || rs.edgeType === 'haystack') {\n var pts = rs.allpts;\n for (var i = 0; i + 3 < pts.length; i += 2) {\n if (inLineVicinity(x, y, pts[i], pts[i + 1], pts[i + 2], pts[i + 3], width2) && widthSq > (sqDist = sqdistToFiniteLine(x, y, pts[i], pts[i + 1], pts[i + 2], pts[i + 3]))) {\n addEle(edge, sqDist);\n return true;\n }\n }\n } else if (rs.edgeType === 'bezier' || rs.edgeType === 'multibezier' || rs.edgeType === 'self' || rs.edgeType === 'compound') {\n var pts = rs.allpts;\n for (var i = 0; i + 5 < rs.allpts.length; i += 4) {\n if (inBezierVicinity(x, y, pts[i], pts[i + 1], pts[i + 2], pts[i + 3], pts[i + 4], pts[i + 5], width2) && widthSq > (sqDist = sqdistToQuadraticBezier(x, y, pts[i], pts[i + 1], pts[i + 2], pts[i + 3], pts[i + 4], pts[i + 5]))) {\n addEle(edge, sqDist);\n return true;\n }\n }\n }\n\n // if we're close to the edge but didn't hit it, maybe we hit its arrows\n\n var src = src || _p.source;\n var tgt = tgt || _p.target;\n var arSize = self.getArrowWidth(styleWidth, scale);\n var arrows = [{\n name: 'source',\n x: rs.arrowStartX,\n y: rs.arrowStartY,\n angle: rs.srcArrowAngle\n }, {\n name: 'target',\n x: rs.arrowEndX,\n y: rs.arrowEndY,\n angle: rs.tgtArrowAngle\n }, {\n name: 'mid-source',\n x: rs.midX,\n y: rs.midY,\n angle: rs.midsrcArrowAngle\n }, {\n name: 'mid-target',\n x: rs.midX,\n y: rs.midY,\n angle: rs.midtgtArrowAngle\n }];\n for (var i = 0; i < arrows.length; i++) {\n var ar = arrows[i];\n var shape = r.arrowShapes[edge.pstyle(ar.name + '-arrow-shape').value];\n var edgeWidth = edge.pstyle('width').pfValue;\n if (shape.roughCollide(x, y, arSize, ar.angle, {\n x: ar.x,\n y: ar.y\n }, edgeWidth, edgeThreshold) && shape.collide(x, y, arSize, ar.angle, {\n x: ar.x,\n y: ar.y\n }, edgeWidth, edgeThreshold)) {\n addEle(edge);\n return true;\n }\n }\n\n // for compound graphs, hitting edge may actually want a connected node instead (b/c edge may have greater z-index precedence)\n if (hasCompounds && near.length > 0) {\n checkNode(src);\n checkNode(tgt);\n }\n }\n function preprop(obj, name, pre) {\n return getPrefixedProperty(obj, name, pre);\n }\n function checkLabel(ele, prefix) {\n var _p = ele._private;\n var th = labelThreshold;\n var prefixDash;\n if (prefix) {\n prefixDash = prefix + '-';\n } else {\n prefixDash = '';\n }\n ele.boundingBox();\n var bb = _p.labelBounds[prefix || 'main'];\n var text = ele.pstyle(prefixDash + 'label').value;\n var eventsEnabled = ele.pstyle('text-events').strValue === 'yes';\n if (!eventsEnabled || !text) {\n return;\n }\n var lx = preprop(_p.rscratch, 'labelX', prefix);\n var ly = preprop(_p.rscratch, 'labelY', prefix);\n var theta = preprop(_p.rscratch, 'labelAngle', prefix);\n var ox = ele.pstyle(prefixDash + 'text-margin-x').pfValue;\n var oy = ele.pstyle(prefixDash + 'text-margin-y').pfValue;\n var lx1 = bb.x1 - th - ox; // (-ox, -oy) as bb already includes margin\n var lx2 = bb.x2 + th - ox; // and rotation is about (lx, ly)\n var ly1 = bb.y1 - th - oy;\n var ly2 = bb.y2 + th - oy;\n if (theta) {\n var cos = Math.cos(theta);\n var sin = Math.sin(theta);\n var rotate = function rotate(x, y) {\n x = x - lx;\n y = y - ly;\n return {\n x: x * cos - y * sin + lx,\n y: x * sin + y * cos + ly\n };\n };\n var px1y1 = rotate(lx1, ly1);\n var px1y2 = rotate(lx1, ly2);\n var px2y1 = rotate(lx2, ly1);\n var px2y2 = rotate(lx2, ly2);\n var points = [\n // with the margin added after the rotation is applied\n px1y1.x + ox, px1y1.y + oy, px2y1.x + ox, px2y1.y + oy, px2y2.x + ox, px2y2.y + oy, px1y2.x + ox, px1y2.y + oy];\n if (pointInsidePolygonPoints(x, y, points)) {\n addEle(ele);\n return true;\n }\n } else {\n // do a cheaper bb check\n if (inBoundingBox(bb, x, y)) {\n addEle(ele);\n return true;\n }\n }\n }\n for (var i = eles.length - 1; i >= 0; i--) {\n // reverse order for precedence\n var ele = eles[i];\n if (ele.isNode()) {\n checkNode(ele) || checkLabel(ele);\n } else {\n // then edge\n checkEdge(ele) || checkLabel(ele) || checkLabel(ele, 'source') || checkLabel(ele, 'target');\n }\n }\n return near;\n};\n\n// 'Give me everything from this box'\nBRp$e.getAllInBox = function (x1, y1, x2, y2) {\n var eles = this.getCachedZSortedEles().interactive;\n var zoom = this.cy.zoom();\n var labelThreshold = 2 / zoom;\n var box = [];\n var x1c = Math.min(x1, x2);\n var x2c = Math.max(x1, x2);\n var y1c = Math.min(y1, y2);\n var y2c = Math.max(y1, y2);\n x1 = x1c;\n x2 = x2c;\n y1 = y1c;\n y2 = y2c;\n var boxBb = makeBoundingBox({\n x1: x1,\n y1: y1,\n x2: x2,\n y2: y2\n });\n var selectionBox = [{\n x: boxBb.x1,\n y: boxBb.y1\n }, {\n x: boxBb.x2,\n y: boxBb.y1\n }, {\n x: boxBb.x2,\n y: boxBb.y2\n }, {\n x: boxBb.x1,\n y: boxBb.y2\n }];\n var boxEdges = [[selectionBox[0], selectionBox[1]], [selectionBox[1], selectionBox[2]], [selectionBox[2], selectionBox[3]], [selectionBox[3], selectionBox[0]]];\n function preprop(obj, name, pre) {\n return getPrefixedProperty(obj, name, pre);\n }\n function getRotatedLabelBox(ele, prefix) {\n var _p = ele._private;\n var th = labelThreshold;\n var prefixDash = '';\n ele.boundingBox();\n var bb = _p.labelBounds['main'];\n\n // If the bounding box is not available, return null.\n // This indicates that the label box cannot be calculated, which is consistent\n // with the expected behavior of this function. Returning null allows the caller\n // to handle the absence of a bounding box explicitly.\n if (!bb) {\n return null;\n }\n var lx = preprop(_p.rscratch, 'labelX', prefix);\n var ly = preprop(_p.rscratch, 'labelY', prefix);\n var theta = preprop(_p.rscratch, 'labelAngle', prefix);\n var ox = ele.pstyle(prefixDash + 'text-margin-x').pfValue;\n var oy = ele.pstyle(prefixDash + 'text-margin-y').pfValue;\n var lx1 = bb.x1 - th - ox;\n var lx2 = bb.x2 + th - ox;\n var ly1 = bb.y1 - th - oy;\n var ly2 = bb.y2 + th - oy;\n if (theta) {\n var cos = Math.cos(theta);\n var sin = Math.sin(theta);\n var rotate = function rotate(x, y) {\n x = x - lx;\n y = y - ly;\n return {\n x: x * cos - y * sin + lx,\n y: x * sin + y * cos + ly\n };\n };\n return [rotate(lx1, ly1), rotate(lx2, ly1), rotate(lx2, ly2), rotate(lx1, ly2)];\n } else {\n return [{\n x: lx1,\n y: ly1\n }, {\n x: lx2,\n y: ly1\n }, {\n x: lx2,\n y: ly2\n }, {\n x: lx1,\n y: ly2\n }];\n }\n }\n function doLinesIntersect(p1, p2, q1, q2) {\n function ccw(a, b, c) {\n return (c.y - a.y) * (b.x - a.x) > (b.y - a.y) * (c.x - a.x);\n }\n return ccw(p1, q1, q2) !== ccw(p2, q1, q2) && ccw(p1, p2, q1) !== ccw(p1, p2, q2);\n }\n for (var e = 0; e < eles.length; e++) {\n var ele = eles[e];\n if (ele.isNode()) {\n var node = ele;\n var textEvents = node.pstyle('text-events').strValue === 'yes';\n var nodeBoxSelectMode = node.pstyle('box-selection').strValue;\n var labelBoxSelectEnabled = node.pstyle('box-select-labels').strValue === 'yes';\n if (nodeBoxSelectMode === 'none') {\n continue;\n }\n var includeLabels = (nodeBoxSelectMode === 'overlap' || labelBoxSelectEnabled) && textEvents;\n var nodeBb = node.boundingBox({\n includeNodes: true,\n includeEdges: false,\n includeLabels: includeLabels\n });\n if (nodeBoxSelectMode === 'contain') {\n var selected = false;\n if (labelBoxSelectEnabled && textEvents) {\n var rotatedLabelBox = getRotatedLabelBox(node);\n if (rotatedLabelBox && satPolygonIntersection(rotatedLabelBox, selectionBox)) {\n box.push(node);\n selected = true;\n }\n }\n if (!selected && boundingBoxInBoundingBox(boxBb, nodeBb)) {\n box.push(node);\n }\n } else if (nodeBoxSelectMode === 'overlap') {\n if (boundingBoxesIntersect(boxBb, nodeBb)) {\n var nodeBodyBb = node.boundingBox({\n includeNodes: true,\n includeEdges: true,\n includeLabels: false,\n includeMainLabels: false,\n includeSourceLabels: false,\n includeTargetLabels: false\n });\n var nodeBodyCorners = [{\n x: nodeBodyBb.x1,\n y: nodeBodyBb.y1\n }, {\n x: nodeBodyBb.x2,\n y: nodeBodyBb.y1\n }, {\n x: nodeBodyBb.x2,\n y: nodeBodyBb.y2\n }, {\n x: nodeBodyBb.x1,\n y: nodeBodyBb.y2\n }];\n\n // if node body intersects, no need to check label\n if (satPolygonIntersection(nodeBodyCorners, selectionBox)) {\n box.push(node);\n } else {\n // only check label if node body didn't intersect\n var _rotatedLabelBox = getRotatedLabelBox(node);\n if (_rotatedLabelBox && satPolygonIntersection(_rotatedLabelBox, selectionBox)) {\n box.push(node);\n }\n }\n }\n }\n } else {\n var edge = ele;\n var _p = edge._private;\n var rs = _p.rscratch;\n var edgeBoxSelectMode = edge.pstyle('box-selection').strValue;\n if (edgeBoxSelectMode === 'none') {\n continue;\n }\n if (edgeBoxSelectMode === 'contain') {\n if (rs.startX != null && rs.startY != null && !inBoundingBox(boxBb, rs.startX, rs.startY)) {\n continue;\n }\n if (rs.endX != null && rs.endY != null && !inBoundingBox(boxBb, rs.endX, rs.endY)) {\n continue;\n }\n if (rs.edgeType === 'bezier' || rs.edgeType === 'multibezier' || rs.edgeType === 'self' || rs.edgeType === 'compound' || rs.edgeType === 'segments' || rs.edgeType === 'haystack') {\n var pts = _p.rstyle.bezierPts || _p.rstyle.linePts || _p.rstyle.haystackPts;\n var allInside = true;\n for (var i = 0; i < pts.length; i++) {\n if (!pointInBoundingBox(boxBb, pts[i])) {\n allInside = false;\n break;\n }\n }\n if (allInside) {\n box.push(edge);\n }\n } else if (rs.edgeType === 'straight') {\n box.push(edge);\n }\n } else if (edgeBoxSelectMode === 'overlap') {\n var _selected = false;\n\n // Check: either endpoint inside box\n if (rs.startX != null && rs.startY != null && rs.endX != null && rs.endY != null && (inBoundingBox(boxBb, rs.startX, rs.startY) || inBoundingBox(boxBb, rs.endX, rs.endY))) {\n box.push(edge);\n _selected = true;\n }\n\n // Haystack fallback (only check if not already selected)\n else if (!_selected && rs.edgeType === 'haystack') {\n var haystackPts = _p.rstyle.haystackPts;\n for (var _i = 0; _i < haystackPts.length; _i++) {\n if (pointInBoundingBox(boxBb, haystackPts[_i])) {\n box.push(edge);\n _selected = true;\n break;\n }\n }\n }\n\n // Segment intersection check (only if not already selected)\n if (!_selected) {\n var _pts = _p.rstyle.bezierPts || _p.rstyle.linePts || _p.rstyle.haystackPts;\n\n // straight edges\n if ((!_pts || _pts.length < 2) && rs.edgeType === 'straight') {\n if (rs.startX != null && rs.startY != null && rs.endX != null && rs.endY != null) {\n _pts = [{\n x: rs.startX,\n y: rs.startY\n }, {\n x: rs.endX,\n y: rs.endY\n }];\n }\n }\n if (!_pts || _pts.length < 2) continue;\n for (var _i2 = 0; _i2 < _pts.length - 1; _i2++) {\n var segStart = _pts[_i2];\n var segEnd = _pts[_i2 + 1];\n for (var b = 0; b < boxEdges.length; b++) {\n var _boxEdges$b = _slicedToArray(boxEdges[b], 2),\n boxStart = _boxEdges$b[0],\n boxEnd = _boxEdges$b[1];\n if (doLinesIntersect(segStart, segEnd, boxStart, boxEnd)) {\n box.push(edge);\n _selected = true;\n break;\n }\n }\n if (_selected) break;\n }\n }\n }\n }\n }\n return box;\n};\n\nvar BRp$d = {};\nBRp$d.calculateArrowAngles = function (edge) {\n var rs = edge._private.rscratch;\n var isHaystack = rs.edgeType === 'haystack';\n var isBezier = rs.edgeType === 'bezier';\n var isMultibezier = rs.edgeType === 'multibezier';\n var isSegments = rs.edgeType === 'segments';\n var isCompound = rs.edgeType === 'compound';\n var isSelf = rs.edgeType === 'self';\n\n // Displacement gives direction for arrowhead orientation\n var dispX, dispY;\n var startX, startY, endX, endY, midX, midY;\n if (isHaystack) {\n startX = rs.haystackPts[0];\n startY = rs.haystackPts[1];\n endX = rs.haystackPts[2];\n endY = rs.haystackPts[3];\n } else {\n startX = rs.arrowStartX;\n startY = rs.arrowStartY;\n endX = rs.arrowEndX;\n endY = rs.arrowEndY;\n }\n midX = rs.midX;\n midY = rs.midY;\n\n // source\n //\n\n if (isSegments) {\n dispX = startX - rs.segpts[0];\n dispY = startY - rs.segpts[1];\n } else if (isMultibezier || isCompound || isSelf || isBezier) {\n var pts = rs.allpts;\n var bX = qbezierAt(pts[0], pts[2], pts[4], 0.1);\n var bY = qbezierAt(pts[1], pts[3], pts[5], 0.1);\n dispX = startX - bX;\n dispY = startY - bY;\n } else {\n dispX = startX - midX;\n dispY = startY - midY;\n }\n rs.srcArrowAngle = getAngleFromDisp(dispX, dispY);\n\n // mid target\n //\n\n var midX = rs.midX;\n var midY = rs.midY;\n if (isHaystack) {\n midX = (startX + endX) / 2;\n midY = (startY + endY) / 2;\n }\n dispX = endX - startX;\n dispY = endY - startY;\n if (isSegments) {\n var pts = rs.allpts;\n if (pts.length / 2 % 2 === 0) {\n var i2 = pts.length / 2;\n var i1 = i2 - 2;\n dispX = pts[i2] - pts[i1];\n dispY = pts[i2 + 1] - pts[i1 + 1];\n } else if (rs.isRound) {\n dispX = rs.midVector[1];\n dispY = -rs.midVector[0];\n } else {\n var i2 = pts.length / 2 - 1;\n var i1 = i2 - 2;\n dispX = pts[i2] - pts[i1];\n dispY = pts[i2 + 1] - pts[i1 + 1];\n }\n } else if (isMultibezier || isCompound || isSelf) {\n var pts = rs.allpts;\n var cpts = rs.ctrlpts;\n var bp0x, bp0y;\n var bp1x, bp1y;\n if (cpts.length / 2 % 2 === 0) {\n var p0 = pts.length / 2 - 1; // startpt\n var ic = p0 + 2;\n var p1 = ic + 2;\n bp0x = qbezierAt(pts[p0], pts[ic], pts[p1], 0.0);\n bp0y = qbezierAt(pts[p0 + 1], pts[ic + 1], pts[p1 + 1], 0.0);\n bp1x = qbezierAt(pts[p0], pts[ic], pts[p1], 0.0001);\n bp1y = qbezierAt(pts[p0 + 1], pts[ic + 1], pts[p1 + 1], 0.0001);\n } else {\n var ic = pts.length / 2 - 1; // ctrpt\n var p0 = ic - 2; // startpt\n var p1 = ic + 2; // endpt\n\n bp0x = qbezierAt(pts[p0], pts[ic], pts[p1], 0.4999);\n bp0y = qbezierAt(pts[p0 + 1], pts[ic + 1], pts[p1 + 1], 0.4999);\n bp1x = qbezierAt(pts[p0], pts[ic], pts[p1], 0.5);\n bp1y = qbezierAt(pts[p0 + 1], pts[ic + 1], pts[p1 + 1], 0.5);\n }\n dispX = bp1x - bp0x;\n dispY = bp1y - bp0y;\n }\n rs.midtgtArrowAngle = getAngleFromDisp(dispX, dispY);\n rs.midDispX = dispX;\n rs.midDispY = dispY;\n\n // mid source\n //\n\n dispX *= -1;\n dispY *= -1;\n if (isSegments) {\n var pts = rs.allpts;\n if (pts.length / 2 % 2 === 0) ; else if (!rs.isRound) {\n var i2 = pts.length / 2 - 1;\n var i3 = i2 + 2;\n dispX = -(pts[i3] - pts[i2]);\n dispY = -(pts[i3 + 1] - pts[i2 + 1]);\n }\n }\n rs.midsrcArrowAngle = getAngleFromDisp(dispX, dispY);\n\n // target\n //\n\n if (isSegments) {\n dispX = endX - rs.segpts[rs.segpts.length - 2];\n dispY = endY - rs.segpts[rs.segpts.length - 1];\n } else if (isMultibezier || isCompound || isSelf || isBezier) {\n var pts = rs.allpts;\n var l = pts.length;\n var bX = qbezierAt(pts[l - 6], pts[l - 4], pts[l - 2], 0.9);\n var bY = qbezierAt(pts[l - 5], pts[l - 3], pts[l - 1], 0.9);\n dispX = endX - bX;\n dispY = endY - bY;\n } else {\n dispX = endX - midX;\n dispY = endY - midY;\n }\n rs.tgtArrowAngle = getAngleFromDisp(dispX, dispY);\n};\nBRp$d.getArrowWidth = BRp$d.getArrowHeight = function (edgeWidth, scale) {\n var cache = this.arrowWidthCache = this.arrowWidthCache || {};\n var cachedVal = cache[edgeWidth + ', ' + scale];\n if (cachedVal) {\n return cachedVal;\n }\n cachedVal = Math.max(Math.pow(edgeWidth * 13.37, 0.9), 29) * scale;\n cache[edgeWidth + ', ' + scale] = cachedVal;\n return cachedVal;\n};\n\n/**\n * Explained by Blindman67 at https://stackoverflow.com/a/44856925/11028828\n */\n\n// Declare reused variable to avoid reallocating variables every time the function is called\nvar x,\n y,\n v1 = {},\n v2 = {},\n sinA,\n sinA90,\n radDirection,\n drawDirection,\n angle,\n halfAngle,\n cRadius,\n lenOut,\n radius,\n limit;\nvar startX, startY, stopX, stopY;\nvar lastPoint;\n\n// convert 2 points into vector form, polar form, and normalised\nvar asVec = function asVec(p, pp, v) {\n v.x = pp.x - p.x;\n v.y = pp.y - p.y;\n v.len = Math.sqrt(v.x * v.x + v.y * v.y);\n v.nx = v.x / v.len;\n v.ny = v.y / v.len;\n v.ang = Math.atan2(v.ny, v.nx);\n};\nvar invertVec = function invertVec(originalV, invertedV) {\n invertedV.x = originalV.x * -1;\n invertedV.y = originalV.y * -1;\n invertedV.nx = originalV.nx * -1;\n invertedV.ny = originalV.ny * -1;\n invertedV.ang = originalV.ang > 0 ? -(Math.PI - originalV.ang) : Math.PI + originalV.ang;\n};\nvar calcCornerArc = function calcCornerArc(previousPoint, currentPoint, nextPoint, radiusMax, isArcRadius) {\n //-----------------------------------------\n // Part 1\n previousPoint !== lastPoint ? asVec(currentPoint, previousPoint, v1) : invertVec(v2, v1); // Avoid recalculating vec if it is the invert of the last one calculated\n asVec(currentPoint, nextPoint, v2);\n sinA = v1.nx * v2.ny - v1.ny * v2.nx;\n sinA90 = v1.nx * v2.nx - v1.ny * -v2.ny;\n angle = Math.asin(Math.max(-1, Math.min(1, sinA)));\n if (Math.abs(angle) < 1e-6) {\n x = currentPoint.x;\n y = currentPoint.y;\n cRadius = radius = 0;\n return;\n }\n //-----------------------------------------\n radDirection = 1;\n drawDirection = false;\n if (sinA90 < 0) {\n if (angle < 0) {\n angle = Math.PI + angle;\n } else {\n angle = Math.PI - angle;\n radDirection = -1;\n drawDirection = true;\n }\n } else {\n if (angle > 0) {\n radDirection = -1;\n drawDirection = true;\n }\n }\n if (currentPoint.radius !== undefined) {\n radius = currentPoint.radius;\n } else {\n radius = radiusMax;\n }\n //-----------------------------------------\n // Part 2\n halfAngle = angle / 2;\n //-----------------------------------------\n\n limit = Math.min(v1.len / 2, v2.len / 2);\n if (isArcRadius) {\n //-----------------------------------------\n // Part 3\n lenOut = Math.abs(Math.cos(halfAngle) * radius / Math.sin(halfAngle));\n\n //-----------------------------------------\n // Special part A\n if (lenOut > limit) {\n lenOut = limit;\n cRadius = Math.abs(lenOut * Math.sin(halfAngle) / Math.cos(halfAngle));\n } else {\n cRadius = radius;\n }\n } else {\n lenOut = Math.min(limit, radius);\n cRadius = Math.abs(lenOut * Math.sin(halfAngle) / Math.cos(halfAngle));\n }\n //-----------------------------------------\n\n //-----------------------------------------\n // Part 4\n stopX = currentPoint.x + v2.nx * lenOut;\n stopY = currentPoint.y + v2.ny * lenOut;\n //-----------------------------------------\n // Part 5\n x = stopX - v2.ny * cRadius * radDirection;\n y = stopY + v2.nx * cRadius * radDirection;\n //-----------------------------------------\n // Additional Part : calculate start point E\n startX = currentPoint.x + v1.nx * lenOut;\n startY = currentPoint.y + v1.ny * lenOut;\n\n // Save last point to avoid recalculating vector when not needed\n lastPoint = currentPoint;\n};\n\n/**\n * Draw corner provided by {@link getRoundCorner}\n *\n * @param ctx :CanvasRenderingContext2D\n * @param roundCorner {{cx:number, cy:number, radius:number, endAngle: number, startAngle: number, counterClockwise: boolean}}\n */\nfunction drawPreparedRoundCorner(ctx, roundCorner) {\n if (roundCorner.radius === 0) ctx.lineTo(roundCorner.cx, roundCorner.cy);else ctx.arc(roundCorner.cx, roundCorner.cy, roundCorner.radius, roundCorner.startAngle, roundCorner.endAngle, roundCorner.counterClockwise);\n}\n\n/**\n * Get round corner from a point and its previous and next neighbours in a path\n *\n * @param previousPoint {{x: number, y:number, radius: number?}}\n * @param currentPoint {{x: number, y:number, radius: number?}}\n * @param nextPoint {{x: number, y:number, radius: number?}}\n * @param radiusMax :number\n * @param isArcRadius :boolean\n * @return {{\n * cx:number, cy:number, radius:number,\n * startX:number, startY:number,\n * stopX:number, stopY: number,\n * endAngle: number, startAngle: number, counterClockwise: boolean\n * }}\n */\nfunction getRoundCorner(previousPoint, currentPoint, nextPoint, radiusMax) {\n var isArcRadius = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : true;\n if (radiusMax === 0 || currentPoint.radius === 0) return {\n cx: currentPoint.x,\n cy: currentPoint.y,\n radius: 0,\n startX: currentPoint.x,\n startY: currentPoint.y,\n stopX: currentPoint.x,\n stopY: currentPoint.y,\n startAngle: undefined,\n endAngle: undefined,\n counterClockwise: undefined\n };\n calcCornerArc(previousPoint, currentPoint, nextPoint, radiusMax, isArcRadius);\n return {\n cx: x,\n cy: y,\n radius: cRadius,\n startX: startX,\n startY: startY,\n stopX: stopX,\n stopY: stopY,\n startAngle: v1.ang + Math.PI / 2 * radDirection,\n endAngle: v2.ang - Math.PI / 2 * radDirection,\n counterClockwise: drawDirection\n };\n}\n\nvar AVOID_IMPOSSIBLE_BEZIER_CONSTANT = 0.01;\nvar AVOID_IMPOSSIBLE_BEZIER_CONSTANT_L = Math.sqrt(2 * AVOID_IMPOSSIBLE_BEZIER_CONSTANT);\nvar BRp$c = {};\nBRp$c.findMidptPtsEtc = function (edge, pairInfo) {\n var posPts = pairInfo.posPts,\n intersectionPts = pairInfo.intersectionPts,\n vectorNormInverse = pairInfo.vectorNormInverse;\n var midptPts;\n\n // n.b. assumes all edges in bezier bundle have same endpoints specified\n var srcManEndpt = edge.pstyle('source-endpoint');\n var tgtManEndpt = edge.pstyle('target-endpoint');\n var haveManualEndPts = srcManEndpt.units != null && tgtManEndpt.units != null;\n var recalcVectorNormInverse = function recalcVectorNormInverse(x1, y1, x2, y2) {\n var dy = y2 - y1;\n var dx = x2 - x1;\n var l = Math.sqrt(dx * dx + dy * dy);\n return {\n x: -dy / l,\n y: dx / l\n };\n };\n var edgeDistances = edge.pstyle('edge-distances').value;\n switch (edgeDistances) {\n case 'node-position':\n midptPts = posPts;\n break;\n case 'intersection':\n midptPts = intersectionPts;\n break;\n case 'endpoints':\n {\n if (haveManualEndPts) {\n var _this$manualEndptToPx = this.manualEndptToPx(edge.source()[0], srcManEndpt),\n _this$manualEndptToPx2 = _slicedToArray(_this$manualEndptToPx, 2),\n x1 = _this$manualEndptToPx2[0],\n y1 = _this$manualEndptToPx2[1];\n var _this$manualEndptToPx3 = this.manualEndptToPx(edge.target()[0], tgtManEndpt),\n _this$manualEndptToPx4 = _slicedToArray(_this$manualEndptToPx3, 2),\n x2 = _this$manualEndptToPx4[0],\n y2 = _this$manualEndptToPx4[1];\n var endPts = {\n x1: x1,\n y1: y1,\n x2: x2,\n y2: y2\n };\n vectorNormInverse = recalcVectorNormInverse(x1, y1, x2, y2);\n midptPts = endPts;\n } else {\n warn(\"Edge \".concat(edge.id(), \" has edge-distances:endpoints specified without manual endpoints specified via source-endpoint and target-endpoint. Falling back on edge-distances:intersection (default).\"));\n midptPts = intersectionPts; // back to default\n }\n break;\n }\n }\n return {\n midptPts: midptPts,\n vectorNormInverse: vectorNormInverse\n };\n};\nBRp$c.findHaystackPoints = function (edges) {\n for (var i = 0; i < edges.length; i++) {\n var edge = edges[i];\n var _p = edge._private;\n var rs = _p.rscratch;\n if (!rs.haystack) {\n var angle = Math.random() * 2 * Math.PI;\n rs.source = {\n x: Math.cos(angle),\n y: Math.sin(angle)\n };\n angle = Math.random() * 2 * Math.PI;\n rs.target = {\n x: Math.cos(angle),\n y: Math.sin(angle)\n };\n }\n var src = _p.source;\n var tgt = _p.target;\n var srcPos = src.position();\n var tgtPos = tgt.position();\n var srcW = src.width();\n var tgtW = tgt.width();\n var srcH = src.height();\n var tgtH = tgt.height();\n var radius = edge.pstyle('haystack-radius').value;\n var halfRadius = radius / 2; // b/c have to half width/height\n\n rs.haystackPts = rs.allpts = [rs.source.x * srcW * halfRadius + srcPos.x, rs.source.y * srcH * halfRadius + srcPos.y, rs.target.x * tgtW * halfRadius + tgtPos.x, rs.target.y * tgtH * halfRadius + tgtPos.y];\n rs.midX = (rs.allpts[0] + rs.allpts[2]) / 2;\n rs.midY = (rs.allpts[1] + rs.allpts[3]) / 2;\n\n // always override as haystack in case set to different type previously\n rs.edgeType = 'haystack';\n rs.haystack = true;\n this.storeEdgeProjections(edge);\n this.calculateArrowAngles(edge);\n this.recalculateEdgeLabelProjections(edge);\n this.calculateLabelAngles(edge);\n }\n};\nBRp$c.findSegmentsPoints = function (edge, pairInfo) {\n // Segments (multiple straight lines)\n\n var rs = edge._private.rscratch;\n var segmentWs = edge.pstyle('segment-weights');\n var segmentDs = edge.pstyle('segment-distances');\n var segmentRs = edge.pstyle('segment-radii');\n var segmentTs = edge.pstyle('radius-type');\n var segmentsN = Math.min(segmentWs.pfValue.length, segmentDs.pfValue.length);\n var lastRadius = segmentRs.pfValue[segmentRs.pfValue.length - 1];\n var lastRadiusType = segmentTs.pfValue[segmentTs.pfValue.length - 1];\n rs.edgeType = 'segments';\n rs.segpts = [];\n rs.radii = [];\n rs.isArcRadius = [];\n for (var s = 0; s < segmentsN; s++) {\n var w = segmentWs.pfValue[s];\n var d = segmentDs.pfValue[s];\n var w1 = 1 - w;\n var w2 = w;\n var _this$findMidptPtsEtc = this.findMidptPtsEtc(edge, pairInfo),\n midptPts = _this$findMidptPtsEtc.midptPts,\n vectorNormInverse = _this$findMidptPtsEtc.vectorNormInverse;\n var adjustedMidpt = {\n x: midptPts.x1 * w1 + midptPts.x2 * w2,\n y: midptPts.y1 * w1 + midptPts.y2 * w2\n };\n rs.segpts.push(adjustedMidpt.x + vectorNormInverse.x * d, adjustedMidpt.y + vectorNormInverse.y * d);\n rs.radii.push(segmentRs.pfValue[s] !== undefined ? segmentRs.pfValue[s] : lastRadius);\n rs.isArcRadius.push((segmentTs.pfValue[s] !== undefined ? segmentTs.pfValue[s] : lastRadiusType) === 'arc-radius');\n }\n};\nBRp$c.findLoopPoints = function (edge, pairInfo, i, edgeIsUnbundled) {\n // Self-edge\n\n var rs = edge._private.rscratch;\n var dirCounts = pairInfo.dirCounts,\n srcPos = pairInfo.srcPos;\n var ctrlptDists = edge.pstyle('control-point-distances');\n var ctrlptDist = ctrlptDists ? ctrlptDists.pfValue[0] : undefined;\n var loopDir = edge.pstyle('loop-direction').pfValue;\n var loopSwp = edge.pstyle('loop-sweep').pfValue;\n var stepSize = edge.pstyle('control-point-step-size').pfValue;\n rs.edgeType = 'self';\n var j = i;\n var loopDist = stepSize;\n if (edgeIsUnbundled) {\n j = 0;\n loopDist = ctrlptDist;\n }\n var loopAngle = loopDir - Math.PI / 2;\n var outAngle = loopAngle - loopSwp / 2;\n var inAngle = loopAngle + loopSwp / 2;\n\n // increase by step size for overlapping loops, keyed on direction and sweep values\n var dc = String(loopDir + '_' + loopSwp);\n j = dirCounts[dc] === undefined ? dirCounts[dc] = 0 : ++dirCounts[dc];\n rs.ctrlpts = [srcPos.x + Math.cos(outAngle) * 1.4 * loopDist * (j / 3 + 1), srcPos.y + Math.sin(outAngle) * 1.4 * loopDist * (j / 3 + 1), srcPos.x + Math.cos(inAngle) * 1.4 * loopDist * (j / 3 + 1), srcPos.y + Math.sin(inAngle) * 1.4 * loopDist * (j / 3 + 1)];\n};\nBRp$c.findCompoundLoopPoints = function (edge, pairInfo, i, edgeIsUnbundled) {\n // Compound edge\n\n var rs = edge._private.rscratch;\n rs.edgeType = 'compound';\n var srcPos = pairInfo.srcPos,\n tgtPos = pairInfo.tgtPos,\n srcW = pairInfo.srcW,\n srcH = pairInfo.srcH,\n tgtW = pairInfo.tgtW,\n tgtH = pairInfo.tgtH;\n var stepSize = edge.pstyle('control-point-step-size').pfValue;\n var ctrlptDists = edge.pstyle('control-point-distances');\n var ctrlptDist = ctrlptDists ? ctrlptDists.pfValue[0] : undefined;\n var j = i;\n var loopDist = stepSize;\n if (edgeIsUnbundled) {\n j = 0;\n loopDist = ctrlptDist;\n }\n var loopW = 50;\n var loopaPos = {\n x: srcPos.x - srcW / 2,\n y: srcPos.y - srcH / 2\n };\n var loopbPos = {\n x: tgtPos.x - tgtW / 2,\n y: tgtPos.y - tgtH / 2\n };\n var loopPos = {\n x: Math.min(loopaPos.x, loopbPos.x),\n y: Math.min(loopaPos.y, loopbPos.y)\n };\n\n // avoids cases with impossible beziers\n var minCompoundStretch = 0.5;\n var compoundStretchA = Math.max(minCompoundStretch, Math.log(srcW * AVOID_IMPOSSIBLE_BEZIER_CONSTANT));\n var compoundStretchB = Math.max(minCompoundStretch, Math.log(tgtW * AVOID_IMPOSSIBLE_BEZIER_CONSTANT));\n rs.ctrlpts = [loopPos.x, loopPos.y - (1 + Math.pow(loopW, 1.12) / 100) * loopDist * (j / 3 + 1) * compoundStretchA, loopPos.x - (1 + Math.pow(loopW, 1.12) / 100) * loopDist * (j / 3 + 1) * compoundStretchB, loopPos.y];\n};\nBRp$c.findStraightEdgePoints = function (edge) {\n // Straight edge within bundle\n\n edge._private.rscratch.edgeType = 'straight';\n};\nBRp$c.findBezierPoints = function (edge, pairInfo, i, edgeIsUnbundled, edgeIsSwapped) {\n var rs = edge._private.rscratch;\n var stepSize = edge.pstyle('control-point-step-size').pfValue;\n var ctrlptDists = edge.pstyle('control-point-distances');\n var ctrlptWs = edge.pstyle('control-point-weights');\n var bezierN = ctrlptDists && ctrlptWs ? Math.min(ctrlptDists.value.length, ctrlptWs.value.length) : 1;\n var ctrlptDist = ctrlptDists ? ctrlptDists.pfValue[0] : undefined;\n var ctrlptWeight = ctrlptWs.value[0];\n\n // (Multi)bezier\n\n var multi = edgeIsUnbundled;\n rs.edgeType = multi ? 'multibezier' : 'bezier';\n rs.ctrlpts = [];\n for (var b = 0; b < bezierN; b++) {\n var normctrlptDist = (0.5 - pairInfo.eles.length / 2 + i) * stepSize * (edgeIsSwapped ? -1 : 1);\n var manctrlptDist = undefined;\n var sign = signum(normctrlptDist);\n if (multi) {\n ctrlptDist = ctrlptDists ? ctrlptDists.pfValue[b] : stepSize; // fall back on step size\n ctrlptWeight = ctrlptWs.value[b];\n }\n if (edgeIsUnbundled) {\n // multi or single unbundled\n manctrlptDist = ctrlptDist;\n } else {\n manctrlptDist = ctrlptDist !== undefined ? sign * ctrlptDist : undefined;\n }\n var distanceFromMidpoint = manctrlptDist !== undefined ? manctrlptDist : normctrlptDist;\n var w1 = 1 - ctrlptWeight;\n var w2 = ctrlptWeight;\n var _this$findMidptPtsEtc2 = this.findMidptPtsEtc(edge, pairInfo),\n midptPts = _this$findMidptPtsEtc2.midptPts,\n vectorNormInverse = _this$findMidptPtsEtc2.vectorNormInverse;\n var adjustedMidpt = {\n x: midptPts.x1 * w1 + midptPts.x2 * w2,\n y: midptPts.y1 * w1 + midptPts.y2 * w2\n };\n rs.ctrlpts.push(adjustedMidpt.x + vectorNormInverse.x * distanceFromMidpoint, adjustedMidpt.y + vectorNormInverse.y * distanceFromMidpoint);\n }\n};\nBRp$c.findTaxiPoints = function (edge, pairInfo) {\n // Taxicab geometry with two turns maximum\n\n var rs = edge._private.rscratch;\n rs.edgeType = 'segments';\n var VERTICAL = 'vertical';\n var HORIZONTAL = 'horizontal';\n var LEFTWARD = 'leftward';\n var RIGHTWARD = 'rightward';\n var DOWNWARD = 'downward';\n var UPWARD = 'upward';\n var AUTO = 'auto';\n var posPts = pairInfo.posPts,\n srcW = pairInfo.srcW,\n srcH = pairInfo.srcH,\n tgtW = pairInfo.tgtW,\n tgtH = pairInfo.tgtH;\n var edgeDistances = edge.pstyle('edge-distances').value;\n var dIncludesNodeBody = edgeDistances !== 'node-position';\n var taxiDir = edge.pstyle('taxi-direction').value;\n var rawTaxiDir = taxiDir; // unprocessed value\n var taxiTurn = edge.pstyle('taxi-turn');\n var turnIsPercent = taxiTurn.units === '%';\n var taxiTurnPfVal = taxiTurn.pfValue;\n var turnIsNegative = taxiTurnPfVal < 0; // i.e. from target side\n var minD = edge.pstyle('taxi-turn-min-distance').pfValue;\n var dw = dIncludesNodeBody ? (srcW + tgtW) / 2 : 0;\n var dh = dIncludesNodeBody ? (srcH + tgtH) / 2 : 0;\n var pdx = posPts.x2 - posPts.x1;\n var pdy = posPts.y2 - posPts.y1;\n\n // take away the effective w/h from the magnitude of the delta value\n var subDWH = function subDWH(dxy, dwh) {\n if (dxy > 0) {\n return Math.max(dxy - dwh, 0);\n } else {\n return Math.min(dxy + dwh, 0);\n }\n };\n var dx = subDWH(pdx, dw);\n var dy = subDWH(pdy, dh);\n var isExplicitDir = false;\n if (rawTaxiDir === AUTO) {\n taxiDir = Math.abs(dx) > Math.abs(dy) ? HORIZONTAL : VERTICAL;\n } else if (rawTaxiDir === UPWARD || rawTaxiDir === DOWNWARD) {\n taxiDir = VERTICAL;\n isExplicitDir = true;\n } else if (rawTaxiDir === LEFTWARD || rawTaxiDir === RIGHTWARD) {\n taxiDir = HORIZONTAL;\n isExplicitDir = true;\n }\n var isVert = taxiDir === VERTICAL;\n var l = isVert ? dy : dx;\n var pl = isVert ? pdy : pdx;\n var sgnL = signum(pl);\n var forcedDir = false;\n if (!(isExplicitDir && (turnIsPercent || turnIsNegative)) // forcing in this case would cause weird growing in the opposite direction\n && (rawTaxiDir === DOWNWARD && pl < 0 || rawTaxiDir === UPWARD && pl > 0 || rawTaxiDir === LEFTWARD && pl > 0 || rawTaxiDir === RIGHTWARD && pl < 0)) {\n sgnL *= -1;\n l = sgnL * Math.abs(l);\n forcedDir = true;\n }\n var d;\n if (turnIsPercent) {\n var p = taxiTurnPfVal < 0 ? 1 + taxiTurnPfVal : taxiTurnPfVal;\n d = p * l;\n } else {\n var k = taxiTurnPfVal < 0 ? l : 0;\n d = k + taxiTurnPfVal * sgnL;\n }\n var getIsTooClose = function getIsTooClose(d) {\n return Math.abs(d) < minD || Math.abs(d) >= Math.abs(l);\n };\n var isTooCloseSrc = getIsTooClose(d);\n var isTooCloseTgt = getIsTooClose(Math.abs(l) - Math.abs(d));\n var isTooClose = isTooCloseSrc || isTooCloseTgt;\n if (isTooClose && !forcedDir) {\n // non-ideal routing\n if (isVert) {\n // vertical fallbacks\n var lShapeInsideSrc = Math.abs(pl) <= srcH / 2;\n var lShapeInsideTgt = Math.abs(pdx) <= tgtW / 2;\n if (lShapeInsideSrc) {\n // horizontal Z-shape (direction not respected)\n var x = (posPts.x1 + posPts.x2) / 2;\n var y1 = posPts.y1,\n y2 = posPts.y2;\n rs.segpts = [x, y1, x, y2];\n } else if (lShapeInsideTgt) {\n // vertical Z-shape (distance not respected)\n var y = (posPts.y1 + posPts.y2) / 2;\n var x1 = posPts.x1,\n x2 = posPts.x2;\n rs.segpts = [x1, y, x2, y];\n } else {\n // L-shape fallback (turn distance not respected, but works well with tree siblings)\n rs.segpts = [posPts.x1, posPts.y2];\n }\n } else {\n // horizontal fallbacks\n var _lShapeInsideSrc = Math.abs(pl) <= srcW / 2;\n var _lShapeInsideTgt = Math.abs(pdy) <= tgtH / 2;\n if (_lShapeInsideSrc) {\n // vertical Z-shape (direction not respected)\n var _y = (posPts.y1 + posPts.y2) / 2;\n var _x = posPts.x1,\n _x2 = posPts.x2;\n rs.segpts = [_x, _y, _x2, _y];\n } else if (_lShapeInsideTgt) {\n // horizontal Z-shape (turn distance not respected)\n var _x3 = (posPts.x1 + posPts.x2) / 2;\n var _y2 = posPts.y1,\n _y3 = posPts.y2;\n rs.segpts = [_x3, _y2, _x3, _y3];\n } else {\n // L-shape (turn distance not respected, but works well for tree siblings)\n rs.segpts = [posPts.x2, posPts.y1];\n }\n }\n } else {\n // ideal routing\n if (isVert) {\n var _y4 = posPts.y1 + d + (dIncludesNodeBody ? srcH / 2 * sgnL : 0);\n var _x4 = posPts.x1,\n _x5 = posPts.x2;\n rs.segpts = [_x4, _y4, _x5, _y4];\n } else {\n // horizontal\n var _x6 = posPts.x1 + d + (dIncludesNodeBody ? srcW / 2 * sgnL : 0);\n var _y5 = posPts.y1,\n _y6 = posPts.y2;\n rs.segpts = [_x6, _y5, _x6, _y6];\n }\n }\n if (rs.isRound) {\n var radius = edge.pstyle('taxi-radius').value;\n var isArcRadius = edge.pstyle('radius-type').value[0] === 'arc-radius';\n rs.radii = new Array(rs.segpts.length / 2).fill(radius);\n rs.isArcRadius = new Array(rs.segpts.length / 2).fill(isArcRadius);\n }\n};\nBRp$c.tryToCorrectInvalidPoints = function (edge, pairInfo) {\n var rs = edge._private.rscratch;\n\n // can only correct beziers for now...\n if (rs.edgeType === 'bezier') {\n var srcPos = pairInfo.srcPos,\n tgtPos = pairInfo.tgtPos,\n srcW = pairInfo.srcW,\n srcH = pairInfo.srcH,\n tgtW = pairInfo.tgtW,\n tgtH = pairInfo.tgtH,\n srcShape = pairInfo.srcShape,\n tgtShape = pairInfo.tgtShape,\n srcCornerRadius = pairInfo.srcCornerRadius,\n tgtCornerRadius = pairInfo.tgtCornerRadius,\n srcRs = pairInfo.srcRs,\n tgtRs = pairInfo.tgtRs;\n var badStart = !number$1(rs.startX) || !number$1(rs.startY);\n var badAStart = !number$1(rs.arrowStartX) || !number$1(rs.arrowStartY);\n var badEnd = !number$1(rs.endX) || !number$1(rs.endY);\n var badAEnd = !number$1(rs.arrowEndX) || !number$1(rs.arrowEndY);\n var minCpADistFactor = 3;\n var arrowW = this.getArrowWidth(edge.pstyle('width').pfValue, edge.pstyle('arrow-scale').value) * this.arrowShapeWidth;\n var minCpADist = minCpADistFactor * arrowW;\n var startACpDist = dist({\n x: rs.ctrlpts[0],\n y: rs.ctrlpts[1]\n }, {\n x: rs.startX,\n y: rs.startY\n });\n var closeStartACp = startACpDist < minCpADist;\n var endACpDist = dist({\n x: rs.ctrlpts[0],\n y: rs.ctrlpts[1]\n }, {\n x: rs.endX,\n y: rs.endY\n });\n var closeEndACp = endACpDist < minCpADist;\n var overlapping = false;\n if (badStart || badAStart || closeStartACp) {\n overlapping = true;\n\n // project control point along line from src centre to outside the src shape\n // (otherwise intersection will yield nothing)\n var cpD = {\n // delta\n x: rs.ctrlpts[0] - srcPos.x,\n y: rs.ctrlpts[1] - srcPos.y\n };\n var cpL = Math.sqrt(cpD.x * cpD.x + cpD.y * cpD.y); // length of line\n var cpM = {\n // normalised delta\n x: cpD.x / cpL,\n y: cpD.y / cpL\n };\n var radius = Math.max(srcW, srcH);\n var cpProj = {\n // *2 radius guarantees outside shape\n x: rs.ctrlpts[0] + cpM.x * 2 * radius,\n y: rs.ctrlpts[1] + cpM.y * 2 * radius\n };\n var srcCtrlPtIntn = srcShape.intersectLine(srcPos.x, srcPos.y, srcW, srcH, cpProj.x, cpProj.y, 0, srcCornerRadius, srcRs);\n if (closeStartACp) {\n rs.ctrlpts[0] = rs.ctrlpts[0] + cpM.x * (minCpADist - startACpDist);\n rs.ctrlpts[1] = rs.ctrlpts[1] + cpM.y * (minCpADist - startACpDist);\n } else {\n rs.ctrlpts[0] = srcCtrlPtIntn[0] + cpM.x * minCpADist;\n rs.ctrlpts[1] = srcCtrlPtIntn[1] + cpM.y * minCpADist;\n }\n }\n if (badEnd || badAEnd || closeEndACp) {\n overlapping = true;\n\n // project control point along line from tgt centre to outside the tgt shape\n // (otherwise intersection will yield nothing)\n var _cpD = {\n // delta\n x: rs.ctrlpts[0] - tgtPos.x,\n y: rs.ctrlpts[1] - tgtPos.y\n };\n var _cpL = Math.sqrt(_cpD.x * _cpD.x + _cpD.y * _cpD.y); // length of line\n var _cpM = {\n // normalised delta\n x: _cpD.x / _cpL,\n y: _cpD.y / _cpL\n };\n var _radius = Math.max(srcW, srcH);\n var _cpProj = {\n // *2 radius guarantees outside shape\n x: rs.ctrlpts[0] + _cpM.x * 2 * _radius,\n y: rs.ctrlpts[1] + _cpM.y * 2 * _radius\n };\n var tgtCtrlPtIntn = tgtShape.intersectLine(tgtPos.x, tgtPos.y, tgtW, tgtH, _cpProj.x, _cpProj.y, 0, tgtCornerRadius, tgtRs);\n if (closeEndACp) {\n rs.ctrlpts[0] = rs.ctrlpts[0] + _cpM.x * (minCpADist - endACpDist);\n rs.ctrlpts[1] = rs.ctrlpts[1] + _cpM.y * (minCpADist - endACpDist);\n } else {\n rs.ctrlpts[0] = tgtCtrlPtIntn[0] + _cpM.x * minCpADist;\n rs.ctrlpts[1] = tgtCtrlPtIntn[1] + _cpM.y * minCpADist;\n }\n }\n if (overlapping) {\n // recalc endpts\n this.findEndpoints(edge);\n }\n }\n};\nBRp$c.storeAllpts = function (edge) {\n var rs = edge._private.rscratch;\n if (rs.edgeType === 'multibezier' || rs.edgeType === 'bezier' || rs.edgeType === 'self' || rs.edgeType === 'compound') {\n rs.allpts = [];\n rs.allpts.push(rs.startX, rs.startY);\n for (var b = 0; b + 1 < rs.ctrlpts.length; b += 2) {\n // ctrl pt itself\n rs.allpts.push(rs.ctrlpts[b], rs.ctrlpts[b + 1]);\n\n // the midpt between ctrlpts as intermediate destination pts\n if (b + 3 < rs.ctrlpts.length) {\n rs.allpts.push((rs.ctrlpts[b] + rs.ctrlpts[b + 2]) / 2, (rs.ctrlpts[b + 1] + rs.ctrlpts[b + 3]) / 2);\n }\n }\n rs.allpts.push(rs.endX, rs.endY);\n var m, mt;\n if (rs.ctrlpts.length / 2 % 2 === 0) {\n m = rs.allpts.length / 2 - 1;\n rs.midX = rs.allpts[m];\n rs.midY = rs.allpts[m + 1];\n } else {\n m = rs.allpts.length / 2 - 3;\n mt = 0.5;\n rs.midX = qbezierAt(rs.allpts[m], rs.allpts[m + 2], rs.allpts[m + 4], mt);\n rs.midY = qbezierAt(rs.allpts[m + 1], rs.allpts[m + 3], rs.allpts[m + 5], mt);\n }\n } else if (rs.edgeType === 'straight') {\n // need to calc these after endpts\n rs.allpts = [rs.startX, rs.startY, rs.endX, rs.endY];\n\n // default midpt for labels etc\n rs.midX = (rs.startX + rs.endX + rs.arrowStartX + rs.arrowEndX) / 4;\n rs.midY = (rs.startY + rs.endY + rs.arrowStartY + rs.arrowEndY) / 4;\n } else if (rs.edgeType === 'segments') {\n rs.allpts = [];\n rs.allpts.push(rs.startX, rs.startY);\n rs.allpts.push.apply(rs.allpts, rs.segpts);\n rs.allpts.push(rs.endX, rs.endY);\n if (rs.isRound) {\n rs.roundCorners = [];\n for (var i = 2; i + 3 < rs.allpts.length; i += 2) {\n var radius = rs.radii[i / 2 - 1];\n var isArcRadius = rs.isArcRadius[i / 2 - 1];\n rs.roundCorners.push(getRoundCorner({\n x: rs.allpts[i - 2],\n y: rs.allpts[i - 1]\n }, {\n x: rs.allpts[i],\n y: rs.allpts[i + 1],\n radius: radius\n }, {\n x: rs.allpts[i + 2],\n y: rs.allpts[i + 3]\n }, radius, isArcRadius));\n }\n }\n if (rs.segpts.length % 4 === 0) {\n var i2 = rs.segpts.length / 2;\n var i1 = i2 - 2;\n rs.midX = (rs.segpts[i1] + rs.segpts[i2]) / 2;\n rs.midY = (rs.segpts[i1 + 1] + rs.segpts[i2 + 1]) / 2;\n } else {\n var _i = rs.segpts.length / 2 - 1;\n if (!rs.isRound) {\n rs.midX = rs.segpts[_i];\n rs.midY = rs.segpts[_i + 1];\n } else {\n var point = {\n x: rs.segpts[_i],\n y: rs.segpts[_i + 1]\n };\n var corner = rs.roundCorners[_i / 2];\n if (corner.radius === 0) {\n // On collinear points\n var nextPoint = {\n x: rs.segpts[_i + 2],\n y: rs.segpts[_i + 3]\n };\n rs.midX = point.x;\n rs.midY = point.y;\n rs.midVector = [point.y - nextPoint.y, nextPoint.x - point.x];\n } else {\n // On rounded points\n var v = [point.x - corner.cx, point.y - corner.cy];\n var factor = corner.radius / Math.sqrt(Math.pow(v[0], 2) + Math.pow(v[1], 2));\n v = v.map(function (c) {\n return c * factor;\n });\n rs.midX = corner.cx + v[0];\n rs.midY = corner.cy + v[1];\n rs.midVector = v;\n }\n }\n }\n }\n};\nBRp$c.checkForInvalidEdgeWarning = function (edge) {\n var rs = edge[0]._private.rscratch;\n if (rs.nodesOverlap || number$1(rs.startX) && number$1(rs.startY) && number$1(rs.endX) && number$1(rs.endY)) {\n rs.loggedErr = false;\n } else {\n if (!rs.loggedErr) {\n rs.loggedErr = true;\n warn('Edge `' + edge.id() + '` has invalid endpoints and so it is impossible to draw. Adjust your edge style (e.g. control points) accordingly or use an alternative edge type. This is expected behaviour when the source node and the target node overlap.');\n }\n }\n};\nBRp$c.findEdgeControlPoints = function (edges) {\n var _this = this;\n if (!edges || edges.length === 0) {\n return;\n }\n var r = this;\n var cy = r.cy;\n var hasCompounds = cy.hasCompoundNodes();\n var hashTable = new Map$1();\n var getKey = function getKey(pairId, edgeIsUnbundled) {\n return [].concat(_toConsumableArray(pairId), [edgeIsUnbundled ? 1 : 0]).join('-');\n };\n var pairIds = [];\n var haystackEdges = [];\n\n // create a table of edge (src, tgt) => list of edges between them\n for (var i = 0; i < edges.length; i++) {\n var edge = edges[i];\n var _p = edge._private;\n var curveStyle = edge.pstyle('curve-style').value;\n\n // ignore edges who are not to be displayed\n // they shouldn't take up space\n if (edge.removed() || !edge.takesUpSpace()) {\n continue;\n }\n if (curveStyle === 'haystack') {\n haystackEdges.push(edge);\n continue;\n }\n var edgeIsUnbundled = curveStyle === 'unbundled-bezier' || endsWith(curveStyle, 'segments') || curveStyle === 'straight' || curveStyle === 'straight-triangle' || endsWith(curveStyle, 'taxi');\n var edgeIsBezier = curveStyle === 'unbundled-bezier' || curveStyle === 'bezier';\n var src = _p.source;\n var tgt = _p.target;\n var srcIndex = src.poolIndex();\n var tgtIndex = tgt.poolIndex();\n var pairId = [srcIndex, tgtIndex].sort();\n var key = getKey(pairId, edgeIsUnbundled);\n var tableEntry = hashTable.get(key);\n if (tableEntry == null) {\n tableEntry = {\n eles: []\n };\n pairIds.push({\n pairId: pairId,\n edgeIsUnbundled: edgeIsUnbundled\n });\n hashTable.set(key, tableEntry);\n }\n tableEntry.eles.push(edge);\n if (edgeIsUnbundled) {\n tableEntry.hasUnbundled = true;\n }\n if (edgeIsBezier) {\n tableEntry.hasBezier = true;\n }\n }\n\n // for each pair (src, tgt), create the ctrl pts\n // Nested for loop is OK; total number of iterations for both loops = edgeCount\n var _loop = function _loop() {\n var _pairIds$p = pairIds[p],\n pairId = _pairIds$p.pairId,\n edgeIsUnbundled = _pairIds$p.edgeIsUnbundled;\n var key = getKey(pairId, edgeIsUnbundled);\n var pairInfo = hashTable.get(key);\n var swappedpairInfo;\n if (!pairInfo.hasUnbundled) {\n var pllEdges = pairInfo.eles[0].parallelEdges().filter(function (e) {\n return e.isBundledBezier();\n });\n clearArray(pairInfo.eles);\n pllEdges.forEach(function (edge) {\n return pairInfo.eles.push(edge);\n });\n\n // for each pair id, the edges should be sorted by index\n pairInfo.eles.sort(function (edge1, edge2) {\n return edge1.poolIndex() - edge2.poolIndex();\n });\n }\n var firstEdge = pairInfo.eles[0];\n var src = firstEdge.source();\n var tgt = firstEdge.target();\n\n // make sure src/tgt distinction is consistent w.r.t. pairId\n if (src.poolIndex() > tgt.poolIndex()) {\n var temp = src;\n src = tgt;\n tgt = temp;\n }\n var srcPos = pairInfo.srcPos = src.position();\n var tgtPos = pairInfo.tgtPos = tgt.position();\n var srcW = pairInfo.srcW = src.outerWidth();\n var srcH = pairInfo.srcH = src.outerHeight();\n var tgtW = pairInfo.tgtW = tgt.outerWidth();\n var tgtH = pairInfo.tgtH = tgt.outerHeight();\n var srcShape = pairInfo.srcShape = r.nodeShapes[_this.getNodeShape(src)];\n var tgtShape = pairInfo.tgtShape = r.nodeShapes[_this.getNodeShape(tgt)];\n var srcCornerRadius = pairInfo.srcCornerRadius = src.pstyle('corner-radius').value === 'auto' ? 'auto' : src.pstyle('corner-radius').pfValue;\n var tgtCornerRadius = pairInfo.tgtCornerRadius = tgt.pstyle('corner-radius').value === 'auto' ? 'auto' : tgt.pstyle('corner-radius').pfValue;\n var tgtRs = pairInfo.tgtRs = tgt._private.rscratch;\n var srcRs = pairInfo.srcRs = src._private.rscratch;\n pairInfo.dirCounts = {\n 'north': 0,\n 'west': 0,\n 'south': 0,\n 'east': 0,\n 'northwest': 0,\n 'southwest': 0,\n 'northeast': 0,\n 'southeast': 0\n };\n for (var _i2 = 0; _i2 < pairInfo.eles.length; _i2++) {\n var _edge = pairInfo.eles[_i2];\n var rs = _edge[0]._private.rscratch;\n var _curveStyle = _edge.pstyle('curve-style').value;\n var _edgeIsUnbundled = _curveStyle === 'unbundled-bezier' || endsWith(_curveStyle, 'segments') || endsWith(_curveStyle, 'taxi');\n\n // whether the normalised pair order is the reverse of the edge's src-tgt order\n var edgeIsSwapped = !src.same(_edge.source());\n if (!pairInfo.calculatedIntersection && src !== tgt && (pairInfo.hasBezier || pairInfo.hasUnbundled)) {\n pairInfo.calculatedIntersection = true;\n\n // pt outside src shape to calc distance/displacement from src to tgt\n var srcOutside = srcShape.intersectLine(srcPos.x, srcPos.y, srcW, srcH, tgtPos.x, tgtPos.y, 0, srcCornerRadius, srcRs);\n var srcIntn = pairInfo.srcIntn = srcOutside;\n\n // pt outside tgt shape to calc distance/displacement from src to tgt\n var tgtOutside = tgtShape.intersectLine(tgtPos.x, tgtPos.y, tgtW, tgtH, srcPos.x, srcPos.y, 0, tgtCornerRadius, tgtRs);\n var tgtIntn = pairInfo.tgtIntn = tgtOutside;\n var intersectionPts = pairInfo.intersectionPts = {\n x1: srcOutside[0],\n x2: tgtOutside[0],\n y1: srcOutside[1],\n y2: tgtOutside[1]\n };\n var posPts = pairInfo.posPts = {\n x1: srcPos.x,\n x2: tgtPos.x,\n y1: srcPos.y,\n y2: tgtPos.y\n };\n var dy = tgtOutside[1] - srcOutside[1];\n var dx = tgtOutside[0] - srcOutside[0];\n var l = Math.sqrt(dx * dx + dy * dy);\n if (number$1(l) && l >= AVOID_IMPOSSIBLE_BEZIER_CONSTANT_L) ; else {\n l = Math.sqrt(Math.max(dx * dx, AVOID_IMPOSSIBLE_BEZIER_CONSTANT) + Math.max(dy * dy, AVOID_IMPOSSIBLE_BEZIER_CONSTANT));\n }\n var vector = pairInfo.vector = {\n x: dx,\n y: dy\n };\n var vectorNorm = pairInfo.vectorNorm = {\n x: vector.x / l,\n y: vector.y / l\n };\n var vectorNormInverse = {\n x: -vectorNorm.y,\n y: vectorNorm.x\n };\n\n // if node shapes overlap, then no ctrl pts to draw\n pairInfo.nodesOverlap = !number$1(l) || tgtShape.checkPoint(srcOutside[0], srcOutside[1], 0, tgtW, tgtH, tgtPos.x, tgtPos.y, tgtCornerRadius, tgtRs) || srcShape.checkPoint(tgtOutside[0], tgtOutside[1], 0, srcW, srcH, srcPos.x, srcPos.y, srcCornerRadius, srcRs);\n pairInfo.vectorNormInverse = vectorNormInverse;\n swappedpairInfo = {\n nodesOverlap: pairInfo.nodesOverlap,\n dirCounts: pairInfo.dirCounts,\n calculatedIntersection: true,\n hasBezier: pairInfo.hasBezier,\n hasUnbundled: pairInfo.hasUnbundled,\n eles: pairInfo.eles,\n srcPos: tgtPos,\n srcRs: tgtRs,\n tgtPos: srcPos,\n tgtRs: srcRs,\n srcW: tgtW,\n srcH: tgtH,\n tgtW: srcW,\n tgtH: srcH,\n srcIntn: tgtIntn,\n tgtIntn: srcIntn,\n srcShape: tgtShape,\n tgtShape: srcShape,\n posPts: {\n x1: posPts.x2,\n y1: posPts.y2,\n x2: posPts.x1,\n y2: posPts.y1\n },\n intersectionPts: {\n x1: intersectionPts.x2,\n y1: intersectionPts.y2,\n x2: intersectionPts.x1,\n y2: intersectionPts.y1\n },\n vector: {\n x: -vector.x,\n y: -vector.y\n },\n vectorNorm: {\n x: -vectorNorm.x,\n y: -vectorNorm.y\n },\n vectorNormInverse: {\n x: -vectorNormInverse.x,\n y: -vectorNormInverse.y\n }\n };\n }\n var passedPairInfo = edgeIsSwapped ? swappedpairInfo : pairInfo;\n rs.nodesOverlap = passedPairInfo.nodesOverlap;\n rs.srcIntn = passedPairInfo.srcIntn;\n rs.tgtIntn = passedPairInfo.tgtIntn;\n rs.isRound = _curveStyle.startsWith('round');\n if (hasCompounds && (src.isParent() || src.isChild() || tgt.isParent() || tgt.isChild()) && (src.parents().anySame(tgt) || tgt.parents().anySame(src) || src.same(tgt) && src.isParent())) {\n _this.findCompoundLoopPoints(_edge, passedPairInfo, _i2, _edgeIsUnbundled);\n } else if (src === tgt) {\n _this.findLoopPoints(_edge, passedPairInfo, _i2, _edgeIsUnbundled);\n } else if (_curveStyle.endsWith('segments')) {\n _this.findSegmentsPoints(_edge, passedPairInfo);\n } else if (_curveStyle.endsWith('taxi')) {\n _this.findTaxiPoints(_edge, passedPairInfo);\n } else if (_curveStyle === 'straight' || !_edgeIsUnbundled && pairInfo.eles.length % 2 === 1 && _i2 === Math.floor(pairInfo.eles.length / 2)) {\n _this.findStraightEdgePoints(_edge);\n } else {\n _this.findBezierPoints(_edge, passedPairInfo, _i2, _edgeIsUnbundled, edgeIsSwapped);\n }\n _this.findEndpoints(_edge);\n _this.tryToCorrectInvalidPoints(_edge, passedPairInfo);\n _this.checkForInvalidEdgeWarning(_edge);\n _this.storeAllpts(_edge);\n _this.storeEdgeProjections(_edge);\n _this.calculateArrowAngles(_edge);\n _this.recalculateEdgeLabelProjections(_edge);\n _this.calculateLabelAngles(_edge);\n } // for pair edges\n };\n for (var p = 0; p < pairIds.length; p++) {\n _loop();\n } // for pair ids\n\n // haystacks avoid the expense of pairInfo stuff (intersections etc.)\n this.findHaystackPoints(haystackEdges);\n};\nfunction getPts(pts) {\n var retPts = [];\n if (pts == null) {\n return;\n }\n for (var i = 0; i < pts.length; i += 2) {\n var x = pts[i];\n var y = pts[i + 1];\n retPts.push({\n x: x,\n y: y\n });\n }\n return retPts;\n}\nBRp$c.getSegmentPoints = function (edge) {\n var rs = edge[0]._private.rscratch;\n this.recalculateRenderedStyle(edge);\n var type = rs.edgeType;\n if (type === 'segments') {\n return getPts(rs.segpts);\n }\n};\nBRp$c.getControlPoints = function (edge) {\n var rs = edge[0]._private.rscratch;\n this.recalculateRenderedStyle(edge);\n var type = rs.edgeType;\n if (type === 'bezier' || type === 'multibezier' || type === 'self' || type === 'compound') {\n return getPts(rs.ctrlpts);\n }\n};\nBRp$c.getEdgeMidpoint = function (edge) {\n var rs = edge[0]._private.rscratch;\n this.recalculateRenderedStyle(edge);\n return {\n x: rs.midX,\n y: rs.midY\n };\n};\n\nvar BRp$b = {};\nBRp$b.manualEndptToPx = function (node, prop) {\n var r = this;\n var npos = node.position();\n var w = node.outerWidth();\n var h = node.outerHeight();\n var rs = node._private.rscratch;\n if (prop.value.length === 2) {\n var p = [prop.pfValue[0], prop.pfValue[1]];\n if (prop.units[0] === '%') {\n p[0] = p[0] * w;\n }\n if (prop.units[1] === '%') {\n p[1] = p[1] * h;\n }\n p[0] += npos.x;\n p[1] += npos.y;\n return p;\n } else {\n var angle = prop.pfValue[0];\n angle = -Math.PI / 2 + angle; // start at 12 o'clock\n\n var l = 2 * Math.max(w, h);\n var _p = [npos.x + Math.cos(angle) * l, npos.y + Math.sin(angle) * l];\n return r.nodeShapes[this.getNodeShape(node)].intersectLine(npos.x, npos.y, w, h, _p[0], _p[1], 0, node.pstyle('corner-radius').value === 'auto' ? 'auto' : node.pstyle('corner-radius').pfValue, rs);\n }\n};\nBRp$b.findEndpoints = function (edge) {\n var _ref, _tgtManEndpt$pfValue, _ref2, _srcManEndpt$pfValue;\n var r = this;\n var intersect;\n var source = edge.source()[0];\n var target = edge.target()[0];\n var srcPos = source.position();\n var tgtPos = target.position();\n var tgtArShape = edge.pstyle('target-arrow-shape').value;\n var srcArShape = edge.pstyle('source-arrow-shape').value;\n var tgtDist = edge.pstyle('target-distance-from-node').pfValue;\n var srcDist = edge.pstyle('source-distance-from-node').pfValue;\n var srcRs = source._private.rscratch;\n var tgtRs = target._private.rscratch;\n var curveStyle = edge.pstyle('curve-style').value;\n var rs = edge._private.rscratch;\n var et = rs.edgeType;\n var taxi = endsWith(curveStyle, 'taxi'); // Covers taxi and round-taxi\n var self = et === 'self' || et === 'compound';\n var bezier = et === 'bezier' || et === 'multibezier' || self;\n var multi = et !== 'bezier';\n var lines = et === 'straight' || et === 'segments';\n var segments = et === 'segments';\n var hasEndpts = bezier || multi || lines;\n var overrideEndpts = self || taxi;\n var srcManEndpt = edge.pstyle('source-endpoint');\n var srcManEndptVal = overrideEndpts ? 'outside-to-node' : srcManEndpt.value;\n var srcCornerRadius = source.pstyle('corner-radius').value === 'auto' ? 'auto' : source.pstyle('corner-radius').pfValue;\n var tgtManEndpt = edge.pstyle('target-endpoint');\n var tgtManEndptVal = overrideEndpts ? 'outside-to-node' : tgtManEndpt.value;\n var tgtCornerRadius = target.pstyle('corner-radius').value === 'auto' ? 'auto' : target.pstyle('corner-radius').pfValue;\n rs.srcManEndpt = srcManEndpt;\n rs.tgtManEndpt = tgtManEndpt;\n var p1; // last known point of edge on target side\n var p2; // last known point of edge on source side\n\n var p1_i; // point to intersect with target shape\n var p2_i; // point to intersect with source shape\n\n var tgtManEndptPt = (_ref = (tgtManEndpt === null || tgtManEndpt === undefined || (_tgtManEndpt$pfValue = tgtManEndpt.pfValue) === null || _tgtManEndpt$pfValue === undefined ? undefined : _tgtManEndpt$pfValue.length) === 2 ? tgtManEndpt.pfValue : null) !== null && _ref !== undefined ? _ref : [0, 0];\n var srcManEndptPt = (_ref2 = (srcManEndpt === null || srcManEndpt === undefined || (_srcManEndpt$pfValue = srcManEndpt.pfValue) === null || _srcManEndpt$pfValue === undefined ? undefined : _srcManEndpt$pfValue.length) === 2 ? srcManEndpt.pfValue : null) !== null && _ref2 !== undefined ? _ref2 : [0, 0];\n if (bezier) {\n var cpStart = [rs.ctrlpts[0], rs.ctrlpts[1]];\n var cpEnd = multi ? [rs.ctrlpts[rs.ctrlpts.length - 2], rs.ctrlpts[rs.ctrlpts.length - 1]] : cpStart;\n p1 = cpEnd;\n p2 = cpStart;\n } else if (lines) {\n var srcArrowFromPt = !segments ? [tgtPos.x + tgtManEndptPt[0], tgtPos.y + tgtManEndptPt[1]] : rs.segpts.slice(0, 2);\n var tgtArrowFromPt = !segments ? [srcPos.x + srcManEndptPt[0], srcPos.y + srcManEndptPt[1]] : rs.segpts.slice(rs.segpts.length - 2);\n p1 = tgtArrowFromPt;\n p2 = srcArrowFromPt;\n }\n if (tgtManEndptVal === 'inside-to-node') {\n intersect = [tgtPos.x, tgtPos.y];\n } else if (tgtManEndpt.units) {\n intersect = this.manualEndptToPx(target, tgtManEndpt);\n } else if (tgtManEndptVal === 'outside-to-line') {\n intersect = rs.tgtIntn; // use cached value from ctrlpt calc\n } else {\n if (tgtManEndptVal === 'outside-to-node' || tgtManEndptVal === 'outside-to-node-or-label') {\n p1_i = p1;\n } else if (tgtManEndptVal === 'outside-to-line' || tgtManEndptVal === 'outside-to-line-or-label') {\n p1_i = [srcPos.x, srcPos.y];\n }\n intersect = r.nodeShapes[this.getNodeShape(target)].intersectLine(tgtPos.x, tgtPos.y, target.outerWidth(), target.outerHeight(), p1_i[0], p1_i[1], 0, tgtCornerRadius, tgtRs);\n if (tgtManEndptVal === 'outside-to-node-or-label' || tgtManEndptVal === 'outside-to-line-or-label') {\n var trs = target._private.rscratch;\n var lw = trs.labelWidth;\n var lh = trs.labelHeight;\n var lx = trs.labelX;\n var ly = trs.labelY;\n var lw2 = lw / 2;\n var lh2 = lh / 2;\n var va = target.pstyle('text-valign').value;\n if (va === 'top') {\n ly -= lh2;\n } else if (va === 'bottom') {\n ly += lh2;\n }\n var ha = target.pstyle('text-halign').value;\n if (ha === 'left') {\n lx -= lw2;\n } else if (ha === 'right') {\n lx += lw2;\n }\n var labelIntersect = polygonIntersectLine(p1_i[0], p1_i[1], [lx - lw2, ly - lh2, lx + lw2, ly - lh2, lx + lw2, ly + lh2, lx - lw2, ly + lh2], tgtPos.x, tgtPos.y);\n if (labelIntersect.length > 0) {\n var refPt = srcPos;\n var intSqdist = sqdist(refPt, array2point(intersect));\n var labIntSqdist = sqdist(refPt, array2point(labelIntersect));\n var minSqDist = intSqdist;\n if (labIntSqdist < intSqdist) {\n intersect = labelIntersect;\n minSqDist = labIntSqdist;\n }\n if (labelIntersect.length > 2) {\n var labInt2SqDist = sqdist(refPt, {\n x: labelIntersect[2],\n y: labelIntersect[3]\n });\n if (labInt2SqDist < minSqDist) {\n intersect = [labelIntersect[2], labelIntersect[3]];\n }\n }\n }\n }\n }\n var arrowEnd = shortenIntersection(intersect, p1, r.arrowShapes[tgtArShape].spacing(edge) + tgtDist);\n var edgeEnd = shortenIntersection(intersect, p1, r.arrowShapes[tgtArShape].gap(edge) + tgtDist);\n rs.endX = edgeEnd[0];\n rs.endY = edgeEnd[1];\n rs.arrowEndX = arrowEnd[0];\n rs.arrowEndY = arrowEnd[1];\n if (srcManEndptVal === 'inside-to-node') {\n intersect = [srcPos.x, srcPos.y];\n } else if (srcManEndpt.units) {\n intersect = this.manualEndptToPx(source, srcManEndpt);\n } else if (srcManEndptVal === 'outside-to-line') {\n intersect = rs.srcIntn; // use cached value from ctrlpt calc\n } else {\n if (srcManEndptVal === 'outside-to-node' || srcManEndptVal === 'outside-to-node-or-label') {\n p2_i = p2;\n } else if (srcManEndptVal === 'outside-to-line' || srcManEndptVal === 'outside-to-line-or-label') {\n p2_i = [tgtPos.x, tgtPos.y];\n }\n intersect = r.nodeShapes[this.getNodeShape(source)].intersectLine(srcPos.x, srcPos.y, source.outerWidth(), source.outerHeight(), p2_i[0], p2_i[1], 0, srcCornerRadius, srcRs);\n if (srcManEndptVal === 'outside-to-node-or-label' || srcManEndptVal === 'outside-to-line-or-label') {\n var srs = source._private.rscratch;\n var _lw = srs.labelWidth;\n var _lh = srs.labelHeight;\n var _lx = srs.labelX;\n var _ly = srs.labelY;\n var _lw2 = _lw / 2;\n var _lh2 = _lh / 2;\n var _va = source.pstyle('text-valign').value;\n if (_va === 'top') {\n _ly -= _lh2;\n } else if (_va === 'bottom') {\n _ly += _lh2;\n }\n var _ha = source.pstyle('text-halign').value;\n if (_ha === 'left') {\n _lx -= _lw2;\n } else if (_ha === 'right') {\n _lx += _lw2;\n }\n var _labelIntersect = polygonIntersectLine(p2_i[0], p2_i[1], [_lx - _lw2, _ly - _lh2, _lx + _lw2, _ly - _lh2, _lx + _lw2, _ly + _lh2, _lx - _lw2, _ly + _lh2], srcPos.x, srcPos.y);\n if (_labelIntersect.length > 0) {\n var _refPt = tgtPos;\n var _intSqdist = sqdist(_refPt, array2point(intersect));\n var _labIntSqdist = sqdist(_refPt, array2point(_labelIntersect));\n var _minSqDist = _intSqdist;\n if (_labIntSqdist < _intSqdist) {\n intersect = [_labelIntersect[0], _labelIntersect[1]];\n _minSqDist = _labIntSqdist;\n }\n if (_labelIntersect.length > 2) {\n var _labInt2SqDist = sqdist(_refPt, {\n x: _labelIntersect[2],\n y: _labelIntersect[3]\n });\n if (_labInt2SqDist < _minSqDist) {\n intersect = [_labelIntersect[2], _labelIntersect[3]];\n }\n }\n }\n }\n }\n var arrowStart = shortenIntersection(intersect, p2, r.arrowShapes[srcArShape].spacing(edge) + srcDist);\n var edgeStart = shortenIntersection(intersect, p2, r.arrowShapes[srcArShape].gap(edge) + srcDist);\n rs.startX = edgeStart[0];\n rs.startY = edgeStart[1];\n rs.arrowStartX = arrowStart[0];\n rs.arrowStartY = arrowStart[1];\n if (hasEndpts) {\n if (!number$1(rs.startX) || !number$1(rs.startY) || !number$1(rs.endX) || !number$1(rs.endY)) {\n rs.badLine = true;\n } else {\n rs.badLine = false;\n }\n }\n};\nBRp$b.getSourceEndpoint = function (edge) {\n var rs = edge[0]._private.rscratch;\n this.recalculateRenderedStyle(edge);\n switch (rs.edgeType) {\n case 'haystack':\n return {\n x: rs.haystackPts[0],\n y: rs.haystackPts[1]\n };\n default:\n return {\n x: rs.arrowStartX,\n y: rs.arrowStartY\n };\n }\n};\nBRp$b.getTargetEndpoint = function (edge) {\n var rs = edge[0]._private.rscratch;\n this.recalculateRenderedStyle(edge);\n switch (rs.edgeType) {\n case 'haystack':\n return {\n x: rs.haystackPts[2],\n y: rs.haystackPts[3]\n };\n default:\n return {\n x: rs.arrowEndX,\n y: rs.arrowEndY\n };\n }\n};\n\nvar BRp$a = {};\nfunction pushBezierPts(r, edge, pts) {\n var qbezierAt$1 = function qbezierAt$1(p1, p2, p3, t) {\n return qbezierAt(p1, p2, p3, t);\n };\n var _p = edge._private;\n var bpts = _p.rstyle.bezierPts;\n for (var i = 0; i < r.bezierProjPcts.length; i++) {\n var p = r.bezierProjPcts[i];\n bpts.push({\n x: qbezierAt$1(pts[0], pts[2], pts[4], p),\n y: qbezierAt$1(pts[1], pts[3], pts[5], p)\n });\n }\n}\nBRp$a.storeEdgeProjections = function (edge) {\n var _p = edge._private;\n var rs = _p.rscratch;\n var et = rs.edgeType;\n\n // clear the cached points state\n _p.rstyle.bezierPts = null;\n _p.rstyle.linePts = null;\n _p.rstyle.haystackPts = null;\n if (et === 'multibezier' || et === 'bezier' || et === 'self' || et === 'compound') {\n _p.rstyle.bezierPts = [];\n for (var i = 0; i + 5 < rs.allpts.length; i += 4) {\n pushBezierPts(this, edge, rs.allpts.slice(i, i + 6));\n }\n } else if (et === 'segments') {\n var lpts = _p.rstyle.linePts = [];\n for (var i = 0; i + 1 < rs.allpts.length; i += 2) {\n lpts.push({\n x: rs.allpts[i],\n y: rs.allpts[i + 1]\n });\n }\n } else if (et === 'haystack') {\n var hpts = rs.haystackPts;\n _p.rstyle.haystackPts = [{\n x: hpts[0],\n y: hpts[1]\n }, {\n x: hpts[2],\n y: hpts[3]\n }];\n }\n _p.rstyle.arrowWidth = this.getArrowWidth(edge.pstyle('width').pfValue, edge.pstyle('arrow-scale').value) * this.arrowShapeWidth;\n};\nBRp$a.recalculateEdgeProjections = function (edges) {\n this.findEdgeControlPoints(edges);\n};\n\nvar BRp$9 = {};\nBRp$9.recalculateNodeLabelProjection = function (node) {\n var content = node.pstyle('label').strValue;\n if (emptyString(content)) {\n return;\n }\n var textX, textY;\n var _p = node._private;\n var nodeWidth = node.width();\n var nodeHeight = node.height();\n var padding = node.padding();\n var nodePos = node.position();\n var textHalign = node.pstyle('text-halign').strValue;\n var textValign = node.pstyle('text-valign').strValue;\n var rs = _p.rscratch;\n var rstyle = _p.rstyle;\n switch (textHalign) {\n case 'left':\n textX = nodePos.x - nodeWidth / 2 - padding;\n break;\n case 'right':\n textX = nodePos.x + nodeWidth / 2 + padding;\n break;\n default:\n // e.g. center\n textX = nodePos.x;\n }\n switch (textValign) {\n case 'top':\n textY = nodePos.y - nodeHeight / 2 - padding;\n break;\n case 'bottom':\n textY = nodePos.y + nodeHeight / 2 + padding;\n break;\n default:\n // e.g. middle\n textY = nodePos.y;\n }\n rs.labelX = textX;\n rs.labelY = textY;\n rstyle.labelX = textX;\n rstyle.labelY = textY;\n this.calculateLabelAngles(node);\n this.applyLabelDimensions(node);\n};\nvar lineAngleFromDelta = function lineAngleFromDelta(dx, dy) {\n var angle = Math.atan(dy / dx);\n if (dx === 0 && angle < 0) {\n angle = angle * -1;\n }\n return angle;\n};\nvar lineAngle = function lineAngle(p0, p1) {\n var dx = p1.x - p0.x;\n var dy = p1.y - p0.y;\n return lineAngleFromDelta(dx, dy);\n};\nvar bezierAngle = function bezierAngle(p0, p1, p2, t) {\n var t0 = bound(0, t - 0.001, 1);\n var t1 = bound(0, t + 0.001, 1);\n var lp0 = qbezierPtAt(p0, p1, p2, t0);\n var lp1 = qbezierPtAt(p0, p1, p2, t1);\n return lineAngle(lp0, lp1);\n};\nBRp$9.recalculateEdgeLabelProjections = function (edge) {\n var p;\n var _p = edge._private;\n var rs = _p.rscratch;\n var r = this;\n var content = {\n mid: edge.pstyle('label').strValue,\n source: edge.pstyle('source-label').strValue,\n target: edge.pstyle('target-label').strValue\n };\n if (content.mid || content.source || content.target) ; else {\n return; // no labels => no calcs\n }\n\n // add center point to style so bounding box calculations can use it\n //\n p = {\n x: rs.midX,\n y: rs.midY\n };\n var setRs = function setRs(propName, prefix, value) {\n setPrefixedProperty(_p.rscratch, propName, prefix, value);\n setPrefixedProperty(_p.rstyle, propName, prefix, value);\n };\n setRs('labelX', null, p.x);\n setRs('labelY', null, p.y);\n var midAngle = lineAngleFromDelta(rs.midDispX, rs.midDispY);\n setRs('labelAutoAngle', null, midAngle);\n var _createControlPointInfo = function createControlPointInfo() {\n if (_createControlPointInfo.cache) {\n return _createControlPointInfo.cache;\n } // use cache so only 1x per edge\n\n var ctrlpts = [];\n\n // store each ctrlpt info init\n for (var i = 0; i + 5 < rs.allpts.length; i += 4) {\n var p0 = {\n x: rs.allpts[i],\n y: rs.allpts[i + 1]\n };\n var p1 = {\n x: rs.allpts[i + 2],\n y: rs.allpts[i + 3]\n }; // ctrlpt\n var p2 = {\n x: rs.allpts[i + 4],\n y: rs.allpts[i + 5]\n };\n ctrlpts.push({\n p0: p0,\n p1: p1,\n p2: p2,\n startDist: 0,\n length: 0,\n segments: []\n });\n }\n var bpts = _p.rstyle.bezierPts;\n var nProjs = r.bezierProjPcts.length;\n function addSegment(cp, p0, p1, t0, t1) {\n var length = dist(p0, p1);\n var prevSegment = cp.segments[cp.segments.length - 1];\n var segment = {\n p0: p0,\n p1: p1,\n t0: t0,\n t1: t1,\n startDist: prevSegment ? prevSegment.startDist + prevSegment.length : 0,\n length: length\n };\n cp.segments.push(segment);\n cp.length += length;\n }\n\n // update each ctrlpt with segment info\n for (var _i = 0; _i < ctrlpts.length; _i++) {\n var cp = ctrlpts[_i];\n var prevCp = ctrlpts[_i - 1];\n if (prevCp) {\n cp.startDist = prevCp.startDist + prevCp.length;\n }\n addSegment(cp, cp.p0, bpts[_i * nProjs], 0, r.bezierProjPcts[0]); // first\n\n for (var j = 0; j < nProjs - 1; j++) {\n addSegment(cp, bpts[_i * nProjs + j], bpts[_i * nProjs + j + 1], r.bezierProjPcts[j], r.bezierProjPcts[j + 1]);\n }\n addSegment(cp, bpts[_i * nProjs + nProjs - 1], cp.p2, r.bezierProjPcts[nProjs - 1], 1); // last\n }\n return _createControlPointInfo.cache = ctrlpts;\n };\n var calculateEndProjection = function calculateEndProjection(prefix) {\n var angle;\n var isSrc = prefix === 'source';\n if (!content[prefix]) {\n return;\n }\n var offset = edge.pstyle(prefix + '-text-offset').pfValue;\n switch (rs.edgeType) {\n case 'self':\n case 'compound':\n case 'bezier':\n case 'multibezier':\n {\n var cps = _createControlPointInfo();\n var selected;\n var startDist = 0;\n var totalDist = 0;\n\n // find the segment we're on\n for (var i = 0; i < cps.length; i++) {\n var _cp = cps[isSrc ? i : cps.length - 1 - i];\n for (var j = 0; j < _cp.segments.length; j++) {\n var _seg = _cp.segments[isSrc ? j : _cp.segments.length - 1 - j];\n var lastSeg = i === cps.length - 1 && j === _cp.segments.length - 1;\n startDist = totalDist;\n totalDist += _seg.length;\n if (totalDist >= offset || lastSeg) {\n selected = {\n cp: _cp,\n segment: _seg\n };\n break;\n }\n }\n if (selected) {\n break;\n }\n }\n var cp = selected.cp;\n var seg = selected.segment;\n var tSegment = (offset - startDist) / seg.length;\n var segDt = seg.t1 - seg.t0;\n var t = isSrc ? seg.t0 + segDt * tSegment : seg.t1 - segDt * tSegment;\n t = bound(0, t, 1);\n p = qbezierPtAt(cp.p0, cp.p1, cp.p2, t);\n angle = bezierAngle(cp.p0, cp.p1, cp.p2, t);\n break;\n }\n case 'straight':\n case 'segments':\n case 'haystack':\n {\n var d = 0,\n di,\n d0;\n var p0, p1;\n var l = rs.allpts.length;\n for (var _i2 = 0; _i2 + 3 < l; _i2 += 2) {\n if (isSrc) {\n p0 = {\n x: rs.allpts[_i2],\n y: rs.allpts[_i2 + 1]\n };\n p1 = {\n x: rs.allpts[_i2 + 2],\n y: rs.allpts[_i2 + 3]\n };\n } else {\n p0 = {\n x: rs.allpts[l - 2 - _i2],\n y: rs.allpts[l - 1 - _i2]\n };\n p1 = {\n x: rs.allpts[l - 4 - _i2],\n y: rs.allpts[l - 3 - _i2]\n };\n }\n di = dist(p0, p1);\n d0 = d;\n d += di;\n if (d >= offset) {\n break;\n }\n }\n var pD = offset - d0;\n var _t = pD / di;\n _t = bound(0, _t, 1);\n p = lineAt(p0, p1, _t);\n angle = lineAngle(p0, p1);\n break;\n }\n }\n setRs('labelX', prefix, p.x);\n setRs('labelY', prefix, p.y);\n setRs('labelAutoAngle', prefix, angle);\n };\n calculateEndProjection('source');\n calculateEndProjection('target');\n this.applyLabelDimensions(edge);\n};\nBRp$9.applyLabelDimensions = function (ele) {\n this.applyPrefixedLabelDimensions(ele);\n if (ele.isEdge()) {\n this.applyPrefixedLabelDimensions(ele, 'source');\n this.applyPrefixedLabelDimensions(ele, 'target');\n }\n};\nBRp$9.applyPrefixedLabelDimensions = function (ele, prefix) {\n var _p = ele._private;\n var text = this.getLabelText(ele, prefix);\n var cacheKey = hashString(text, ele._private.labelDimsKey);\n\n // save recalc if the label is the same as before\n if (getPrefixedProperty(_p.rscratch, 'prefixedLabelDimsKey', prefix) === cacheKey) {\n return; // then the label dimensions + text are the same\n }\n\n // save the key\n setPrefixedProperty(_p.rscratch, 'prefixedLabelDimsKey', prefix, cacheKey);\n var labelDims = this.calculateLabelDimensions(ele, text);\n var lineHeight = ele.pstyle('line-height').pfValue;\n var textWrap = ele.pstyle('text-wrap').strValue;\n var lines = getPrefixedProperty(_p.rscratch, 'labelWrapCachedLines', prefix) || [];\n var numLines = textWrap !== 'wrap' ? 1 : Math.max(lines.length, 1);\n var normPerLineHeight = labelDims.height / numLines;\n var labelLineHeight = normPerLineHeight * lineHeight;\n var width = labelDims.width;\n var height = labelDims.height + (numLines - 1) * (lineHeight - 1) * normPerLineHeight;\n setPrefixedProperty(_p.rstyle, 'labelWidth', prefix, width);\n setPrefixedProperty(_p.rscratch, 'labelWidth', prefix, width);\n setPrefixedProperty(_p.rstyle, 'labelHeight', prefix, height);\n setPrefixedProperty(_p.rscratch, 'labelHeight', prefix, height);\n setPrefixedProperty(_p.rscratch, 'labelLineHeight', prefix, labelLineHeight);\n};\nBRp$9.getLabelText = function (ele, prefix) {\n var _p = ele._private;\n var pfd = prefix ? prefix + '-' : '';\n var text = ele.pstyle(pfd + 'label').strValue;\n var textTransform = ele.pstyle('text-transform').value;\n var rscratch = function rscratch(propName, value) {\n if (value) {\n setPrefixedProperty(_p.rscratch, propName, prefix, value);\n return value;\n } else {\n return getPrefixedProperty(_p.rscratch, propName, prefix);\n }\n };\n\n // for empty text, skip all processing\n if (!text) {\n return '';\n }\n if (textTransform == 'none') ; else if (textTransform == 'uppercase') {\n text = text.toUpperCase();\n } else if (textTransform == 'lowercase') {\n text = text.toLowerCase();\n }\n var wrapStyle = ele.pstyle('text-wrap').value;\n if (wrapStyle === 'wrap') {\n var labelKey = rscratch('labelKey');\n\n // save recalc if the label is the same as before\n if (labelKey != null && rscratch('labelWrapKey') === labelKey) {\n return rscratch('labelWrapCachedText');\n }\n var zwsp = \"\\u200B\";\n var lines = text.split('\\n');\n var maxW = ele.pstyle('text-max-width').pfValue;\n var overflow = ele.pstyle('text-overflow-wrap').value;\n var overflowAny = overflow === 'anywhere';\n var wrappedLines = [];\n var separatorRegex = /[\\s\\u200b]+|$/g; // Include end of string to add last word\n\n for (var l = 0; l < lines.length; l++) {\n var line = lines[l];\n var lineDims = this.calculateLabelDimensions(ele, line);\n var lineW = lineDims.width;\n if (overflowAny) {\n var processedLine = line.split('').join(zwsp);\n line = processedLine;\n }\n if (lineW > maxW) {\n // line is too long\n var separatorMatches = line.matchAll(separatorRegex);\n var subline = '';\n var previousIndex = 0;\n // Add fake match\n var _iterator = _createForOfIteratorHelper(separatorMatches),\n _step;\n try {\n for (_iterator.s(); !(_step = _iterator.n()).done;) {\n var separatorMatch = _step.value;\n var wordSeparator = separatorMatch[0];\n var word = line.substring(previousIndex, separatorMatch.index);\n previousIndex = separatorMatch.index + wordSeparator.length;\n var testLine = subline.length === 0 ? word : subline + word + wordSeparator;\n var testDims = this.calculateLabelDimensions(ele, testLine);\n var testW = testDims.width;\n if (testW <= maxW) {\n // word fits on current line\n subline += word + wordSeparator;\n } else {\n // word starts new line\n if (subline) {\n wrappedLines.push(subline);\n }\n subline = word + wordSeparator;\n }\n }\n\n // if there's remaining text, put it in a wrapped line\n } catch (err) {\n _iterator.e(err);\n } finally {\n _iterator.f();\n }\n if (!subline.match(/^[\\s\\u200b]+$/)) {\n wrappedLines.push(subline);\n }\n } else {\n // line is already short enough\n wrappedLines.push(line);\n }\n } // for\n\n rscratch('labelWrapCachedLines', wrappedLines);\n text = rscratch('labelWrapCachedText', wrappedLines.join('\\n'));\n rscratch('labelWrapKey', labelKey);\n } else if (wrapStyle === 'ellipsis') {\n var _maxW = ele.pstyle('text-max-width').pfValue;\n var ellipsized = '';\n var ellipsis = \"\\u2026\";\n var incLastCh = false;\n if (this.calculateLabelDimensions(ele, text).width < _maxW) {\n // the label already fits\n return text;\n }\n for (var i = 0; i < text.length; i++) {\n var widthWithNextCh = this.calculateLabelDimensions(ele, ellipsized + text[i] + ellipsis).width;\n if (widthWithNextCh > _maxW) {\n break;\n }\n ellipsized += text[i];\n if (i === text.length - 1) {\n incLastCh = true;\n }\n }\n if (!incLastCh) {\n ellipsized += ellipsis;\n }\n return ellipsized;\n } // if ellipsize\n\n return text;\n};\nBRp$9.getLabelJustification = function (ele) {\n var justification = ele.pstyle('text-justification').strValue;\n var textHalign = ele.pstyle('text-halign').strValue;\n if (justification === 'auto') {\n if (ele.isNode()) {\n switch (textHalign) {\n case 'left':\n return 'right';\n case 'right':\n return 'left';\n default:\n return 'center';\n }\n } else {\n return 'center';\n }\n } else {\n return justification;\n }\n};\nBRp$9.calculateLabelDimensions = function (ele, text) {\n var r = this;\n var containerWindow = r.cy.window();\n var document = containerWindow.document;\n var padding = 0; // add padding around text dims, as the measurement isn't that accurate\n var fStyle = ele.pstyle('font-style').strValue;\n var size = ele.pstyle('font-size').pfValue;\n var family = ele.pstyle('font-family').strValue;\n var weight = ele.pstyle('font-weight').strValue;\n var canvas = this.labelCalcCanvas;\n var c2d = this.labelCalcCanvasContext;\n if (!canvas) {\n canvas = this.labelCalcCanvas = document.createElement('canvas');\n c2d = this.labelCalcCanvasContext = canvas.getContext('2d');\n var ds = canvas.style;\n ds.position = 'absolute';\n ds.left = '-9999px';\n ds.top = '-9999px';\n ds.zIndex = '-1';\n ds.visibility = 'hidden';\n ds.pointerEvents = 'none';\n }\n c2d.font = \"\".concat(fStyle, \" \").concat(weight, \" \").concat(size, \"px \").concat(family);\n var width = 0;\n var height = 0;\n var lines = text.split('\\n');\n for (var i = 0; i < lines.length; i++) {\n var line = lines[i];\n var metrics = c2d.measureText(line);\n var w = Math.ceil(metrics.width);\n var h = size;\n width = Math.max(w, width);\n height += h;\n }\n width += padding;\n height += padding;\n return {\n width: width,\n height: height\n };\n};\nBRp$9.calculateLabelAngle = function (ele, prefix) {\n var _p = ele._private;\n var rs = _p.rscratch;\n var isEdge = ele.isEdge();\n var prefixDash = prefix ? prefix + '-' : '';\n var rot = ele.pstyle(prefixDash + 'text-rotation');\n var rotStr = rot.strValue;\n if (rotStr === 'none') {\n return 0;\n } else if (isEdge && rotStr === 'autorotate') {\n return rs.labelAutoAngle;\n } else if (rotStr === 'autorotate') {\n return 0;\n } else {\n return rot.pfValue;\n }\n};\nBRp$9.calculateLabelAngles = function (ele) {\n var r = this;\n var isEdge = ele.isEdge();\n var _p = ele._private;\n var rs = _p.rscratch;\n rs.labelAngle = r.calculateLabelAngle(ele);\n if (isEdge) {\n rs.sourceLabelAngle = r.calculateLabelAngle(ele, 'source');\n rs.targetLabelAngle = r.calculateLabelAngle(ele, 'target');\n }\n};\n\nvar BRp$8 = {};\nvar TOO_SMALL_CUT_RECT = 28;\nvar warnedCutRect = false;\nBRp$8.getNodeShape = function (node) {\n var r = this;\n var shape = node.pstyle('shape').value;\n if (shape === 'cutrectangle' && (node.width() < TOO_SMALL_CUT_RECT || node.height() < TOO_SMALL_CUT_RECT)) {\n if (!warnedCutRect) {\n warn('The `cutrectangle` node shape can not be used at small sizes so `rectangle` is used instead');\n warnedCutRect = true;\n }\n return 'rectangle';\n }\n if (node.isParent()) {\n if (shape === 'rectangle' || shape === 'roundrectangle' || shape === 'round-rectangle' || shape === 'cutrectangle' || shape === 'cut-rectangle' || shape === 'barrel') {\n return shape;\n } else {\n return 'rectangle';\n }\n }\n if (shape === 'polygon') {\n var points = node.pstyle('shape-polygon-points').value;\n return r.nodeShapes.makePolygon(points).name;\n }\n return shape;\n};\n\nvar BRp$7 = {};\nBRp$7.registerCalculationListeners = function () {\n var cy = this.cy;\n var elesToUpdate = cy.collection();\n var r = this;\n var enqueue = function enqueue(eles) {\n var dirtyStyleCaches = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : true;\n elesToUpdate.merge(eles);\n if (dirtyStyleCaches) {\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n var _p = ele._private;\n var rstyle = _p.rstyle;\n rstyle.clean = false;\n rstyle.cleanConnected = false;\n }\n }\n };\n r.binder(cy).on('bounds.* dirty.*', function onDirtyBounds(e) {\n var ele = e.target;\n enqueue(ele);\n }).on('style.* background.*', function onDirtyStyle(e) {\n var ele = e.target;\n enqueue(ele, false);\n });\n var updateEleCalcs = function updateEleCalcs(willDraw) {\n if (willDraw) {\n var fns = r.onUpdateEleCalcsFns;\n\n // because we need to have up-to-date style (e.g. stylesheet mappers)\n // before calculating rendered style (and pstyle might not be called yet)\n elesToUpdate.cleanStyle();\n for (var i = 0; i < elesToUpdate.length; i++) {\n var ele = elesToUpdate[i];\n var rstyle = ele._private.rstyle;\n if (ele.isNode() && !rstyle.cleanConnected) {\n enqueue(ele.connectedEdges());\n rstyle.cleanConnected = true;\n }\n }\n if (fns) {\n for (var _i = 0; _i < fns.length; _i++) {\n var fn = fns[_i];\n fn(willDraw, elesToUpdate);\n }\n }\n r.recalculateRenderedStyle(elesToUpdate);\n elesToUpdate = cy.collection();\n }\n };\n r.flushRenderedStyleQueue = function () {\n updateEleCalcs(true);\n };\n r.beforeRender(updateEleCalcs, r.beforeRenderPriorities.eleCalcs);\n};\nBRp$7.onUpdateEleCalcs = function (fn) {\n var fns = this.onUpdateEleCalcsFns = this.onUpdateEleCalcsFns || [];\n fns.push(fn);\n};\nBRp$7.recalculateRenderedStyle = function (eles, useCache) {\n var isCleanConnected = function isCleanConnected(ele) {\n return ele._private.rstyle.cleanConnected;\n };\n if (eles.length === 0) {\n return;\n }\n var edges = [];\n var nodes = [];\n\n // the renderer can't be used for calcs when destroyed, e.g. ele.boundingBox()\n if (this.destroyed) {\n return;\n }\n\n // use cache by default for perf\n if (useCache === undefined) {\n useCache = true;\n }\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n var _p = ele._private;\n var rstyle = _p.rstyle;\n\n // an edge may be implicitly dirty b/c of one of its connected nodes\n // (and a request for recalc may come in between frames)\n if (ele.isEdge() && (!isCleanConnected(ele.source()) || !isCleanConnected(ele.target()))) {\n rstyle.clean = false;\n }\n if (ele.isEdge() && ele.isBundledBezier()) {\n if (ele.parallelEdges().some(function (ele) {\n return !ele._private.rstyle.clean && ele.isBundledBezier();\n })) {\n rstyle.clean = false;\n }\n }\n\n // only update if dirty and in graph\n if (useCache && rstyle.clean || ele.removed()) {\n continue;\n }\n\n // only update if not display: none\n if (ele.pstyle('display').value === 'none') {\n continue;\n }\n if (_p.group === 'nodes') {\n nodes.push(ele);\n } else {\n // edges\n edges.push(ele);\n }\n rstyle.clean = true;\n }\n\n // update node data from projections\n for (var _i2 = 0; _i2 < nodes.length; _i2++) {\n var _ele = nodes[_i2];\n var _p2 = _ele._private;\n var _rstyle = _p2.rstyle;\n var pos = _ele.position();\n this.recalculateNodeLabelProjection(_ele);\n _rstyle.nodeX = pos.x;\n _rstyle.nodeY = pos.y;\n _rstyle.nodeW = _ele.pstyle('width').pfValue;\n _rstyle.nodeH = _ele.pstyle('height').pfValue;\n }\n this.recalculateEdgeProjections(edges);\n\n // update edge data from projections\n for (var _i3 = 0; _i3 < edges.length; _i3++) {\n var _ele2 = edges[_i3];\n var _p3 = _ele2._private;\n var _rstyle2 = _p3.rstyle;\n var rs = _p3.rscratch;\n\n // update rstyle positions\n _rstyle2.srcX = rs.arrowStartX;\n _rstyle2.srcY = rs.arrowStartY;\n _rstyle2.tgtX = rs.arrowEndX;\n _rstyle2.tgtY = rs.arrowEndY;\n _rstyle2.midX = rs.midX;\n _rstyle2.midY = rs.midY;\n _rstyle2.labelAngle = rs.labelAngle;\n _rstyle2.sourceLabelAngle = rs.sourceLabelAngle;\n _rstyle2.targetLabelAngle = rs.targetLabelAngle;\n }\n};\n\nvar BRp$6 = {};\nBRp$6.updateCachedGrabbedEles = function () {\n var eles = this.cachedZSortedEles;\n if (!eles) {\n // just let this be recalculated on the next z sort tick\n return;\n }\n eles.drag = [];\n eles.nondrag = [];\n var grabTargets = [];\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n var rs = ele._private.rscratch;\n if (ele.grabbed() && !ele.isParent()) {\n grabTargets.push(ele);\n } else if (rs.inDragLayer) {\n eles.drag.push(ele);\n } else {\n eles.nondrag.push(ele);\n }\n }\n\n // put the grab target nodes last so it's on top of its neighbourhood\n for (var i = 0; i < grabTargets.length; i++) {\n var ele = grabTargets[i];\n eles.drag.push(ele);\n }\n};\nBRp$6.invalidateCachedZSortedEles = function () {\n this.cachedZSortedEles = null;\n};\nBRp$6.getCachedZSortedEles = function (forceRecalc) {\n if (forceRecalc || !this.cachedZSortedEles) {\n var eles = this.cy.mutableElements().toArray();\n eles.sort(zIndexSort);\n eles.interactive = eles.filter(function (ele) {\n return ele.interactive();\n });\n this.cachedZSortedEles = eles;\n this.updateCachedGrabbedEles();\n } else {\n eles = this.cachedZSortedEles;\n }\n return eles;\n};\n\nvar BRp$5 = {};\n[BRp$e, BRp$d, BRp$c, BRp$b, BRp$a, BRp$9, BRp$8, BRp$7, BRp$6].forEach(function (props) {\n extend(BRp$5, props);\n});\n\nvar BRp$4 = {};\nBRp$4.getCachedImage = function (url, crossOrigin, onLoad) {\n var r = this;\n var imageCache = r.imageCache = r.imageCache || {};\n var cache = imageCache[url];\n if (cache) {\n if (!cache.image.complete) {\n cache.image.addEventListener('load', onLoad);\n }\n return cache.image;\n } else {\n cache = imageCache[url] = imageCache[url] || {};\n var image = cache.image = new Image(); // eslint-disable-line no-undef\n\n image.addEventListener('load', onLoad);\n image.addEventListener('error', function () {\n image.error = true;\n });\n\n // #1582 safari doesn't load data uris with crossOrigin properly\n // https://bugs.webkit.org/show_bug.cgi?id=123978\n var dataUriPrefix = 'data:';\n var isDataUri = url.substring(0, dataUriPrefix.length).toLowerCase() === dataUriPrefix;\n if (!isDataUri) {\n // if crossorigin is 'null'(stringified), then manually set it to null \n crossOrigin = crossOrigin === 'null' ? null : crossOrigin;\n image.crossOrigin = crossOrigin; // prevent tainted canvas\n }\n image.src = url;\n return image;\n }\n};\n\nvar setGrabState = function setGrabState(ele, grabbed) {\n var ele0 = ele[0];\n if (!ele0 || ele0._private.grabbed === grabbed) {\n return;\n }\n ele0._private.grabbed = grabbed;\n ele.updateStyle(false);\n};\nvar setGrabbed = function setGrabbed(ele) {\n setGrabState(ele, true);\n};\nvar setFreed = function setFreed(ele) {\n setGrabState(ele, false);\n};\n\nvar BRp$3 = {};\n\n/* global document, ResizeObserver, MutationObserver */\n\nBRp$3.registerBinding = function (target, event, handler, useCapture) {\n // eslint-disable-line no-unused-vars\n var args = Array.prototype.slice.apply(arguments, [1]); // copy\n\n if (Array.isArray(target)) {\n var res = [];\n for (var i = 0; i < target.length; i++) {\n var t = target[i];\n if (t !== undefined) {\n var b = this.binder(t);\n res.push(b.on.apply(b, args));\n }\n }\n return res;\n }\n var b = this.binder(target);\n return b.on.apply(b, args);\n};\nBRp$3.binder = function (tgt) {\n var r = this;\n var containerWindow = r.cy.window();\n var tgtIsDom = tgt === containerWindow || tgt === containerWindow.document || tgt === containerWindow.document.body || domElement(tgt);\n if (r.supportsPassiveEvents == null) {\n // from https://github.com/WICG/EventListenerOptions/blob/gh-pages/explainer.md#feature-detection\n var supportsPassive = false;\n try {\n var opts = Object.defineProperty({}, 'passive', {\n get: function get() {\n supportsPassive = true;\n return true;\n }\n });\n containerWindow.addEventListener('test', null, opts);\n } catch (err) {\n // not supported\n }\n r.supportsPassiveEvents = supportsPassive;\n }\n var on = function on(event, handler, useCapture) {\n var args = Array.prototype.slice.call(arguments);\n if (tgtIsDom && r.supportsPassiveEvents) {\n // replace useCapture w/ opts obj\n args[2] = {\n capture: useCapture != null ? useCapture : false,\n passive: false,\n once: false\n };\n }\n r.bindings.push({\n target: tgt,\n args: args\n });\n (tgt.addEventListener || tgt.on).apply(tgt, args);\n return this;\n };\n return {\n on: on,\n addEventListener: on,\n addListener: on,\n bind: on\n };\n};\nBRp$3.nodeIsDraggable = function (node) {\n return node && node.isNode() && !node.locked() && node.grabbable();\n};\nBRp$3.nodeIsGrabbable = function (node) {\n return this.nodeIsDraggable(node) && node.interactive();\n};\nBRp$3.load = function () {\n var r = this;\n var containerWindow = r.cy.window();\n var isSelected = function isSelected(ele) {\n return ele.selected();\n };\n var getShadowRoot = function getShadowRoot(element) {\n var rootNode = element.getRootNode();\n // Check if the root node is a shadow root\n if (rootNode && rootNode.nodeType === 11 && rootNode.host !== undefined) {\n return rootNode;\n }\n };\n var triggerEvents = function triggerEvents(target, names, e, position) {\n if (target == null) {\n target = r.cy;\n }\n for (var i = 0; i < names.length; i++) {\n var name = names[i];\n target.emit({\n originalEvent: e,\n type: name,\n position: position\n });\n }\n };\n var isMultSelKeyDown = function isMultSelKeyDown(e) {\n return e.shiftKey || e.metaKey || e.ctrlKey; // maybe e.altKey\n };\n var allowPanningPassthrough = function allowPanningPassthrough(down, downs) {\n var allowPassthrough = true;\n if (r.cy.hasCompoundNodes() && down && down.pannable()) {\n // a grabbable compound node below the ele => no passthrough panning\n for (var i = 0; downs && i < downs.length; i++) {\n var down = downs[i];\n\n //if any parent node in event hierarchy isn't pannable, reject passthrough\n if (down.isNode() && down.isParent() && !down.pannable()) {\n allowPassthrough = false;\n break;\n }\n }\n } else {\n allowPassthrough = true;\n }\n return allowPassthrough;\n };\n var setInDragLayer = function setInDragLayer(ele) {\n ele[0]._private.rscratch.inDragLayer = true;\n };\n var setOutDragLayer = function setOutDragLayer(ele) {\n ele[0]._private.rscratch.inDragLayer = false;\n };\n var setGrabTarget = function setGrabTarget(ele) {\n ele[0]._private.rscratch.isGrabTarget = true;\n };\n var removeGrabTarget = function removeGrabTarget(ele) {\n ele[0]._private.rscratch.isGrabTarget = false;\n };\n var addToDragList = function addToDragList(ele, opts) {\n var list = opts.addToList;\n var listHasEle = list.has(ele);\n if (!listHasEle && ele.grabbable() && !ele.locked()) {\n list.merge(ele);\n setGrabbed(ele);\n }\n };\n\n // helper function to determine which child nodes and inner edges\n // of a compound node to be dragged as well as the grabbed and selected nodes\n var addDescendantsToDrag = function addDescendantsToDrag(node, opts) {\n if (!node.cy().hasCompoundNodes()) {\n return;\n }\n if (opts.inDragLayer == null && opts.addToList == null) {\n return;\n } // nothing to do\n\n var innerNodes = node.descendants();\n if (opts.inDragLayer) {\n innerNodes.forEach(setInDragLayer);\n innerNodes.connectedEdges().forEach(setInDragLayer);\n }\n if (opts.addToList) {\n addToDragList(innerNodes, opts);\n }\n };\n\n // adds the given nodes and its neighbourhood to the drag layer\n var addNodesToDrag = function addNodesToDrag(nodes, opts) {\n opts = opts || {};\n var hasCompoundNodes = nodes.cy().hasCompoundNodes();\n if (opts.inDragLayer) {\n nodes.forEach(setInDragLayer);\n nodes.neighborhood().stdFilter(function (ele) {\n return !hasCompoundNodes || ele.isEdge();\n }).forEach(setInDragLayer);\n }\n if (opts.addToList) {\n nodes.forEach(function (ele) {\n addToDragList(ele, opts);\n });\n }\n addDescendantsToDrag(nodes, opts); // always add to drag\n\n // also add nodes and edges related to the topmost ancestor\n updateAncestorsInDragLayer(nodes, {\n inDragLayer: opts.inDragLayer\n });\n r.updateCachedGrabbedEles();\n };\n var addNodeToDrag = addNodesToDrag;\n var freeDraggedElements = function freeDraggedElements(grabbedEles) {\n if (!grabbedEles) {\n return;\n }\n\n // just go over all elements rather than doing a bunch of (possibly expensive) traversals\n r.getCachedZSortedEles().forEach(function (ele) {\n setFreed(ele);\n setOutDragLayer(ele);\n removeGrabTarget(ele);\n });\n r.updateCachedGrabbedEles();\n };\n\n // helper function to determine which ancestor nodes and edges should go\n // to the drag layer (or should be removed from drag layer).\n var updateAncestorsInDragLayer = function updateAncestorsInDragLayer(node, opts) {\n if (opts.inDragLayer == null && opts.addToList == null) {\n return;\n } // nothing to do\n\n if (!node.cy().hasCompoundNodes()) {\n return;\n }\n\n // find top-level parent\n var parent = node.ancestors().orphans();\n\n // no parent node: no nodes to add to the drag layer\n if (parent.same(node)) {\n return;\n }\n var nodes = parent.descendants().spawnSelf().merge(parent).unmerge(node).unmerge(node.descendants());\n var edges = nodes.connectedEdges();\n if (opts.inDragLayer) {\n edges.forEach(setInDragLayer);\n nodes.forEach(setInDragLayer);\n }\n if (opts.addToList) {\n nodes.forEach(function (ele) {\n addToDragList(ele, opts);\n });\n }\n };\n var blurActiveDomElement = function blurActiveDomElement() {\n if (document.activeElement != null && document.activeElement.blur != null) {\n document.activeElement.blur();\n }\n };\n var haveMutationsApi = typeof MutationObserver !== 'undefined';\n var haveResizeObserverApi = typeof ResizeObserver !== 'undefined';\n\n // watch for when the cy container is removed from the dom\n if (haveMutationsApi) {\n r.removeObserver = new MutationObserver(function (mutns) {\n // eslint-disable-line no-undef\n for (var i = 0; i < mutns.length; i++) {\n var mutn = mutns[i];\n var rNodes = mutn.removedNodes;\n if (rNodes) {\n for (var j = 0; j < rNodes.length; j++) {\n var rNode = rNodes[j];\n if (rNode === r.container) {\n r.destroy();\n break;\n }\n }\n }\n }\n });\n if (r.container.parentNode) {\n r.removeObserver.observe(r.container.parentNode, {\n childList: true\n });\n }\n } else {\n r.registerBinding(r.container, 'DOMNodeRemoved', function (e) {\n // eslint-disable-line no-unused-vars\n r.destroy();\n });\n }\n var onResize = debounce(function () {\n r.cy.resize();\n }, 100);\n if (haveMutationsApi) {\n r.styleObserver = new MutationObserver(onResize); // eslint-disable-line no-undef\n\n r.styleObserver.observe(r.container, {\n attributes: true\n });\n }\n\n // auto resize\n r.registerBinding(containerWindow, 'resize', onResize); // eslint-disable-line no-undef\n\n if (haveResizeObserverApi) {\n r.resizeObserver = new ResizeObserver(onResize); // eslint-disable-line no-undef\n\n r.resizeObserver.observe(r.container);\n }\n var forEachUp = function forEachUp(domEle, fn) {\n while (domEle != null) {\n fn(domEle);\n domEle = domEle.parentNode;\n }\n };\n var invalidateCoords = function invalidateCoords() {\n r.invalidateContainerClientCoordsCache();\n };\n forEachUp(r.container, function (domEle) {\n r.registerBinding(domEle, 'transitionend', invalidateCoords);\n r.registerBinding(domEle, 'animationend', invalidateCoords);\n r.registerBinding(domEle, 'scroll', invalidateCoords);\n });\n\n // stop right click menu from appearing on cy\n r.registerBinding(r.container, 'contextmenu', function (e) {\n e.preventDefault();\n });\n var inBoxSelection = function inBoxSelection() {\n return r.selection[4] !== 0;\n };\n var eventInContainer = function eventInContainer(e) {\n // save cycles if mouse events aren't to be captured\n var containerPageCoords = r.findContainerClientCoords();\n var x = containerPageCoords[0];\n var y = containerPageCoords[1];\n var width = containerPageCoords[2];\n var height = containerPageCoords[3];\n var positions = e.touches ? e.touches : [e];\n var atLeastOnePosInside = false;\n for (var i = 0; i < positions.length; i++) {\n var p = positions[i];\n if (x <= p.clientX && p.clientX <= x + width && y <= p.clientY && p.clientY <= y + height) {\n atLeastOnePosInside = true;\n break;\n }\n }\n if (!atLeastOnePosInside) {\n return false;\n }\n var container = r.container;\n var target = e.target;\n var tParent = target.parentNode;\n var containerIsTarget = false;\n while (tParent) {\n if (tParent === container) {\n containerIsTarget = true;\n break;\n }\n tParent = tParent.parentNode;\n }\n if (!containerIsTarget) {\n return false;\n } // if target is outisde cy container, then this event is not for us\n\n return true;\n };\n\n // Primary key\n r.registerBinding(r.container, 'mousedown', function mousedownHandler(e) {\n if (!eventInContainer(e)) {\n return;\n }\n\n // during left mouse button gestures, ignore other buttons\n if (r.hoverData.which === 1 && e.which !== 1) {\n return;\n }\n e.preventDefault();\n blurActiveDomElement();\n r.hoverData.capture = true;\n r.hoverData.which = e.which;\n var cy = r.cy;\n var gpos = [e.clientX, e.clientY];\n var pos = r.projectIntoViewport(gpos[0], gpos[1]);\n var select = r.selection;\n var nears = r.findNearestElements(pos[0], pos[1], true, false);\n var near = nears[0];\n var draggedElements = r.dragData.possibleDragElements;\n r.hoverData.mdownPos = pos;\n r.hoverData.mdownGPos = gpos;\n var makeEvent = function makeEvent(type) {\n return {\n originalEvent: e,\n type: type,\n position: {\n x: pos[0],\n y: pos[1]\n }\n };\n };\n var checkForTaphold = function checkForTaphold() {\n r.hoverData.tapholdCancelled = false;\n clearTimeout(r.hoverData.tapholdTimeout);\n r.hoverData.tapholdTimeout = setTimeout(function () {\n if (r.hoverData.tapholdCancelled) {\n return;\n } else {\n var ele = r.hoverData.down;\n if (ele) {\n ele.emit(makeEvent('taphold'));\n } else {\n cy.emit(makeEvent('taphold'));\n }\n }\n }, r.tapholdDuration);\n };\n\n // Right click button\n if (e.which == 3) {\n r.hoverData.cxtStarted = true;\n var cxtEvt = {\n originalEvent: e,\n type: 'cxttapstart',\n position: {\n x: pos[0],\n y: pos[1]\n }\n };\n if (near) {\n near.activate();\n near.emit(cxtEvt);\n r.hoverData.down = near;\n } else {\n cy.emit(cxtEvt);\n }\n r.hoverData.downTime = new Date().getTime();\n r.hoverData.cxtDragged = false;\n\n // Primary button\n } else if (e.which == 1) {\n if (near) {\n near.activate();\n }\n\n // Element dragging\n {\n // If something is under the cursor and it is draggable, prepare to grab it\n if (near != null) {\n if (r.nodeIsGrabbable(near)) {\n var triggerGrab = function triggerGrab(ele) {\n ele.emit(makeEvent('grab'));\n };\n setGrabTarget(near);\n if (!near.selected()) {\n draggedElements = r.dragData.possibleDragElements = cy.collection();\n addNodeToDrag(near, {\n addToList: draggedElements\n });\n near.emit(makeEvent('grabon')).emit(makeEvent('grab'));\n } else {\n draggedElements = r.dragData.possibleDragElements = cy.collection();\n var selectedNodes = cy.$(function (ele) {\n return ele.isNode() && ele.selected() && r.nodeIsGrabbable(ele);\n });\n addNodesToDrag(selectedNodes, {\n addToList: draggedElements\n });\n near.emit(makeEvent('grabon'));\n selectedNodes.forEach(triggerGrab);\n }\n r.redrawHint('eles', true);\n r.redrawHint('drag', true);\n }\n }\n r.hoverData.down = near;\n r.hoverData.downs = nears;\n r.hoverData.downTime = new Date().getTime();\n }\n triggerEvents(near, ['mousedown', 'tapstart', 'vmousedown'], e, {\n x: pos[0],\n y: pos[1]\n });\n if (near == null) {\n select[4] = 1;\n r.data.bgActivePosistion = {\n x: pos[0],\n y: pos[1]\n };\n r.redrawHint('select', true);\n r.redraw();\n } else if (near.pannable()) {\n select[4] = 1; // for future pan\n }\n checkForTaphold();\n }\n\n // Initialize selection box coordinates\n select[0] = select[2] = pos[0];\n select[1] = select[3] = pos[1];\n }, false);\n var shadowRoot = getShadowRoot(r.container);\n r.registerBinding([containerWindow, shadowRoot], 'mousemove', function mousemoveHandler(e) {\n // eslint-disable-line no-undef\n var capture = r.hoverData.capture;\n if (!capture && !eventInContainer(e)) {\n return;\n }\n var preventDefault = false;\n var cy = r.cy;\n var zoom = cy.zoom();\n var gpos = [e.clientX, e.clientY];\n var pos = r.projectIntoViewport(gpos[0], gpos[1]);\n var mdownPos = r.hoverData.mdownPos;\n var mdownGPos = r.hoverData.mdownGPos;\n var select = r.selection;\n var near = null;\n if (!r.hoverData.draggingEles && !r.hoverData.dragging && !r.hoverData.selecting) {\n near = r.findNearestElement(pos[0], pos[1], true, false);\n }\n var last = r.hoverData.last;\n var down = r.hoverData.down;\n var disp = [pos[0] - select[2], pos[1] - select[3]];\n var draggedElements = r.dragData.possibleDragElements;\n var isOverThresholdDrag;\n if (mdownGPos) {\n var dx = gpos[0] - mdownGPos[0];\n var dx2 = dx * dx;\n var dy = gpos[1] - mdownGPos[1];\n var dy2 = dy * dy;\n var dist2 = dx2 + dy2;\n r.hoverData.isOverThresholdDrag = isOverThresholdDrag = dist2 >= r.desktopTapThreshold2;\n }\n var multSelKeyDown = isMultSelKeyDown(e);\n if (isOverThresholdDrag) {\n r.hoverData.tapholdCancelled = true;\n }\n var updateDragDelta = function updateDragDelta() {\n var dragDelta = r.hoverData.dragDelta = r.hoverData.dragDelta || [];\n if (dragDelta.length === 0) {\n dragDelta.push(disp[0]);\n dragDelta.push(disp[1]);\n } else {\n dragDelta[0] += disp[0];\n dragDelta[1] += disp[1];\n }\n };\n preventDefault = true;\n triggerEvents(near, ['mousemove', 'vmousemove', 'tapdrag'], e, {\n x: pos[0],\n y: pos[1]\n });\n var makeEvent = function makeEvent(type) {\n return {\n originalEvent: e,\n type: type,\n position: {\n x: pos[0],\n y: pos[1]\n }\n };\n };\n var goIntoBoxMode = function goIntoBoxMode() {\n r.data.bgActivePosistion = undefined;\n if (!r.hoverData.selecting) {\n cy.emit(makeEvent('boxstart'));\n }\n select[4] = 1;\n r.hoverData.selecting = true;\n r.redrawHint('select', true);\n r.redraw();\n };\n\n // trigger context drag if rmouse down\n if (r.hoverData.which === 3) {\n // but only if over threshold\n if (isOverThresholdDrag) {\n var cxtEvt = makeEvent('cxtdrag');\n if (down) {\n down.emit(cxtEvt);\n } else {\n cy.emit(cxtEvt);\n }\n r.hoverData.cxtDragged = true;\n if (!r.hoverData.cxtOver || near !== r.hoverData.cxtOver) {\n if (r.hoverData.cxtOver) {\n r.hoverData.cxtOver.emit(makeEvent('cxtdragout'));\n }\n r.hoverData.cxtOver = near;\n if (near) {\n near.emit(makeEvent('cxtdragover'));\n }\n }\n }\n\n // Check if we are drag panning the entire graph\n } else if (r.hoverData.dragging) {\n preventDefault = true;\n if (cy.panningEnabled() && cy.userPanningEnabled()) {\n var deltaP;\n if (r.hoverData.justStartedPan) {\n var mdPos = r.hoverData.mdownPos;\n deltaP = {\n x: (pos[0] - mdPos[0]) * zoom,\n y: (pos[1] - mdPos[1]) * zoom\n };\n r.hoverData.justStartedPan = false;\n } else {\n deltaP = {\n x: disp[0] * zoom,\n y: disp[1] * zoom\n };\n }\n cy.panBy(deltaP);\n cy.emit(makeEvent('dragpan'));\n r.hoverData.dragged = true;\n }\n\n // Needs reproject due to pan changing viewport\n pos = r.projectIntoViewport(e.clientX, e.clientY);\n\n // Checks primary button down & out of time & mouse not moved much\n } else if (select[4] == 1 && (down == null || down.pannable())) {\n if (isOverThresholdDrag) {\n if (!r.hoverData.dragging && cy.boxSelectionEnabled() && (multSelKeyDown || !cy.panningEnabled() || !cy.userPanningEnabled())) {\n goIntoBoxMode();\n } else if (!r.hoverData.selecting && cy.panningEnabled() && cy.userPanningEnabled()) {\n var allowPassthrough = allowPanningPassthrough(down, r.hoverData.downs);\n if (allowPassthrough) {\n r.hoverData.dragging = true;\n r.hoverData.justStartedPan = true;\n select[4] = 0;\n r.data.bgActivePosistion = array2point(mdownPos);\n r.redrawHint('select', true);\n r.redraw();\n }\n }\n if (down && down.pannable() && down.active()) {\n down.unactivate();\n }\n }\n } else {\n if (down && down.pannable() && down.active()) {\n down.unactivate();\n }\n if ((!down || !down.grabbed()) && near != last) {\n if (last) {\n triggerEvents(last, ['mouseout', 'tapdragout'], e, {\n x: pos[0],\n y: pos[1]\n });\n }\n if (near) {\n triggerEvents(near, ['mouseover', 'tapdragover'], e, {\n x: pos[0],\n y: pos[1]\n });\n }\n r.hoverData.last = near;\n }\n if (down) {\n if (isOverThresholdDrag) {\n // then we can take action\n\n if (cy.boxSelectionEnabled() && multSelKeyDown) {\n // then selection overrides\n if (down && down.grabbed()) {\n freeDraggedElements(draggedElements);\n down.emit(makeEvent('freeon'));\n draggedElements.emit(makeEvent('free'));\n if (r.dragData.didDrag) {\n down.emit(makeEvent('dragfreeon'));\n draggedElements.emit(makeEvent('dragfree'));\n }\n }\n goIntoBoxMode();\n } else if (down && down.grabbed() && r.nodeIsDraggable(down)) {\n // drag node\n var justStartedDrag = !r.dragData.didDrag;\n if (justStartedDrag) {\n r.redrawHint('eles', true);\n }\n r.dragData.didDrag = true; // indicate that we actually did drag the node\n\n // now, add the elements to the drag layer if not done already\n if (!r.hoverData.draggingEles) {\n addNodesToDrag(draggedElements, {\n inDragLayer: true\n });\n }\n var totalShift = {\n x: 0,\n y: 0\n };\n if (number$1(disp[0]) && number$1(disp[1])) {\n totalShift.x += disp[0];\n totalShift.y += disp[1];\n if (justStartedDrag) {\n var dragDelta = r.hoverData.dragDelta;\n if (dragDelta && number$1(dragDelta[0]) && number$1(dragDelta[1])) {\n totalShift.x += dragDelta[0];\n totalShift.y += dragDelta[1];\n }\n }\n }\n r.hoverData.draggingEles = true;\n draggedElements.silentShift(totalShift).emit(makeEvent('position')).emit(makeEvent('drag'));\n r.redrawHint('drag', true);\n r.redraw();\n }\n } else {\n // otherwise save drag delta for when we actually start dragging so the relative grab pos is constant\n updateDragDelta();\n }\n }\n\n // prevent the dragging from triggering text selection on the page\n preventDefault = true;\n }\n select[2] = pos[0];\n select[3] = pos[1];\n if (preventDefault) {\n if (e.stopPropagation) e.stopPropagation();\n if (e.preventDefault) e.preventDefault();\n return false;\n }\n }, false);\n var clickTimeout, didDoubleClick, prevClickTimeStamp;\n r.registerBinding(containerWindow, 'mouseup', function mouseupHandler(e) {\n // eslint-disable-line no-undef\n // during left mouse button gestures, ignore other buttons\n if (r.hoverData.which === 1 && e.which !== 1 && r.hoverData.capture) {\n return;\n }\n var capture = r.hoverData.capture;\n if (!capture) {\n return;\n }\n r.hoverData.capture = false;\n var cy = r.cy;\n var pos = r.projectIntoViewport(e.clientX, e.clientY);\n var select = r.selection;\n var near = r.findNearestElement(pos[0], pos[1], true, false);\n var draggedElements = r.dragData.possibleDragElements;\n var down = r.hoverData.down;\n var multSelKeyDown = isMultSelKeyDown(e);\n if (r.data.bgActivePosistion) {\n r.redrawHint('select', true);\n r.redraw();\n }\n r.hoverData.tapholdCancelled = true;\n r.data.bgActivePosistion = undefined; // not active bg now\n\n if (down) {\n down.unactivate();\n }\n var makeEvent = function makeEvent(type) {\n return {\n originalEvent: e,\n type: type,\n position: {\n x: pos[0],\n y: pos[1]\n }\n };\n };\n if (r.hoverData.which === 3) {\n var cxtEvt = makeEvent('cxttapend');\n if (down) {\n down.emit(cxtEvt);\n } else {\n cy.emit(cxtEvt);\n }\n if (!r.hoverData.cxtDragged) {\n var cxtTap = makeEvent('cxttap');\n if (down) {\n down.emit(cxtTap);\n } else {\n cy.emit(cxtTap);\n }\n }\n r.hoverData.cxtDragged = false;\n r.hoverData.which = null;\n } else if (r.hoverData.which === 1) {\n triggerEvents(near, ['mouseup', 'tapend', 'vmouseup'], e, {\n x: pos[0],\n y: pos[1]\n });\n if (!r.dragData.didDrag &&\n // didn't move a node around\n !r.hoverData.dragged &&\n // didn't pan\n !r.hoverData.selecting &&\n // not box selection\n !r.hoverData.isOverThresholdDrag // didn't move too much\n ) {\n triggerEvents(down, [\"click\", \"tap\", \"vclick\"], e, {\n x: pos[0],\n y: pos[1]\n });\n didDoubleClick = false;\n if (e.timeStamp - prevClickTimeStamp <= cy.multiClickDebounceTime()) {\n clickTimeout && clearTimeout(clickTimeout);\n didDoubleClick = true;\n prevClickTimeStamp = null;\n triggerEvents(down, [\"dblclick\", \"dbltap\", \"vdblclick\"], e, {\n x: pos[0],\n y: pos[1]\n });\n } else {\n clickTimeout = setTimeout(function () {\n if (didDoubleClick) return;\n triggerEvents(down, [\"oneclick\", \"onetap\", \"voneclick\"], e, {\n x: pos[0],\n y: pos[1]\n });\n }, cy.multiClickDebounceTime());\n prevClickTimeStamp = e.timeStamp;\n }\n }\n\n // Deselect all elements if nothing is currently under the mouse cursor and we aren't dragging something\n if (down == null // not mousedown on node\n && !r.dragData.didDrag // didn't move the node around\n && !r.hoverData.selecting // not box selection\n && !r.hoverData.dragged // didn't pan\n && !isMultSelKeyDown(e)) {\n cy.$(isSelected).unselect(['tapunselect']);\n if (draggedElements.length > 0) {\n r.redrawHint('eles', true);\n }\n r.dragData.possibleDragElements = draggedElements = cy.collection();\n }\n\n // Single selection\n if (near == down && !r.dragData.didDrag && !r.hoverData.selecting) {\n if (near != null && near._private.selectable) {\n if (r.hoverData.dragging) ; else if (cy.selectionType() === 'additive' || multSelKeyDown) {\n if (near.selected()) {\n near.unselect(['tapunselect']);\n } else {\n near.select(['tapselect']);\n }\n } else {\n if (!multSelKeyDown) {\n cy.$(isSelected).unmerge(near).unselect(['tapunselect']);\n near.select(['tapselect']);\n }\n }\n r.redrawHint('eles', true);\n }\n }\n if (r.hoverData.selecting) {\n var box = cy.collection(r.getAllInBox(select[0], select[1], select[2], select[3]));\n r.redrawHint('select', true);\n if (box.length > 0) {\n r.redrawHint('eles', true);\n }\n cy.emit(makeEvent('boxend'));\n var eleWouldBeSelected = function eleWouldBeSelected(ele) {\n return ele.selectable() && !ele.selected();\n };\n if (cy.selectionType() === 'additive') {\n box.emit(makeEvent('box')).stdFilter(eleWouldBeSelected).select().emit(makeEvent('boxselect'));\n } else {\n if (!multSelKeyDown) {\n cy.$(isSelected).unmerge(box).unselect();\n }\n box.emit(makeEvent('box')).stdFilter(eleWouldBeSelected).select().emit(makeEvent('boxselect'));\n }\n\n // always need redraw in case eles unselectable\n r.redraw();\n }\n\n // Cancel drag pan\n if (r.hoverData.dragging) {\n r.hoverData.dragging = false;\n r.redrawHint('select', true);\n r.redrawHint('eles', true);\n r.redraw();\n }\n if (!select[4]) {\n r.redrawHint('drag', true);\n r.redrawHint('eles', true);\n var downWasGrabbed = down && down.grabbed();\n freeDraggedElements(draggedElements);\n if (downWasGrabbed) {\n down.emit(makeEvent('freeon'));\n draggedElements.emit(makeEvent('free'));\n if (r.dragData.didDrag) {\n down.emit(makeEvent('dragfreeon'));\n draggedElements.emit(makeEvent('dragfree'));\n }\n }\n }\n } // else not right mouse\n\n select[4] = 0;\n r.hoverData.down = null;\n r.hoverData.cxtStarted = false;\n r.hoverData.draggingEles = false;\n r.hoverData.selecting = false;\n r.hoverData.isOverThresholdDrag = false;\n r.dragData.didDrag = false;\n r.hoverData.dragged = false;\n r.hoverData.dragDelta = [];\n r.hoverData.mdownPos = null;\n r.hoverData.mdownGPos = null;\n r.hoverData.which = null;\n }, false);\n var wheelDeltas = []; // log of first N wheel deltas\n var wheelDeltaN = 4; // how many events to log\n var inaccurateScrollDevice;\n var inaccurateScrollFactor = 100000; // base of inaccurate wheel deltas (e.g. base 5 could yield wheels of 10, 25, 50, etc.)\n\n var allAreDivisibleBy = function allAreDivisibleBy(list, factor) {\n for (var i = 0; i < list.length; i++) {\n if (list[i] % factor !== 0) {\n return false;\n }\n }\n return true;\n };\n var allAreSameMagnitude = function allAreSameMagnitude(list) {\n var firstMag = Math.abs(list[0]);\n for (var i = 1; i < list.length; i++) {\n if (Math.abs(list[i]) !== firstMag) {\n return false;\n }\n }\n return true;\n };\n var wheelHandler = function wheelHandler(e) {\n var clamp = false;\n var delta = e.deltaY;\n if (delta == null) {\n // compatibility with old browsers\n if (e.wheelDeltaY != null) {\n delta = e.wheelDeltaY / 4;\n } else if (e.wheelDelta != null) {\n delta = e.wheelDelta / 4;\n }\n }\n if (delta === 0) {\n return; // no change in zoom (Bug: Zoom becomes erratic on rapid scroll due to deltaY: 0 event #3394)\n }\n if (inaccurateScrollDevice == null) {\n if (wheelDeltas.length >= wheelDeltaN) {\n // use log to determine if inaccurate\n var wds = wheelDeltas;\n inaccurateScrollDevice = allAreDivisibleBy(wds, 5);\n if (!inaccurateScrollDevice) {\n // check for all large values of exact same magnitude\n var firstMag = Math.abs(wds[0]);\n inaccurateScrollDevice = allAreSameMagnitude(wds) && firstMag > 5;\n }\n if (inaccurateScrollDevice) {\n for (var i = 0; i < wds.length; i++) {\n inaccurateScrollFactor = Math.min(Math.abs(wds[i]), inaccurateScrollFactor);\n }\n }\n\n // console.log('Sampled wheel deltas:', wds);\n // console.log('inaccurateScrollDevice:', inaccurateScrollDevice);\n // console.log('inaccurateScrollFactor:', inaccurateScrollFactor);\n } else {\n // clamp and log until we reach N\n wheelDeltas.push(delta);\n clamp = true;\n // console.log('Clamping initial wheel events until we get a good sample');\n }\n } else if (inaccurateScrollDevice) {\n // keep updating\n inaccurateScrollFactor = Math.min(Math.abs(delta), inaccurateScrollFactor);\n // console.log('Keep updating inaccurateScrollFactor beyond sample in case we did not get the smallest possible val:', inaccurateScrollFactor);\n }\n if (r.scrollingPage) {\n return;\n } // while scrolling, ignore wheel-to-zoom\n\n var cy = r.cy;\n var zoom = cy.zoom();\n var pan = cy.pan();\n var pos = r.projectIntoViewport(e.clientX, e.clientY);\n var rpos = [pos[0] * zoom + pan.x, pos[1] * zoom + pan.y];\n if (r.hoverData.draggingEles || r.hoverData.dragging || r.hoverData.cxtStarted || inBoxSelection()) {\n // if pan dragging or cxt dragging, wheel movements make no zoom\n e.preventDefault();\n return;\n }\n if (cy.panningEnabled() && cy.userPanningEnabled() && cy.zoomingEnabled() && cy.userZoomingEnabled()) {\n e.preventDefault();\n r.data.wheelZooming = true;\n clearTimeout(r.data.wheelTimeout);\n r.data.wheelTimeout = setTimeout(function () {\n r.data.wheelZooming = false;\n r.redrawHint('eles', true);\n r.redraw();\n }, 150);\n var diff;\n if (clamp && Math.abs(delta) > 5) {\n delta = signum(delta) * 5;\n }\n diff = delta / -250;\n if (inaccurateScrollDevice) {\n diff /= inaccurateScrollFactor;\n diff *= 3;\n }\n diff = diff * r.wheelSensitivity;\n\n // console.log(`delta = ${delta}, diff = ${diff}, mode = ${e.deltaMode}`)\n\n var needsWheelFix = e.deltaMode === 1;\n if (needsWheelFix) {\n // fixes slow wheel events on ff/linux and ff/windows\n diff *= 33;\n }\n var newZoom = cy.zoom() * Math.pow(10, diff);\n if (e.type === 'gesturechange') {\n newZoom = r.gestureStartZoom * e.scale;\n }\n cy.zoom({\n level: newZoom,\n renderedPosition: {\n x: rpos[0],\n y: rpos[1]\n }\n });\n cy.emit({\n type: e.type === 'gesturechange' ? 'pinchzoom' : 'scrollzoom',\n originalEvent: e,\n position: {\n x: pos[0],\n y: pos[1]\n }\n });\n }\n };\n\n // Functions to help with whether mouse wheel should trigger zooming\n // --\n r.registerBinding(r.container, 'wheel', wheelHandler, true);\n\n // disable nonstandard wheel events\n // r.registerBinding(r.container, 'mousewheel', wheelHandler, true);\n // r.registerBinding(r.container, 'DOMMouseScroll', wheelHandler, true);\n // r.registerBinding(r.container, 'MozMousePixelScroll', wheelHandler, true); // older firefox\n\n r.registerBinding(containerWindow, 'scroll', function scrollHandler(e) {\n // eslint-disable-line no-unused-vars\n r.scrollingPage = true;\n clearTimeout(r.scrollingPageTimeout);\n r.scrollingPageTimeout = setTimeout(function () {\n r.scrollingPage = false;\n }, 250);\n }, true);\n\n // desktop safari pinch to zoom start\n r.registerBinding(r.container, 'gesturestart', function gestureStartHandler(e) {\n r.gestureStartZoom = r.cy.zoom();\n if (!r.hasTouchStarted) {\n // don't affect touch devices like iphone\n e.preventDefault();\n }\n }, true);\n r.registerBinding(r.container, 'gesturechange', function (e) {\n if (!r.hasTouchStarted) {\n // don't affect touch devices like iphone\n wheelHandler(e);\n }\n }, true);\n\n // Functions to help with handling mouseout/mouseover on the Cytoscape container\n // Handle mouseout on Cytoscape container\n r.registerBinding(r.container, 'mouseout', function mouseOutHandler(e) {\n var pos = r.projectIntoViewport(e.clientX, e.clientY);\n r.cy.emit({\n originalEvent: e,\n type: 'mouseout',\n position: {\n x: pos[0],\n y: pos[1]\n }\n });\n }, false);\n r.registerBinding(r.container, 'mouseover', function mouseOverHandler(e) {\n var pos = r.projectIntoViewport(e.clientX, e.clientY);\n r.cy.emit({\n originalEvent: e,\n type: 'mouseover',\n position: {\n x: pos[0],\n y: pos[1]\n }\n });\n }, false);\n var f1x1, f1y1, f2x1, f2y1; // starting points for pinch-to-zoom\n var distance1, distance1Sq; // initial distance between finger 1 and finger 2 for pinch-to-zoom\n var center1, modelCenter1; // center point on start pinch to zoom\n var offsetLeft, offsetTop;\n var containerWidth, containerHeight;\n var twoFingersStartInside;\n var distance = function distance(x1, y1, x2, y2) {\n return Math.sqrt((x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1));\n };\n var distanceSq = function distanceSq(x1, y1, x2, y2) {\n return (x2 - x1) * (x2 - x1) + (y2 - y1) * (y2 - y1);\n };\n var touchstartHandler;\n r.registerBinding(r.container, 'touchstart', touchstartHandler = function touchstartHandler(e) {\n r.hasTouchStarted = true;\n if (!eventInContainer(e)) {\n return;\n }\n blurActiveDomElement();\n r.touchData.capture = true;\n r.data.bgActivePosistion = undefined;\n var cy = r.cy;\n var now = r.touchData.now;\n var earlier = r.touchData.earlier;\n if (e.touches[0]) {\n var pos = r.projectIntoViewport(e.touches[0].clientX, e.touches[0].clientY);\n now[0] = pos[0];\n now[1] = pos[1];\n }\n if (e.touches[1]) {\n var pos = r.projectIntoViewport(e.touches[1].clientX, e.touches[1].clientY);\n now[2] = pos[0];\n now[3] = pos[1];\n }\n if (e.touches[2]) {\n var pos = r.projectIntoViewport(e.touches[2].clientX, e.touches[2].clientY);\n now[4] = pos[0];\n now[5] = pos[1];\n }\n var makeEvent = function makeEvent(type) {\n return {\n originalEvent: e,\n type: type,\n position: {\n x: now[0],\n y: now[1]\n }\n };\n };\n\n // record starting points for pinch-to-zoom\n if (e.touches[1]) {\n r.touchData.singleTouchMoved = true;\n freeDraggedElements(r.dragData.touchDragEles);\n var offsets = r.findContainerClientCoords();\n offsetLeft = offsets[0];\n offsetTop = offsets[1];\n containerWidth = offsets[2];\n containerHeight = offsets[3];\n f1x1 = e.touches[0].clientX - offsetLeft;\n f1y1 = e.touches[0].clientY - offsetTop;\n f2x1 = e.touches[1].clientX - offsetLeft;\n f2y1 = e.touches[1].clientY - offsetTop;\n twoFingersStartInside = 0 <= f1x1 && f1x1 <= containerWidth && 0 <= f2x1 && f2x1 <= containerWidth && 0 <= f1y1 && f1y1 <= containerHeight && 0 <= f2y1 && f2y1 <= containerHeight;\n var pan = cy.pan();\n var zoom = cy.zoom();\n distance1 = distance(f1x1, f1y1, f2x1, f2y1);\n distance1Sq = distanceSq(f1x1, f1y1, f2x1, f2y1);\n center1 = [(f1x1 + f2x1) / 2, (f1y1 + f2y1) / 2];\n modelCenter1 = [(center1[0] - pan.x) / zoom, (center1[1] - pan.y) / zoom];\n\n // consider context tap\n var cxtDistThreshold = 200;\n var cxtDistThresholdSq = cxtDistThreshold * cxtDistThreshold;\n if (distance1Sq < cxtDistThresholdSq && !e.touches[2]) {\n var near1 = r.findNearestElement(now[0], now[1], true, true);\n var near2 = r.findNearestElement(now[2], now[3], true, true);\n if (near1 && near1.isNode()) {\n near1.activate().emit(makeEvent('cxttapstart'));\n r.touchData.start = near1;\n } else if (near2 && near2.isNode()) {\n near2.activate().emit(makeEvent('cxttapstart'));\n r.touchData.start = near2;\n } else {\n cy.emit(makeEvent('cxttapstart'));\n }\n if (r.touchData.start) {\n r.touchData.start._private.grabbed = false;\n }\n r.touchData.cxt = true;\n r.touchData.cxtDragged = false;\n r.data.bgActivePosistion = undefined;\n r.redraw();\n return;\n }\n }\n if (e.touches[2]) {\n // ignore\n\n // safari on ios pans the page otherwise (normally you should be able to preventdefault on touchmove...)\n if (cy.boxSelectionEnabled()) {\n e.preventDefault();\n }\n } else if (e.touches[1]) ; else if (e.touches[0]) {\n var nears = r.findNearestElements(now[0], now[1], true, true);\n var near = nears[0];\n if (near != null) {\n near.activate();\n r.touchData.start = near;\n r.touchData.starts = nears;\n if (r.nodeIsGrabbable(near)) {\n var draggedEles = r.dragData.touchDragEles = cy.collection();\n var selectedNodes = null;\n r.redrawHint('eles', true);\n r.redrawHint('drag', true);\n if (near.selected()) {\n // reset drag elements, since near will be added again\n\n selectedNodes = cy.$(function (ele) {\n return ele.selected() && r.nodeIsGrabbable(ele);\n });\n addNodesToDrag(selectedNodes, {\n addToList: draggedEles\n });\n } else {\n addNodeToDrag(near, {\n addToList: draggedEles\n });\n }\n setGrabTarget(near);\n near.emit(makeEvent('grabon'));\n if (selectedNodes) {\n selectedNodes.forEach(function (n) {\n n.emit(makeEvent('grab'));\n });\n } else {\n near.emit(makeEvent('grab'));\n }\n }\n }\n triggerEvents(near, ['touchstart', 'tapstart', 'vmousedown'], e, {\n x: now[0],\n y: now[1]\n });\n if (near == null) {\n r.data.bgActivePosistion = {\n x: pos[0],\n y: pos[1]\n };\n r.redrawHint('select', true);\n r.redraw();\n }\n\n // Tap, taphold\n // -----\n\n r.touchData.singleTouchMoved = false;\n r.touchData.singleTouchStartTime = +new Date();\n clearTimeout(r.touchData.tapholdTimeout);\n r.touchData.tapholdTimeout = setTimeout(function () {\n if (r.touchData.singleTouchMoved === false && !r.pinching // if pinching, then taphold unselect shouldn't take effect\n && !r.touchData.selecting // box selection shouldn't allow taphold through\n ) {\n triggerEvents(r.touchData.start, ['taphold'], e, {\n x: now[0],\n y: now[1]\n });\n }\n }, r.tapholdDuration);\n }\n if (e.touches.length >= 1) {\n var sPos = r.touchData.startPosition = [null, null, null, null, null, null];\n for (var i = 0; i < now.length; i++) {\n sPos[i] = earlier[i] = now[i];\n }\n var touch0 = e.touches[0];\n r.touchData.startGPosition = [touch0.clientX, touch0.clientY];\n }\n }, false);\n var touchmoveHandler;\n r.registerBinding(containerWindow, 'touchmove', touchmoveHandler = function touchmoveHandler(e) {\n // eslint-disable-line no-undef\n var capture = r.touchData.capture;\n if (!capture && !eventInContainer(e)) {\n return;\n }\n var select = r.selection;\n var cy = r.cy;\n var now = r.touchData.now;\n var earlier = r.touchData.earlier;\n var zoom = cy.zoom();\n if (e.touches[0]) {\n var pos = r.projectIntoViewport(e.touches[0].clientX, e.touches[0].clientY);\n now[0] = pos[0];\n now[1] = pos[1];\n }\n if (e.touches[1]) {\n var pos = r.projectIntoViewport(e.touches[1].clientX, e.touches[1].clientY);\n now[2] = pos[0];\n now[3] = pos[1];\n }\n if (e.touches[2]) {\n var pos = r.projectIntoViewport(e.touches[2].clientX, e.touches[2].clientY);\n now[4] = pos[0];\n now[5] = pos[1];\n }\n var makeEvent = function makeEvent(type) {\n return {\n originalEvent: e,\n type: type,\n position: {\n x: now[0],\n y: now[1]\n }\n };\n };\n var startGPos = r.touchData.startGPosition;\n var isOverThresholdDrag;\n if (capture && e.touches[0] && startGPos) {\n var disp = [];\n for (var j = 0; j < now.length; j++) {\n disp[j] = now[j] - earlier[j];\n }\n var dx = e.touches[0].clientX - startGPos[0];\n var dx2 = dx * dx;\n var dy = e.touches[0].clientY - startGPos[1];\n var dy2 = dy * dy;\n var dist2 = dx2 + dy2;\n isOverThresholdDrag = dist2 >= r.touchTapThreshold2;\n }\n\n // context swipe cancelling\n if (capture && r.touchData.cxt) {\n e.preventDefault();\n var f1x2 = e.touches[0].clientX - offsetLeft,\n f1y2 = e.touches[0].clientY - offsetTop;\n var f2x2 = e.touches[1].clientX - offsetLeft,\n f2y2 = e.touches[1].clientY - offsetTop;\n // var distance2 = distance( f1x2, f1y2, f2x2, f2y2 );\n var distance2Sq = distanceSq(f1x2, f1y2, f2x2, f2y2);\n var factorSq = distance2Sq / distance1Sq;\n var distThreshold = 150;\n var distThresholdSq = distThreshold * distThreshold;\n var factorThreshold = 1.5;\n var factorThresholdSq = factorThreshold * factorThreshold;\n\n // cancel ctx gestures if the distance b/t the fingers increases\n if (factorSq >= factorThresholdSq || distance2Sq >= distThresholdSq) {\n r.touchData.cxt = false;\n r.data.bgActivePosistion = undefined;\n r.redrawHint('select', true);\n var cxtEvt = makeEvent('cxttapend');\n if (r.touchData.start) {\n r.touchData.start.unactivate().emit(cxtEvt);\n r.touchData.start = null;\n } else {\n cy.emit(cxtEvt);\n }\n }\n }\n\n // context swipe\n if (capture && r.touchData.cxt) {\n var cxtEvt = makeEvent('cxtdrag');\n r.data.bgActivePosistion = undefined;\n r.redrawHint('select', true);\n if (r.touchData.start) {\n r.touchData.start.emit(cxtEvt);\n } else {\n cy.emit(cxtEvt);\n }\n if (r.touchData.start) {\n r.touchData.start._private.grabbed = false;\n }\n r.touchData.cxtDragged = true;\n var near = r.findNearestElement(now[0], now[1], true, true);\n if (!r.touchData.cxtOver || near !== r.touchData.cxtOver) {\n if (r.touchData.cxtOver) {\n r.touchData.cxtOver.emit(makeEvent('cxtdragout'));\n }\n r.touchData.cxtOver = near;\n if (near) {\n near.emit(makeEvent('cxtdragover'));\n }\n }\n\n // box selection\n } else if (capture && e.touches[2] && cy.boxSelectionEnabled()) {\n e.preventDefault();\n r.data.bgActivePosistion = undefined;\n this.lastThreeTouch = +new Date();\n if (!r.touchData.selecting) {\n cy.emit(makeEvent('boxstart'));\n }\n r.touchData.selecting = true;\n r.touchData.didSelect = true;\n select[4] = 1;\n if (!select || select.length === 0 || select[0] === undefined) {\n select[0] = (now[0] + now[2] + now[4]) / 3;\n select[1] = (now[1] + now[3] + now[5]) / 3;\n select[2] = (now[0] + now[2] + now[4]) / 3 + 1;\n select[3] = (now[1] + now[3] + now[5]) / 3 + 1;\n } else {\n select[2] = (now[0] + now[2] + now[4]) / 3;\n select[3] = (now[1] + now[3] + now[5]) / 3;\n }\n r.redrawHint('select', true);\n r.redraw();\n\n // pinch to zoom\n } else if (capture && e.touches[1] && !r.touchData.didSelect // don't allow box selection to degrade to pinch-to-zoom\n && cy.zoomingEnabled() && cy.panningEnabled() && cy.userZoomingEnabled() && cy.userPanningEnabled()) {\n // two fingers => pinch to zoom\n e.preventDefault();\n r.data.bgActivePosistion = undefined;\n r.redrawHint('select', true);\n var draggedEles = r.dragData.touchDragEles;\n if (draggedEles) {\n r.redrawHint('drag', true);\n for (var i = 0; i < draggedEles.length; i++) {\n var de_p = draggedEles[i]._private;\n de_p.grabbed = false;\n de_p.rscratch.inDragLayer = false;\n }\n }\n var _start = r.touchData.start;\n\n // (x2, y2) for fingers 1 and 2\n var f1x2 = e.touches[0].clientX - offsetLeft,\n f1y2 = e.touches[0].clientY - offsetTop;\n var f2x2 = e.touches[1].clientX - offsetLeft,\n f2y2 = e.touches[1].clientY - offsetTop;\n var distance2 = distance(f1x2, f1y2, f2x2, f2y2);\n // var distance2Sq = distanceSq( f1x2, f1y2, f2x2, f2y2 );\n // var factor = Math.sqrt( distance2Sq ) / Math.sqrt( distance1Sq );\n var factor = distance2 / distance1;\n if (twoFingersStartInside) {\n // delta finger1\n var df1x = f1x2 - f1x1;\n var df1y = f1y2 - f1y1;\n\n // delta finger 2\n var df2x = f2x2 - f2x1;\n var df2y = f2y2 - f2y1;\n\n // translation is the normalised vector of the two fingers movement\n // i.e. so pinching cancels out and moving together pans\n var tx = (df1x + df2x) / 2;\n var ty = (df1y + df2y) / 2;\n\n // now calculate the zoom\n var zoom1 = cy.zoom();\n var zoom2 = zoom1 * factor;\n var pan1 = cy.pan();\n\n // the model center point converted to the current rendered pos\n var ctrx = modelCenter1[0] * zoom1 + pan1.x;\n var ctry = modelCenter1[1] * zoom1 + pan1.y;\n var pan2 = {\n x: -zoom2 / zoom1 * (ctrx - pan1.x - tx) + ctrx,\n y: -zoom2 / zoom1 * (ctry - pan1.y - ty) + ctry\n };\n\n // remove dragged eles\n if (_start && _start.active()) {\n var draggedEles = r.dragData.touchDragEles;\n freeDraggedElements(draggedEles);\n r.redrawHint('drag', true);\n r.redrawHint('eles', true);\n _start.unactivate().emit(makeEvent('freeon'));\n draggedEles.emit(makeEvent('free'));\n if (r.dragData.didDrag) {\n _start.emit(makeEvent('dragfreeon'));\n draggedEles.emit(makeEvent('dragfree'));\n }\n }\n cy.viewport({\n zoom: zoom2,\n pan: pan2,\n cancelOnFailedZoom: true\n });\n cy.emit(makeEvent('pinchzoom'));\n distance1 = distance2;\n f1x1 = f1x2;\n f1y1 = f1y2;\n f2x1 = f2x2;\n f2y1 = f2y2;\n r.pinching = true;\n }\n\n // Re-project\n if (e.touches[0]) {\n var pos = r.projectIntoViewport(e.touches[0].clientX, e.touches[0].clientY);\n now[0] = pos[0];\n now[1] = pos[1];\n }\n if (e.touches[1]) {\n var pos = r.projectIntoViewport(e.touches[1].clientX, e.touches[1].clientY);\n now[2] = pos[0];\n now[3] = pos[1];\n }\n if (e.touches[2]) {\n var pos = r.projectIntoViewport(e.touches[2].clientX, e.touches[2].clientY);\n now[4] = pos[0];\n now[5] = pos[1];\n }\n } else if (e.touches[0] && !r.touchData.didSelect // don't allow box selection to degrade to single finger events like panning\n ) {\n var start = r.touchData.start;\n var last = r.touchData.last;\n var near;\n if (!r.hoverData.draggingEles && !r.swipePanning) {\n near = r.findNearestElement(now[0], now[1], true, true);\n }\n if (capture && start != null) {\n e.preventDefault();\n }\n\n // dragging nodes\n if (capture && start != null && r.nodeIsDraggable(start)) {\n if (isOverThresholdDrag) {\n // then dragging can happen\n var draggedEles = r.dragData.touchDragEles;\n var justStartedDrag = !r.dragData.didDrag;\n if (justStartedDrag) {\n addNodesToDrag(draggedEles, {\n inDragLayer: true\n });\n }\n r.dragData.didDrag = true;\n var totalShift = {\n x: 0,\n y: 0\n };\n if (number$1(disp[0]) && number$1(disp[1])) {\n totalShift.x += disp[0];\n totalShift.y += disp[1];\n if (justStartedDrag) {\n r.redrawHint('eles', true);\n var dragDelta = r.touchData.dragDelta;\n if (dragDelta && number$1(dragDelta[0]) && number$1(dragDelta[1])) {\n totalShift.x += dragDelta[0];\n totalShift.y += dragDelta[1];\n }\n }\n }\n r.hoverData.draggingEles = true;\n draggedEles.silentShift(totalShift).emit(makeEvent('position')).emit(makeEvent('drag'));\n r.redrawHint('drag', true);\n if (r.touchData.startPosition[0] == earlier[0] && r.touchData.startPosition[1] == earlier[1]) {\n r.redrawHint('eles', true);\n }\n r.redraw();\n } else {\n // otherwise keep track of drag delta for later\n var dragDelta = r.touchData.dragDelta = r.touchData.dragDelta || [];\n if (dragDelta.length === 0) {\n dragDelta.push(disp[0]);\n dragDelta.push(disp[1]);\n } else {\n dragDelta[0] += disp[0];\n dragDelta[1] += disp[1];\n }\n }\n }\n\n // touchmove\n {\n triggerEvents(start || near, ['touchmove', 'tapdrag', 'vmousemove'], e, {\n x: now[0],\n y: now[1]\n });\n if ((!start || !start.grabbed()) && near != last) {\n if (last) {\n last.emit(makeEvent('tapdragout'));\n }\n if (near) {\n near.emit(makeEvent('tapdragover'));\n }\n }\n r.touchData.last = near;\n }\n\n // check to cancel taphold\n if (capture) {\n for (var i = 0; i < now.length; i++) {\n if (now[i] && r.touchData.startPosition[i] && isOverThresholdDrag) {\n r.touchData.singleTouchMoved = true;\n }\n }\n }\n\n // panning\n if (capture && (start == null || start.pannable()) && cy.panningEnabled() && cy.userPanningEnabled()) {\n var allowPassthrough = allowPanningPassthrough(start, r.touchData.starts);\n if (allowPassthrough) {\n e.preventDefault();\n if (!r.data.bgActivePosistion) {\n r.data.bgActivePosistion = array2point(r.touchData.startPosition);\n }\n if (r.swipePanning) {\n cy.panBy({\n x: disp[0] * zoom,\n y: disp[1] * zoom\n });\n cy.emit(makeEvent('dragpan'));\n } else if (isOverThresholdDrag) {\n r.swipePanning = true;\n cy.panBy({\n x: dx * zoom,\n y: dy * zoom\n });\n cy.emit(makeEvent('dragpan'));\n if (start) {\n start.unactivate();\n r.redrawHint('select', true);\n r.touchData.start = null;\n }\n }\n }\n\n // Re-project\n var pos = r.projectIntoViewport(e.touches[0].clientX, e.touches[0].clientY);\n now[0] = pos[0];\n now[1] = pos[1];\n }\n }\n for (var j = 0; j < now.length; j++) {\n earlier[j] = now[j];\n }\n\n // the active bg indicator should be removed when making a swipe that is neither for dragging nodes or panning\n if (capture && e.touches.length > 0 && !r.hoverData.draggingEles && !r.swipePanning && r.data.bgActivePosistion != null) {\n r.data.bgActivePosistion = undefined;\n r.redrawHint('select', true);\n r.redraw();\n }\n }, false);\n var touchcancelHandler;\n r.registerBinding(containerWindow, 'touchcancel', touchcancelHandler = function touchcancelHandler(e) {\n // eslint-disable-line no-unused-vars\n var start = r.touchData.start;\n r.touchData.capture = false;\n if (start) {\n start.unactivate();\n }\n });\n var touchendHandler, didDoubleTouch, touchTimeout, prevTouchTimeStamp;\n r.registerBinding(containerWindow, 'touchend', touchendHandler = function touchendHandler(e) {\n // eslint-disable-line no-unused-vars\n var start = r.touchData.start;\n var capture = r.touchData.capture;\n if (capture) {\n if (e.touches.length === 0) {\n r.touchData.capture = false;\n }\n e.preventDefault();\n } else {\n return;\n }\n var select = r.selection;\n r.swipePanning = false;\n r.hoverData.draggingEles = false;\n var cy = r.cy;\n var zoom = cy.zoom();\n var now = r.touchData.now;\n var earlier = r.touchData.earlier;\n if (e.touches[0]) {\n var pos = r.projectIntoViewport(e.touches[0].clientX, e.touches[0].clientY);\n now[0] = pos[0];\n now[1] = pos[1];\n }\n if (e.touches[1]) {\n var pos = r.projectIntoViewport(e.touches[1].clientX, e.touches[1].clientY);\n now[2] = pos[0];\n now[3] = pos[1];\n }\n if (e.touches[2]) {\n var pos = r.projectIntoViewport(e.touches[2].clientX, e.touches[2].clientY);\n now[4] = pos[0];\n now[5] = pos[1];\n }\n var makeEvent = function makeEvent(type) {\n return {\n originalEvent: e,\n type: type,\n position: {\n x: now[0],\n y: now[1]\n }\n };\n };\n if (start) {\n start.unactivate();\n }\n var ctxTapend;\n if (r.touchData.cxt) {\n ctxTapend = makeEvent('cxttapend');\n if (start) {\n start.emit(ctxTapend);\n } else {\n cy.emit(ctxTapend);\n }\n if (!r.touchData.cxtDragged) {\n var ctxTap = makeEvent('cxttap');\n if (start) {\n start.emit(ctxTap);\n } else {\n cy.emit(ctxTap);\n }\n }\n if (r.touchData.start) {\n r.touchData.start._private.grabbed = false;\n }\n r.touchData.cxt = false;\n r.touchData.start = null;\n r.redraw();\n return;\n }\n\n // no more box selection if we don't have three fingers\n if (!e.touches[2] && cy.boxSelectionEnabled() && r.touchData.selecting) {\n r.touchData.selecting = false;\n var box = cy.collection(r.getAllInBox(select[0], select[1], select[2], select[3]));\n select[0] = undefined;\n select[1] = undefined;\n select[2] = undefined;\n select[3] = undefined;\n select[4] = 0;\n r.redrawHint('select', true);\n cy.emit(makeEvent('boxend'));\n var eleWouldBeSelected = function eleWouldBeSelected(ele) {\n return ele.selectable() && !ele.selected();\n };\n box.emit(makeEvent('box')).stdFilter(eleWouldBeSelected).select().emit(makeEvent('boxselect'));\n if (box.nonempty()) {\n r.redrawHint('eles', true);\n }\n r.redraw();\n }\n if (start != null) {\n start.unactivate();\n }\n if (e.touches[2]) {\n r.data.bgActivePosistion = undefined;\n r.redrawHint('select', true);\n } else if (e.touches[1]) ; else if (e.touches[0]) ; else if (!e.touches[0]) {\n r.data.bgActivePosistion = undefined;\n r.redrawHint('select', true);\n var draggedEles = r.dragData.touchDragEles;\n if (start != null) {\n var startWasGrabbed = start._private.grabbed;\n freeDraggedElements(draggedEles);\n r.redrawHint('drag', true);\n r.redrawHint('eles', true);\n if (startWasGrabbed) {\n start.emit(makeEvent('freeon'));\n draggedEles.emit(makeEvent('free'));\n if (r.dragData.didDrag) {\n start.emit(makeEvent('dragfreeon'));\n draggedEles.emit(makeEvent('dragfree'));\n }\n }\n triggerEvents(start, ['touchend', 'tapend', 'vmouseup', 'tapdragout'], e, {\n x: now[0],\n y: now[1]\n });\n start.unactivate();\n r.touchData.start = null;\n } else {\n var near = r.findNearestElement(now[0], now[1], true, true);\n triggerEvents(near, ['touchend', 'tapend', 'vmouseup', 'tapdragout'], e, {\n x: now[0],\n y: now[1]\n });\n }\n var dx = r.touchData.startPosition[0] - now[0];\n var dx2 = dx * dx;\n var dy = r.touchData.startPosition[1] - now[1];\n var dy2 = dy * dy;\n var dist2 = dx2 + dy2;\n var rdist2 = dist2 * zoom * zoom;\n\n // Tap event, roughly same as mouse click event for touch\n if (!r.touchData.singleTouchMoved) {\n if (!start) {\n cy.$(':selected').unselect(['tapunselect']);\n }\n triggerEvents(start, ['tap', 'vclick'], e, {\n x: now[0],\n y: now[1]\n });\n didDoubleTouch = false;\n if (e.timeStamp - prevTouchTimeStamp <= cy.multiClickDebounceTime()) {\n touchTimeout && clearTimeout(touchTimeout);\n didDoubleTouch = true;\n prevTouchTimeStamp = null;\n triggerEvents(start, ['dbltap', 'vdblclick'], e, {\n x: now[0],\n y: now[1]\n });\n } else {\n touchTimeout = setTimeout(function () {\n if (didDoubleTouch) return;\n triggerEvents(start, ['onetap', 'voneclick'], e, {\n x: now[0],\n y: now[1]\n });\n }, cy.multiClickDebounceTime());\n prevTouchTimeStamp = e.timeStamp;\n }\n }\n\n // Prepare to select the currently touched node, only if it hasn't been dragged past a certain distance\n if (start != null && !r.dragData.didDrag // didn't drag nodes around\n && start._private.selectable && rdist2 < r.touchTapThreshold2 && !r.pinching // pinch to zoom should not affect selection\n ) {\n if (cy.selectionType() === 'single') {\n cy.$(isSelected).unmerge(start).unselect(['tapunselect']);\n start.select(['tapselect']);\n } else {\n if (start.selected()) {\n start.unselect(['tapunselect']);\n } else {\n start.select(['tapselect']);\n }\n }\n r.redrawHint('eles', true);\n }\n r.touchData.singleTouchMoved = true;\n }\n for (var j = 0; j < now.length; j++) {\n earlier[j] = now[j];\n }\n r.dragData.didDrag = false; // reset for next touchstart\n\n if (e.touches.length === 0) {\n r.touchData.dragDelta = [];\n r.touchData.startPosition = [null, null, null, null, null, null];\n r.touchData.startGPosition = null;\n r.touchData.didSelect = false;\n }\n if (e.touches.length < 2) {\n if (e.touches.length === 1) {\n // the old start global pos'n may not be the same finger that remains\n r.touchData.startGPosition = [e.touches[0].clientX, e.touches[0].clientY];\n }\n r.pinching = false;\n r.redrawHint('eles', true);\n r.redraw();\n }\n\n //r.redraw();\n }, false);\n\n // fallback compatibility layer for ms pointer events\n if (typeof TouchEvent === 'undefined') {\n var pointers = [];\n var makeTouch = function makeTouch(e) {\n return {\n clientX: e.clientX,\n clientY: e.clientY,\n force: 1,\n identifier: e.pointerId,\n pageX: e.pageX,\n pageY: e.pageY,\n radiusX: e.width / 2,\n radiusY: e.height / 2,\n screenX: e.screenX,\n screenY: e.screenY,\n target: e.target\n };\n };\n var makePointer = function makePointer(e) {\n return {\n event: e,\n touch: makeTouch(e)\n };\n };\n var addPointer = function addPointer(e) {\n pointers.push(makePointer(e));\n };\n var removePointer = function removePointer(e) {\n for (var i = 0; i < pointers.length; i++) {\n var p = pointers[i];\n if (p.event.pointerId === e.pointerId) {\n pointers.splice(i, 1);\n return;\n }\n }\n };\n var updatePointer = function updatePointer(e) {\n var p = pointers.filter(function (p) {\n return p.event.pointerId === e.pointerId;\n })[0];\n p.event = e;\n p.touch = makeTouch(e);\n };\n var addTouchesToEvent = function addTouchesToEvent(e) {\n e.touches = pointers.map(function (p) {\n return p.touch;\n });\n };\n var pointerIsMouse = function pointerIsMouse(e) {\n return e.pointerType === 'mouse' || e.pointerType === 4;\n };\n r.registerBinding(r.container, 'pointerdown', function (e) {\n if (pointerIsMouse(e)) {\n return;\n } // mouse already handled\n\n e.preventDefault();\n addPointer(e);\n addTouchesToEvent(e);\n touchstartHandler(e);\n });\n r.registerBinding(r.container, 'pointerup', function (e) {\n if (pointerIsMouse(e)) {\n return;\n } // mouse already handled\n\n removePointer(e);\n addTouchesToEvent(e);\n touchendHandler(e);\n });\n r.registerBinding(r.container, 'pointercancel', function (e) {\n if (pointerIsMouse(e)) {\n return;\n } // mouse already handled\n\n removePointer(e);\n addTouchesToEvent(e);\n touchcancelHandler(e);\n });\n r.registerBinding(r.container, 'pointermove', function (e) {\n if (pointerIsMouse(e)) {\n return;\n } // mouse already handled\n\n e.preventDefault();\n updatePointer(e);\n addTouchesToEvent(e);\n touchmoveHandler(e);\n });\n }\n};\n\nvar BRp$2 = {};\nBRp$2.generatePolygon = function (name, points) {\n return this.nodeShapes[name] = {\n renderer: this,\n name: name,\n points: points,\n draw: function draw(context, centerX, centerY, width, height, cornerRadius) {\n this.renderer.nodeShapeImpl('polygon', context, centerX, centerY, width, height, this.points);\n },\n intersectLine: function intersectLine(nodeX, nodeY, width, height, x, y, padding, cornerRadius) {\n return polygonIntersectLine(x, y, this.points, nodeX, nodeY, width / 2, height / 2, padding);\n },\n checkPoint: function checkPoint(x, y, padding, width, height, centerX, centerY, cornerRadius) {\n return pointInsidePolygon(x, y, this.points, centerX, centerY, width, height, [0, -1], padding);\n },\n hasMiterBounds: name !== 'rectangle',\n miterBounds: function miterBounds(centerX, centerY, width, height, strokeWidth, strokePosition) {\n return miterBox(this.points, centerX, centerY, width, height, strokeWidth);\n }\n };\n};\nBRp$2.generateEllipse = function () {\n return this.nodeShapes['ellipse'] = {\n renderer: this,\n name: 'ellipse',\n draw: function draw(context, centerX, centerY, width, height, cornerRadius) {\n this.renderer.nodeShapeImpl(this.name, context, centerX, centerY, width, height);\n },\n intersectLine: function intersectLine(nodeX, nodeY, width, height, x, y, padding, cornerRadius) {\n return intersectLineEllipse(x, y, nodeX, nodeY, width / 2 + padding, height / 2 + padding);\n },\n checkPoint: function checkPoint(x, y, padding, width, height, centerX, centerY, cornerRadius) {\n return checkInEllipse(x, y, width, height, centerX, centerY, padding);\n }\n };\n};\nBRp$2.generateRoundPolygon = function (name, points) {\n return this.nodeShapes[name] = {\n renderer: this,\n name: name,\n points: points,\n getOrCreateCorners: function getOrCreateCorners(centerX, centerY, width, height, cornerRadius, rs, field) {\n if (rs[field] !== undefined && rs[field + '-cx'] === centerX && rs[field + '-cy'] === centerY) {\n return rs[field];\n }\n rs[field] = new Array(points.length / 2);\n rs[field + '-cx'] = centerX;\n rs[field + '-cy'] = centerY;\n var halfW = width / 2;\n var halfH = height / 2;\n cornerRadius = cornerRadius === 'auto' ? getRoundPolygonRadius(width, height) : cornerRadius;\n var p = new Array(points.length / 2);\n for (var _i = 0; _i < points.length / 2; _i++) {\n p[_i] = {\n x: centerX + halfW * points[_i * 2],\n y: centerY + halfH * points[_i * 2 + 1]\n };\n }\n var i,\n p1,\n p2,\n p3,\n len = p.length;\n p1 = p[len - 1];\n // for each point\n for (i = 0; i < len; i++) {\n p2 = p[i % len];\n p3 = p[(i + 1) % len];\n rs[field][i] = getRoundCorner(p1, p2, p3, cornerRadius);\n p1 = p2;\n p2 = p3;\n }\n return rs[field];\n },\n draw: function draw(context, centerX, centerY, width, height, cornerRadius, rs) {\n this.renderer.nodeShapeImpl('round-polygon', context, centerX, centerY, width, height, this.points, this.getOrCreateCorners(centerX, centerY, width, height, cornerRadius, rs, 'drawCorners'));\n },\n intersectLine: function intersectLine(nodeX, nodeY, width, height, x, y, padding, cornerRadius, rs) {\n return roundPolygonIntersectLine(x, y, this.points, nodeX, nodeY, width, height, padding, this.getOrCreateCorners(nodeX, nodeY, width, height, cornerRadius, rs, 'corners'));\n },\n checkPoint: function checkPoint(x, y, padding, width, height, centerX, centerY, cornerRadius, rs) {\n return pointInsideRoundPolygon(x, y, this.points, centerX, centerY, width, height, this.getOrCreateCorners(centerX, centerY, width, height, cornerRadius, rs, 'corners'));\n }\n };\n};\nBRp$2.generateRoundRectangle = function () {\n return this.nodeShapes['round-rectangle'] = this.nodeShapes['roundrectangle'] = {\n renderer: this,\n name: 'round-rectangle',\n points: generateUnitNgonPointsFitToSquare(4, 0),\n draw: function draw(context, centerX, centerY, width, height, cornerRadius) {\n this.renderer.nodeShapeImpl(this.name, context, centerX, centerY, width, height, this.points, cornerRadius);\n },\n intersectLine: function intersectLine(nodeX, nodeY, width, height, x, y, padding, cornerRadius) {\n return roundRectangleIntersectLine(x, y, nodeX, nodeY, width, height, padding, cornerRadius);\n },\n checkPoint: function checkPoint(x, y, padding, width, height, centerX, centerY, cornerRadius) {\n var halfWidth = width / 2;\n var halfHeight = height / 2;\n cornerRadius = cornerRadius === 'auto' ? getRoundRectangleRadius(width, height) : cornerRadius;\n cornerRadius = Math.min(halfWidth, halfHeight, cornerRadius);\n var diam = cornerRadius * 2;\n\n // Check hBox\n if (pointInsidePolygon(x, y, this.points, centerX, centerY, width, height - diam, [0, -1], padding)) {\n return true;\n }\n\n // Check vBox\n if (pointInsidePolygon(x, y, this.points, centerX, centerY, width - diam, height, [0, -1], padding)) {\n return true;\n }\n\n // Check top left quarter circle\n if (checkInEllipse(x, y, diam, diam, centerX - halfWidth + cornerRadius, centerY - halfHeight + cornerRadius, padding)) {\n return true;\n }\n\n // Check top right quarter circle\n if (checkInEllipse(x, y, diam, diam, centerX + halfWidth - cornerRadius, centerY - halfHeight + cornerRadius, padding)) {\n return true;\n }\n\n // Check bottom right quarter circle\n if (checkInEllipse(x, y, diam, diam, centerX + halfWidth - cornerRadius, centerY + halfHeight - cornerRadius, padding)) {\n return true;\n }\n\n // Check bottom left quarter circle\n if (checkInEllipse(x, y, diam, diam, centerX - halfWidth + cornerRadius, centerY + halfHeight - cornerRadius, padding)) {\n return true;\n }\n return false;\n }\n };\n};\nBRp$2.generateCutRectangle = function () {\n return this.nodeShapes['cut-rectangle'] = this.nodeShapes['cutrectangle'] = {\n renderer: this,\n name: 'cut-rectangle',\n cornerLength: getCutRectangleCornerLength(),\n points: generateUnitNgonPointsFitToSquare(4, 0),\n draw: function draw(context, centerX, centerY, width, height, cornerRadius) {\n this.renderer.nodeShapeImpl(this.name, context, centerX, centerY, width, height, null, cornerRadius);\n },\n generateCutTrianglePts: function generateCutTrianglePts(width, height, centerX, centerY, cornerRadius) {\n var cl = cornerRadius === 'auto' ? this.cornerLength : cornerRadius;\n var hh = height / 2;\n var hw = width / 2;\n var xBegin = centerX - hw;\n var xEnd = centerX + hw;\n var yBegin = centerY - hh;\n var yEnd = centerY + hh;\n\n // points are in clockwise order, inner (imaginary) triangle pt on [4, 5]\n return {\n topLeft: [xBegin, yBegin + cl, xBegin + cl, yBegin, xBegin + cl, yBegin + cl],\n topRight: [xEnd - cl, yBegin, xEnd, yBegin + cl, xEnd - cl, yBegin + cl],\n bottomRight: [xEnd, yEnd - cl, xEnd - cl, yEnd, xEnd - cl, yEnd - cl],\n bottomLeft: [xBegin + cl, yEnd, xBegin, yEnd - cl, xBegin + cl, yEnd - cl]\n };\n },\n intersectLine: function intersectLine(nodeX, nodeY, width, height, x, y, padding, cornerRadius) {\n var cPts = this.generateCutTrianglePts(width + 2 * padding, height + 2 * padding, nodeX, nodeY, cornerRadius);\n var pts = [].concat.apply([], [cPts.topLeft.splice(0, 4), cPts.topRight.splice(0, 4), cPts.bottomRight.splice(0, 4), cPts.bottomLeft.splice(0, 4)]);\n return polygonIntersectLine(x, y, pts, nodeX, nodeY);\n },\n checkPoint: function checkPoint(x, y, padding, width, height, centerX, centerY, cornerRadius) {\n var cl = cornerRadius === 'auto' ? this.cornerLength : cornerRadius;\n // Check hBox\n if (pointInsidePolygon(x, y, this.points, centerX, centerY, width, height - 2 * cl, [0, -1], padding)) {\n return true;\n }\n\n // Check vBox\n if (pointInsidePolygon(x, y, this.points, centerX, centerY, width - 2 * cl, height, [0, -1], padding)) {\n return true;\n }\n var cutTrianglePts = this.generateCutTrianglePts(width, height, centerX, centerY);\n return pointInsidePolygonPoints(x, y, cutTrianglePts.topLeft) || pointInsidePolygonPoints(x, y, cutTrianglePts.topRight) || pointInsidePolygonPoints(x, y, cutTrianglePts.bottomRight) || pointInsidePolygonPoints(x, y, cutTrianglePts.bottomLeft);\n }\n };\n};\nBRp$2.generateBarrel = function () {\n return this.nodeShapes['barrel'] = {\n renderer: this,\n name: 'barrel',\n points: generateUnitNgonPointsFitToSquare(4, 0),\n draw: function draw(context, centerX, centerY, width, height, cornerRadius) {\n this.renderer.nodeShapeImpl(this.name, context, centerX, centerY, width, height);\n },\n intersectLine: function intersectLine(nodeX, nodeY, width, height, x, y, padding, cornerRadius) {\n // use two fixed t values for the bezier curve approximation\n\n var t0 = 0.15;\n var t1 = 0.5;\n var t2 = 0.85;\n var bPts = this.generateBarrelBezierPts(width + 2 * padding, height + 2 * padding, nodeX, nodeY);\n var approximateBarrelCurvePts = function approximateBarrelCurvePts(pts) {\n // approximate curve pts based on the two t values\n var m0 = qbezierPtAt({\n x: pts[0],\n y: pts[1]\n }, {\n x: pts[2],\n y: pts[3]\n }, {\n x: pts[4],\n y: pts[5]\n }, t0);\n var m1 = qbezierPtAt({\n x: pts[0],\n y: pts[1]\n }, {\n x: pts[2],\n y: pts[3]\n }, {\n x: pts[4],\n y: pts[5]\n }, t1);\n var m2 = qbezierPtAt({\n x: pts[0],\n y: pts[1]\n }, {\n x: pts[2],\n y: pts[3]\n }, {\n x: pts[4],\n y: pts[5]\n }, t2);\n return [pts[0], pts[1], m0.x, m0.y, m1.x, m1.y, m2.x, m2.y, pts[4], pts[5]];\n };\n var pts = [].concat(approximateBarrelCurvePts(bPts.topLeft), approximateBarrelCurvePts(bPts.topRight), approximateBarrelCurvePts(bPts.bottomRight), approximateBarrelCurvePts(bPts.bottomLeft));\n return polygonIntersectLine(x, y, pts, nodeX, nodeY);\n },\n generateBarrelBezierPts: function generateBarrelBezierPts(width, height, centerX, centerY) {\n var hh = height / 2;\n var hw = width / 2;\n var xBegin = centerX - hw;\n var xEnd = centerX + hw;\n var yBegin = centerY - hh;\n var yEnd = centerY + hh;\n var curveConstants = getBarrelCurveConstants(width, height);\n var hOffset = curveConstants.heightOffset;\n var wOffset = curveConstants.widthOffset;\n var ctrlPtXOffset = curveConstants.ctrlPtOffsetPct * width;\n\n // points are in clockwise order, inner (imaginary) control pt on [4, 5]\n var pts = {\n topLeft: [xBegin, yBegin + hOffset, xBegin + ctrlPtXOffset, yBegin, xBegin + wOffset, yBegin],\n topRight: [xEnd - wOffset, yBegin, xEnd - ctrlPtXOffset, yBegin, xEnd, yBegin + hOffset],\n bottomRight: [xEnd, yEnd - hOffset, xEnd - ctrlPtXOffset, yEnd, xEnd - wOffset, yEnd],\n bottomLeft: [xBegin + wOffset, yEnd, xBegin + ctrlPtXOffset, yEnd, xBegin, yEnd - hOffset]\n };\n pts.topLeft.isTop = true;\n pts.topRight.isTop = true;\n pts.bottomLeft.isBottom = true;\n pts.bottomRight.isBottom = true;\n return pts;\n },\n checkPoint: function checkPoint(x, y, padding, width, height, centerX, centerY, cornerRadius) {\n var curveConstants = getBarrelCurveConstants(width, height);\n var hOffset = curveConstants.heightOffset;\n var wOffset = curveConstants.widthOffset;\n\n // Check hBox\n if (pointInsidePolygon(x, y, this.points, centerX, centerY, width, height - 2 * hOffset, [0, -1], padding)) {\n return true;\n }\n\n // Check vBox\n if (pointInsidePolygon(x, y, this.points, centerX, centerY, width - 2 * wOffset, height, [0, -1], padding)) {\n return true;\n }\n var barrelCurvePts = this.generateBarrelBezierPts(width, height, centerX, centerY);\n var getCurveT = function getCurveT(x, y, curvePts) {\n var x0 = curvePts[4];\n var x1 = curvePts[2];\n var x2 = curvePts[0];\n var y0 = curvePts[5];\n // var y1 = curvePts[ 3 ];\n var y2 = curvePts[1];\n var xMin = Math.min(x0, x2);\n var xMax = Math.max(x0, x2);\n var yMin = Math.min(y0, y2);\n var yMax = Math.max(y0, y2);\n if (xMin <= x && x <= xMax && yMin <= y && y <= yMax) {\n var coeff = bezierPtsToQuadCoeff(x0, x1, x2);\n var roots = solveQuadratic(coeff[0], coeff[1], coeff[2], x);\n var validRoots = roots.filter(function (r) {\n return 0 <= r && r <= 1;\n });\n if (validRoots.length > 0) {\n return validRoots[0];\n }\n }\n return null;\n };\n var curveRegions = Object.keys(barrelCurvePts);\n for (var i = 0; i < curveRegions.length; i++) {\n var corner = curveRegions[i];\n var cornerPts = barrelCurvePts[corner];\n var t = getCurveT(x, y, cornerPts);\n if (t == null) {\n continue;\n }\n var y0 = cornerPts[5];\n var y1 = cornerPts[3];\n var y2 = cornerPts[1];\n var bezY = qbezierAt(y0, y1, y2, t);\n if (cornerPts.isTop && bezY <= y) {\n return true;\n }\n if (cornerPts.isBottom && y <= bezY) {\n return true;\n }\n }\n return false;\n }\n };\n};\nBRp$2.generateBottomRoundrectangle = function () {\n return this.nodeShapes['bottom-round-rectangle'] = this.nodeShapes['bottomroundrectangle'] = {\n renderer: this,\n name: 'bottom-round-rectangle',\n points: generateUnitNgonPointsFitToSquare(4, 0),\n draw: function draw(context, centerX, centerY, width, height, cornerRadius) {\n this.renderer.nodeShapeImpl(this.name, context, centerX, centerY, width, height, this.points, cornerRadius);\n },\n intersectLine: function intersectLine(nodeX, nodeY, width, height, x, y, padding, cornerRadius) {\n var topStartX = nodeX - (width / 2 + padding);\n var topStartY = nodeY - (height / 2 + padding);\n var topEndY = topStartY;\n var topEndX = nodeX + (width / 2 + padding);\n var topIntersections = finiteLinesIntersect(x, y, nodeX, nodeY, topStartX, topStartY, topEndX, topEndY, false);\n if (topIntersections.length > 0) {\n return topIntersections;\n }\n return roundRectangleIntersectLine(x, y, nodeX, nodeY, width, height, padding, cornerRadius);\n },\n checkPoint: function checkPoint(x, y, padding, width, height, centerX, centerY, cornerRadius) {\n cornerRadius = cornerRadius === 'auto' ? getRoundRectangleRadius(width, height) : cornerRadius;\n var diam = 2 * cornerRadius;\n\n // Check hBox\n if (pointInsidePolygon(x, y, this.points, centerX, centerY, width, height - diam, [0, -1], padding)) {\n return true;\n }\n\n // Check vBox\n if (pointInsidePolygon(x, y, this.points, centerX, centerY, width - diam, height, [0, -1], padding)) {\n return true;\n }\n\n // check non-rounded top side\n var outerWidth = width / 2 + 2 * padding;\n var outerHeight = height / 2 + 2 * padding;\n var points = [centerX - outerWidth, centerY - outerHeight, centerX - outerWidth, centerY, centerX + outerWidth, centerY, centerX + outerWidth, centerY - outerHeight];\n if (pointInsidePolygonPoints(x, y, points)) {\n return true;\n }\n\n // Check bottom right quarter circle\n if (checkInEllipse(x, y, diam, diam, centerX + width / 2 - cornerRadius, centerY + height / 2 - cornerRadius, padding)) {\n return true;\n }\n\n // Check bottom left quarter circle\n if (checkInEllipse(x, y, diam, diam, centerX - width / 2 + cornerRadius, centerY + height / 2 - cornerRadius, padding)) {\n return true;\n }\n return false;\n }\n };\n};\nBRp$2.registerNodeShapes = function () {\n var nodeShapes = this.nodeShapes = {};\n var renderer = this;\n this.generateEllipse();\n this.generatePolygon('triangle', generateUnitNgonPointsFitToSquare(3, 0));\n this.generateRoundPolygon('round-triangle', generateUnitNgonPointsFitToSquare(3, 0));\n this.generatePolygon('rectangle', generateUnitNgonPointsFitToSquare(4, 0));\n nodeShapes['square'] = nodeShapes['rectangle'];\n this.generateRoundRectangle();\n this.generateCutRectangle();\n this.generateBarrel();\n this.generateBottomRoundrectangle();\n {\n var diamondPoints = [0, 1, 1, 0, 0, -1, -1, 0];\n this.generatePolygon('diamond', diamondPoints);\n this.generateRoundPolygon('round-diamond', diamondPoints);\n }\n this.generatePolygon('pentagon', generateUnitNgonPointsFitToSquare(5, 0));\n this.generateRoundPolygon('round-pentagon', generateUnitNgonPointsFitToSquare(5, 0));\n this.generatePolygon('hexagon', generateUnitNgonPointsFitToSquare(6, 0));\n this.generateRoundPolygon('round-hexagon', generateUnitNgonPointsFitToSquare(6, 0));\n this.generatePolygon('heptagon', generateUnitNgonPointsFitToSquare(7, 0));\n this.generateRoundPolygon('round-heptagon', generateUnitNgonPointsFitToSquare(7, 0));\n this.generatePolygon('octagon', generateUnitNgonPointsFitToSquare(8, 0));\n this.generateRoundPolygon('round-octagon', generateUnitNgonPointsFitToSquare(8, 0));\n var star5Points = new Array(20);\n {\n var outerPoints = generateUnitNgonPoints(5, 0);\n var innerPoints = generateUnitNgonPoints(5, Math.PI / 5);\n\n // Outer radius is 1; inner radius of star is smaller\n var innerRadius = 0.5 * (3 - Math.sqrt(5));\n innerRadius *= 1.57;\n for (var i = 0; i < innerPoints.length / 2; i++) {\n innerPoints[i * 2] *= innerRadius;\n innerPoints[i * 2 + 1] *= innerRadius;\n }\n for (var i = 0; i < 20 / 4; i++) {\n star5Points[i * 4] = outerPoints[i * 2];\n star5Points[i * 4 + 1] = outerPoints[i * 2 + 1];\n star5Points[i * 4 + 2] = innerPoints[i * 2];\n star5Points[i * 4 + 3] = innerPoints[i * 2 + 1];\n }\n }\n star5Points = fitPolygonToSquare(star5Points);\n this.generatePolygon('star', star5Points);\n this.generatePolygon('vee', [-1, -1, 0, -0.333, 1, -1, 0, 1]);\n this.generatePolygon('rhomboid', [-1, -1, 0.333, -1, 1, 1, -0.333, 1]);\n this.generatePolygon('right-rhomboid', [-0.333, -1, 1, -1, 0.333, 1, -1, 1]);\n this.nodeShapes['concavehexagon'] = this.generatePolygon('concave-hexagon', [-1, -0.95, -0.75, 0, -1, 0.95, 1, 0.95, 0.75, 0, 1, -0.95]);\n {\n var tagPoints = [-1, -1, 0.25, -1, 1, 0, 0.25, 1, -1, 1];\n this.generatePolygon('tag', tagPoints);\n this.generateRoundPolygon('round-tag', tagPoints);\n }\n nodeShapes.makePolygon = function (points) {\n // use caching on user-specified polygons so they are as fast as native shapes\n\n var key = points.join('$');\n var name = 'polygon-' + key;\n var shape;\n if (shape = this[name]) {\n // got cached shape\n return shape;\n }\n\n // create and cache new shape\n return renderer.generatePolygon(name, points);\n };\n};\n\nvar BRp$1 = {};\nBRp$1.timeToRender = function () {\n return this.redrawTotalTime / this.redrawCount;\n};\nBRp$1.redraw = function (options) {\n options = options || staticEmptyObject();\n var r = this;\n if (r.averageRedrawTime === undefined) {\n r.averageRedrawTime = 0;\n }\n if (r.lastRedrawTime === undefined) {\n r.lastRedrawTime = 0;\n }\n if (r.lastDrawTime === undefined) {\n r.lastDrawTime = 0;\n }\n r.requestedFrame = true;\n r.renderOptions = options;\n};\nBRp$1.beforeRender = function (fn, priority) {\n // the renderer can't add tick callbacks when destroyed\n if (this.destroyed) {\n return;\n }\n if (priority == null) {\n error('Priority is not optional for beforeRender');\n }\n var cbs = this.beforeRenderCallbacks;\n cbs.push({\n fn: fn,\n priority: priority\n });\n\n // higher priority callbacks executed first\n cbs.sort(function (a, b) {\n return b.priority - a.priority;\n });\n};\nvar beforeRenderCallbacks = function beforeRenderCallbacks(r, willDraw, startTime) {\n var cbs = r.beforeRenderCallbacks;\n for (var i = 0; i < cbs.length; i++) {\n cbs[i].fn(willDraw, startTime);\n }\n};\nBRp$1.startRenderLoop = function () {\n var r = this;\n var cy = r.cy;\n if (r.renderLoopStarted) {\n return;\n } else {\n r.renderLoopStarted = true;\n }\n var _renderFn = function renderFn(requestTime) {\n if (r.destroyed) {\n return;\n }\n if (cy.batching()) ; else if (r.requestedFrame && !r.skipFrame) {\n beforeRenderCallbacks(r, true, requestTime);\n var startTime = performanceNow();\n r.render(r.renderOptions);\n var endTime = r.lastDrawTime = performanceNow();\n if (r.averageRedrawTime === undefined) {\n r.averageRedrawTime = endTime - startTime;\n }\n if (r.redrawCount === undefined) {\n r.redrawCount = 0;\n }\n r.redrawCount++;\n if (r.redrawTotalTime === undefined) {\n r.redrawTotalTime = 0;\n }\n var duration = endTime - startTime;\n r.redrawTotalTime += duration;\n r.lastRedrawTime = duration;\n\n // use a weighted average with a bias from the previous average so we don't spike so easily\n r.averageRedrawTime = r.averageRedrawTime / 2 + duration / 2;\n r.requestedFrame = false;\n } else {\n beforeRenderCallbacks(r, false, requestTime);\n }\n r.skipFrame = false;\n requestAnimationFrame(_renderFn);\n };\n requestAnimationFrame(_renderFn);\n};\n\nvar BaseRenderer = function BaseRenderer(options) {\n this.init(options);\n};\nvar BR = BaseRenderer;\nvar BRp = BR.prototype;\nBRp.clientFunctions = ['redrawHint', 'render', 'renderTo', 'matchCanvasSize', 'nodeShapeImpl', 'arrowShapeImpl'];\nBRp.init = function (options) {\n var r = this;\n r.options = options;\n r.cy = options.cy;\n var ctr = r.container = options.cy.container();\n var containerWindow = r.cy.window();\n\n // prepend a stylesheet in the head such that\n if (containerWindow) {\n var document = containerWindow.document;\n var head = document.head;\n var stylesheetId = '__________cytoscape_stylesheet';\n var className = '__________cytoscape_container';\n var stylesheetAlreadyExists = document.getElementById(stylesheetId) != null;\n if (ctr.className.indexOf(className) < 0) {\n ctr.className = (ctr.className || '') + ' ' + className;\n }\n if (!stylesheetAlreadyExists) {\n var stylesheet = document.createElement('style');\n stylesheet.id = stylesheetId;\n stylesheet.textContent = '.' + className + ' { position: relative; }';\n head.insertBefore(stylesheet, head.children[0]); // first so lowest priority\n }\n var computedStyle = containerWindow.getComputedStyle(ctr);\n var position = computedStyle.getPropertyValue('position');\n if (position === 'static') {\n warn('A Cytoscape container has style position:static and so can not use UI extensions properly');\n }\n }\n r.selection = [undefined, undefined, undefined, undefined, 0]; // Coordinates for selection box, plus enabled flag\n\n r.bezierProjPcts = [0.05, 0.225, 0.4, 0.5, 0.6, 0.775, 0.95];\n\n //--Pointer-related data\n r.hoverData = {\n down: null,\n last: null,\n downTime: null,\n triggerMode: null,\n dragging: false,\n initialPan: [null, null],\n capture: false\n };\n r.dragData = {\n possibleDragElements: []\n };\n r.touchData = {\n start: null,\n capture: false,\n // These 3 fields related to tap, taphold events\n startPosition: [null, null, null, null, null, null],\n singleTouchStartTime: null,\n singleTouchMoved: true,\n now: [null, null, null, null, null, null],\n earlier: [null, null, null, null, null, null]\n };\n r.redraws = 0;\n r.showFps = options.showFps;\n r.debug = options.debug;\n r.webgl = options.webgl;\n r.hideEdgesOnViewport = options.hideEdgesOnViewport;\n r.textureOnViewport = options.textureOnViewport;\n r.wheelSensitivity = options.wheelSensitivity;\n r.motionBlurEnabled = options.motionBlur; // on by default\n r.forcedPixelRatio = number$1(options.pixelRatio) ? options.pixelRatio : null;\n r.motionBlur = options.motionBlur; // for initial kick off\n r.motionBlurOpacity = options.motionBlurOpacity;\n r.motionBlurTransparency = 1 - r.motionBlurOpacity;\n r.motionBlurPxRatio = 1;\n r.mbPxRBlurry = 1; //0.8;\n r.minMbLowQualFrames = 4;\n r.fullQualityMb = false;\n r.clearedForMotionBlur = [];\n r.desktopTapThreshold = options.desktopTapThreshold;\n r.desktopTapThreshold2 = options.desktopTapThreshold * options.desktopTapThreshold;\n r.touchTapThreshold = options.touchTapThreshold;\n r.touchTapThreshold2 = options.touchTapThreshold * options.touchTapThreshold;\n r.tapholdDuration = 500;\n r.bindings = [];\n r.beforeRenderCallbacks = [];\n r.beforeRenderPriorities = {\n // higher priority execs before lower one\n animations: 400,\n eleCalcs: 300,\n eleTxrDeq: 200,\n lyrTxrDeq: 150,\n lyrTxrSkip: 100\n };\n r.registerNodeShapes();\n r.registerArrowShapes();\n r.registerCalculationListeners();\n};\nBRp.notify = function (eventName, eles) {\n var r = this;\n var cy = r.cy;\n\n // the renderer can't be notified after it's destroyed\n if (this.destroyed) {\n return;\n }\n if (eventName === 'init') {\n r.load();\n return;\n }\n if (eventName === 'destroy') {\n r.destroy();\n return;\n }\n if (eventName === 'add' || eventName === 'remove' || eventName === 'move' && cy.hasCompoundNodes() || eventName === 'load' || eventName === 'zorder' || eventName === 'mount') {\n r.invalidateCachedZSortedEles();\n }\n if (eventName === 'viewport') {\n r.redrawHint('select', true);\n }\n if (eventName === 'gc') {\n r.redrawHint('gc', true);\n }\n if (eventName === 'load' || eventName === 'resize' || eventName === 'mount') {\n r.invalidateContainerClientCoordsCache();\n r.matchCanvasSize(r.container);\n }\n r.redrawHint('eles', true);\n r.redrawHint('drag', true);\n this.startRenderLoop();\n this.redraw();\n};\nBRp.destroy = function () {\n var r = this;\n r.destroyed = true;\n r.cy.stopAnimationLoop();\n for (var i = 0; i < r.bindings.length; i++) {\n var binding = r.bindings[i];\n var b = binding;\n var tgt = b.target;\n (tgt.off || tgt.removeEventListener).apply(tgt, b.args);\n }\n r.bindings = [];\n r.beforeRenderCallbacks = [];\n r.onUpdateEleCalcsFns = [];\n if (r.removeObserver) {\n r.removeObserver.disconnect();\n }\n if (r.styleObserver) {\n r.styleObserver.disconnect();\n }\n if (r.resizeObserver) {\n r.resizeObserver.disconnect();\n }\n if (r.labelCalcDiv) {\n try {\n document.body.removeChild(r.labelCalcDiv); // eslint-disable-line no-undef\n } catch (e) {\n // ie10 issue #1014\n }\n }\n};\nBRp.isHeadless = function () {\n return false;\n};\n[BRp$f, BRp$5, BRp$4, BRp$3, BRp$2, BRp$1].forEach(function (props) {\n extend(BRp, props);\n});\n\nvar fullFpsTime = 1000 / 60; // assume 60 frames per second\n\nvar defs = {\n setupDequeueing: function setupDequeueing(opts) {\n return function setupDequeueingImpl() {\n var self = this;\n var r = this.renderer;\n if (self.dequeueingSetup) {\n return;\n } else {\n self.dequeueingSetup = true;\n }\n var queueRedraw = debounce(function () {\n r.redrawHint('eles', true);\n r.redrawHint('drag', true);\n r.redraw();\n }, opts.deqRedrawThreshold);\n var dequeue = function dequeue(willDraw, frameStartTime) {\n var startTime = performanceNow();\n var avgRenderTime = r.averageRedrawTime;\n var renderTime = r.lastRedrawTime;\n var deqd = [];\n var extent = r.cy.extent();\n var pixelRatio = r.getPixelRatio();\n\n // if we aren't in a tick that causes a draw, then the rendered style\n // queue won't automatically be flushed before dequeueing starts\n if (!willDraw) {\n r.flushRenderedStyleQueue();\n }\n while (true) {\n // eslint-disable-line no-constant-condition\n var now = performanceNow();\n var duration = now - startTime;\n var frameDuration = now - frameStartTime;\n if (renderTime < fullFpsTime) {\n // if we're rendering faster than the ideal fps, then do dequeueing\n // during all of the remaining frame time\n\n var timeAvailable = fullFpsTime - (willDraw ? avgRenderTime : 0);\n if (frameDuration >= opts.deqFastCost * timeAvailable) {\n break;\n }\n } else {\n if (willDraw) {\n if (duration >= opts.deqCost * renderTime || duration >= opts.deqAvgCost * avgRenderTime) {\n break;\n }\n } else if (frameDuration >= opts.deqNoDrawCost * fullFpsTime) {\n break;\n }\n }\n var thisDeqd = opts.deq(self, pixelRatio, extent);\n if (thisDeqd.length > 0) {\n for (var i = 0; i < thisDeqd.length; i++) {\n deqd.push(thisDeqd[i]);\n }\n } else {\n break;\n }\n }\n\n // callbacks on dequeue\n if (deqd.length > 0) {\n opts.onDeqd(self, deqd);\n if (!willDraw && opts.shouldRedraw(self, deqd, pixelRatio, extent)) {\n queueRedraw();\n }\n }\n };\n var priority = opts.priority || noop$1;\n r.beforeRender(dequeue, priority(self));\n };\n }\n};\n\n// Allows lookups for (ele, lvl) => cache.\n// Uses keys so elements may share the same cache.\nvar ElementTextureCacheLookup = /*#__PURE__*/function () {\n function ElementTextureCacheLookup(getKey) {\n var doesEleInvalidateKey = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : falsify;\n _classCallCheck(this, ElementTextureCacheLookup);\n this.idsByKey = new Map$1();\n this.keyForId = new Map$1();\n this.cachesByLvl = new Map$1();\n this.lvls = [];\n this.getKey = getKey;\n this.doesEleInvalidateKey = doesEleInvalidateKey;\n }\n return _createClass(ElementTextureCacheLookup, [{\n key: \"getIdsFor\",\n value: function getIdsFor(key) {\n if (key == null) {\n error(\"Can not get id list for null key\");\n }\n var idsByKey = this.idsByKey;\n var ids = this.idsByKey.get(key);\n if (!ids) {\n ids = new Set$1();\n idsByKey.set(key, ids);\n }\n return ids;\n }\n }, {\n key: \"addIdForKey\",\n value: function addIdForKey(key, id) {\n if (key != null) {\n this.getIdsFor(key).add(id);\n }\n }\n }, {\n key: \"deleteIdForKey\",\n value: function deleteIdForKey(key, id) {\n if (key != null) {\n this.getIdsFor(key)[\"delete\"](id);\n }\n }\n }, {\n key: \"getNumberOfIdsForKey\",\n value: function getNumberOfIdsForKey(key) {\n if (key == null) {\n return 0;\n } else {\n return this.getIdsFor(key).size;\n }\n }\n }, {\n key: \"updateKeyMappingFor\",\n value: function updateKeyMappingFor(ele) {\n var id = ele.id();\n var prevKey = this.keyForId.get(id);\n var currKey = this.getKey(ele);\n this.deleteIdForKey(prevKey, id);\n this.addIdForKey(currKey, id);\n this.keyForId.set(id, currKey);\n }\n }, {\n key: \"deleteKeyMappingFor\",\n value: function deleteKeyMappingFor(ele) {\n var id = ele.id();\n var prevKey = this.keyForId.get(id);\n this.deleteIdForKey(prevKey, id);\n this.keyForId[\"delete\"](id);\n }\n }, {\n key: \"keyHasChangedFor\",\n value: function keyHasChangedFor(ele) {\n var id = ele.id();\n var prevKey = this.keyForId.get(id);\n var newKey = this.getKey(ele);\n return prevKey !== newKey;\n }\n }, {\n key: \"isInvalid\",\n value: function isInvalid(ele) {\n return this.keyHasChangedFor(ele) || this.doesEleInvalidateKey(ele);\n }\n }, {\n key: \"getCachesAt\",\n value: function getCachesAt(lvl) {\n var cachesByLvl = this.cachesByLvl,\n lvls = this.lvls;\n var caches = cachesByLvl.get(lvl);\n if (!caches) {\n caches = new Map$1();\n cachesByLvl.set(lvl, caches);\n lvls.push(lvl);\n }\n return caches;\n }\n }, {\n key: \"getCache\",\n value: function getCache(key, lvl) {\n return this.getCachesAt(lvl).get(key);\n }\n }, {\n key: \"get\",\n value: function get(ele, lvl) {\n var key = this.getKey(ele);\n var cache = this.getCache(key, lvl);\n\n // getting for an element may need to add to the id list b/c eles can share keys\n if (cache != null) {\n this.updateKeyMappingFor(ele);\n }\n return cache;\n }\n }, {\n key: \"getForCachedKey\",\n value: function getForCachedKey(ele, lvl) {\n var key = this.keyForId.get(ele.id()); // n.b. use cached key, not newly computed key\n var cache = this.getCache(key, lvl);\n return cache;\n }\n }, {\n key: \"hasCache\",\n value: function hasCache(key, lvl) {\n return this.getCachesAt(lvl).has(key);\n }\n }, {\n key: \"has\",\n value: function has(ele, lvl) {\n var key = this.getKey(ele);\n return this.hasCache(key, lvl);\n }\n }, {\n key: \"setCache\",\n value: function setCache(key, lvl, cache) {\n cache.key = key;\n this.getCachesAt(lvl).set(key, cache);\n }\n }, {\n key: \"set\",\n value: function set(ele, lvl, cache) {\n var key = this.getKey(ele);\n this.setCache(key, lvl, cache);\n this.updateKeyMappingFor(ele);\n }\n }, {\n key: \"deleteCache\",\n value: function deleteCache(key, lvl) {\n this.getCachesAt(lvl)[\"delete\"](key);\n }\n }, {\n key: \"delete\",\n value: function _delete(ele, lvl) {\n var key = this.getKey(ele);\n this.deleteCache(key, lvl);\n }\n }, {\n key: \"invalidateKey\",\n value: function invalidateKey(key) {\n var _this = this;\n this.lvls.forEach(function (lvl) {\n return _this.deleteCache(key, lvl);\n });\n }\n\n // returns true if no other eles reference the invalidated cache (n.b. other eles may need the cache with the same key)\n }, {\n key: \"invalidate\",\n value: function invalidate(ele) {\n var id = ele.id();\n var key = this.keyForId.get(id); // n.b. use stored key rather than current (potential key)\n\n this.deleteKeyMappingFor(ele);\n var entireKeyInvalidated = this.doesEleInvalidateKey(ele);\n if (entireKeyInvalidated) {\n // clear mapping for current key\n this.invalidateKey(key);\n }\n return entireKeyInvalidated || this.getNumberOfIdsForKey(key) === 0;\n }\n }]);\n}();\n\nvar minTxrH = 25; // the size of the texture cache for small height eles (special case)\nvar txrStepH = 50; // the min size of the regular cache, and the size it increases with each step up\nvar minLvl$1 = -4; // when scaling smaller than that we don't need to re-render\nvar maxLvl$1 = 3; // when larger than this scale just render directly (caching is not helpful)\nvar maxZoom$1 = 7.99; // beyond this zoom level, layered textures are not used\nvar eleTxrSpacing = 8; // spacing between elements on textures to avoid blitting overlaps\nvar defTxrWidth = 1024; // default/minimum texture width\nvar maxTxrW = 1024; // the maximum width of a texture\nvar maxTxrH = 1024; // the maximum height of a texture\nvar minUtility = 0.2; // if usage of texture is less than this, it is retired\nvar maxFullness = 0.8; // fullness of texture after which queue removal is checked\nvar maxFullnessChecks = 10; // dequeued after this many checks\nvar deqCost$1 = 0.15; // % of add'l rendering cost allowed for dequeuing ele caches each frame\nvar deqAvgCost$1 = 0.1; // % of add'l rendering cost compared to average overall redraw time\nvar deqNoDrawCost$1 = 0.9; // % of avg frame time that can be used for dequeueing when not drawing\nvar deqFastCost$1 = 0.9; // % of frame time to be used when >60fps\nvar deqRedrawThreshold$1 = 100; // time to batch redraws together from dequeueing to allow more dequeueing calcs to happen in the meanwhile\nvar maxDeqSize$1 = 1; // number of eles to dequeue and render at higher texture in each batch\n\nvar getTxrReasons = {\n dequeue: 'dequeue',\n downscale: 'downscale',\n highQuality: 'highQuality'\n};\nvar initDefaults = defaults$g({\n getKey: null,\n doesEleInvalidateKey: falsify,\n drawElement: null,\n getBoundingBox: null,\n getRotationPoint: null,\n getRotationOffset: null,\n isVisible: trueify,\n allowEdgeTxrCaching: true,\n allowParentTxrCaching: true\n});\nvar ElementTextureCache = function ElementTextureCache(renderer, initOptions) {\n var self = this;\n self.renderer = renderer;\n self.onDequeues = [];\n var opts = initDefaults(initOptions);\n extend(self, opts);\n self.lookup = new ElementTextureCacheLookup(opts.getKey, opts.doesEleInvalidateKey);\n self.setupDequeueing();\n};\nvar ETCp = ElementTextureCache.prototype;\nETCp.reasons = getTxrReasons;\n\n// the list of textures in which new subtextures for elements can be placed\nETCp.getTextureQueue = function (txrH) {\n var self = this;\n self.eleImgCaches = self.eleImgCaches || {};\n return self.eleImgCaches[txrH] = self.eleImgCaches[txrH] || [];\n};\n\n// the list of usused textures which can be recycled (in use in texture queue)\nETCp.getRetiredTextureQueue = function (txrH) {\n var self = this;\n var rtxtrQs = self.eleImgCaches.retired = self.eleImgCaches.retired || {};\n var rtxtrQ = rtxtrQs[txrH] = rtxtrQs[txrH] || [];\n return rtxtrQ;\n};\n\n// queue of element draw requests at different scale levels\nETCp.getElementQueue = function () {\n var self = this;\n var q = self.eleCacheQueue = self.eleCacheQueue || new Heap(function (a, b) {\n return b.reqs - a.reqs;\n });\n return q;\n};\n\n// queue of element draw requests at different scale levels (element id lookup)\nETCp.getElementKeyToQueue = function () {\n var self = this;\n var k2q = self.eleKeyToCacheQueue = self.eleKeyToCacheQueue || {};\n return k2q;\n};\nETCp.getElement = function (ele, bb, pxRatio, lvl, reason) {\n var self = this;\n var r = this.renderer;\n var zoom = r.cy.zoom();\n var lookup = this.lookup;\n if (!bb || bb.w === 0 || bb.h === 0 || isNaN(bb.w) || isNaN(bb.h) || !ele.visible() || ele.removed()) {\n return null;\n }\n if (!self.allowEdgeTxrCaching && ele.isEdge() || !self.allowParentTxrCaching && ele.isParent()) {\n return null;\n }\n if (lvl == null) {\n lvl = Math.ceil(log2(zoom * pxRatio));\n }\n if (lvl < minLvl$1) {\n lvl = minLvl$1;\n } else if (zoom >= maxZoom$1 || lvl > maxLvl$1) {\n return null;\n }\n var scale = Math.pow(2, lvl);\n var eleScaledH = bb.h * scale;\n var eleScaledW = bb.w * scale;\n var scaledLabelShown = r.eleTextBiggerThanMin(ele, scale);\n if (!this.isVisible(ele, scaledLabelShown)) {\n return null;\n }\n var eleCache = lookup.get(ele, lvl);\n\n // if this get was on an unused/invalidated cache, then restore the texture usage metric\n if (eleCache && eleCache.invalidated) {\n eleCache.invalidated = false;\n eleCache.texture.invalidatedWidth -= eleCache.width;\n }\n if (eleCache) {\n return eleCache;\n }\n var txrH; // which texture height this ele belongs to\n\n if (eleScaledH <= minTxrH) {\n txrH = minTxrH;\n } else if (eleScaledH <= txrStepH) {\n txrH = txrStepH;\n } else {\n txrH = Math.ceil(eleScaledH / txrStepH) * txrStepH;\n }\n if (eleScaledH > maxTxrH || eleScaledW > maxTxrW) {\n return null; // caching large elements is not efficient\n }\n var txrQ = self.getTextureQueue(txrH);\n\n // first try the second last one in case it has space at the end\n var txr = txrQ[txrQ.length - 2];\n var addNewTxr = function addNewTxr() {\n return self.recycleTexture(txrH, eleScaledW) || self.addTexture(txrH, eleScaledW);\n };\n\n // try the last one if there is no second last one\n if (!txr) {\n txr = txrQ[txrQ.length - 1];\n }\n\n // if the last one doesn't exist, we need a first one\n if (!txr) {\n txr = addNewTxr();\n }\n\n // if there's no room in the current texture, we need a new one\n if (txr.width - txr.usedWidth < eleScaledW) {\n txr = addNewTxr();\n }\n var scalableFrom = function scalableFrom(otherCache) {\n return otherCache && otherCache.scaledLabelShown === scaledLabelShown;\n };\n var deqing = reason && reason === getTxrReasons.dequeue;\n var highQualityReq = reason && reason === getTxrReasons.highQuality;\n var downscaleReq = reason && reason === getTxrReasons.downscale;\n var higherCache; // the nearest cache with a higher level\n for (var l = lvl + 1; l <= maxLvl$1; l++) {\n var c = lookup.get(ele, l);\n if (c) {\n higherCache = c;\n break;\n }\n }\n var oneUpCache = higherCache && higherCache.level === lvl + 1 ? higherCache : null;\n var downscale = function downscale() {\n txr.context.drawImage(oneUpCache.texture.canvas, oneUpCache.x, 0, oneUpCache.width, oneUpCache.height, txr.usedWidth, 0, eleScaledW, eleScaledH);\n };\n\n // reset ele area in texture\n txr.context.setTransform(1, 0, 0, 1, 0, 0);\n txr.context.clearRect(txr.usedWidth, 0, eleScaledW, txrH);\n if (scalableFrom(oneUpCache)) {\n // then we can relatively cheaply rescale the existing image w/o rerendering\n downscale();\n } else if (scalableFrom(higherCache)) {\n // then use the higher cache for now and queue the next level down\n // to cheaply scale towards the smaller level\n\n if (highQualityReq) {\n for (var _l = higherCache.level; _l > lvl; _l--) {\n oneUpCache = self.getElement(ele, bb, pxRatio, _l, getTxrReasons.downscale);\n }\n downscale();\n } else {\n self.queueElement(ele, higherCache.level - 1);\n return higherCache;\n }\n } else {\n var lowerCache; // the nearest cache with a lower level\n if (!deqing && !highQualityReq && !downscaleReq) {\n for (var _l2 = lvl - 1; _l2 >= minLvl$1; _l2--) {\n var _c = lookup.get(ele, _l2);\n if (_c) {\n lowerCache = _c;\n break;\n }\n }\n }\n if (scalableFrom(lowerCache)) {\n // then use the lower quality cache for now and queue the better one for later\n\n self.queueElement(ele, lvl);\n return lowerCache;\n }\n txr.context.translate(txr.usedWidth, 0);\n txr.context.scale(scale, scale);\n this.drawElement(txr.context, ele, bb, scaledLabelShown, false);\n txr.context.scale(1 / scale, 1 / scale);\n txr.context.translate(-txr.usedWidth, 0);\n }\n eleCache = {\n x: txr.usedWidth,\n texture: txr,\n level: lvl,\n scale: scale,\n width: eleScaledW,\n height: eleScaledH,\n scaledLabelShown: scaledLabelShown\n };\n txr.usedWidth += Math.ceil(eleScaledW + eleTxrSpacing);\n txr.eleCaches.push(eleCache);\n lookup.set(ele, lvl, eleCache);\n self.checkTextureFullness(txr);\n return eleCache;\n};\nETCp.invalidateElements = function (eles) {\n for (var i = 0; i < eles.length; i++) {\n this.invalidateElement(eles[i]);\n }\n};\nETCp.invalidateElement = function (ele) {\n var self = this;\n var lookup = self.lookup;\n var caches = [];\n var invalid = lookup.isInvalid(ele);\n if (!invalid) {\n return; // override the invalidation request if the element key has not changed\n }\n for (var lvl = minLvl$1; lvl <= maxLvl$1; lvl++) {\n var cache = lookup.getForCachedKey(ele, lvl);\n if (cache) {\n caches.push(cache);\n }\n }\n var noOtherElesUseCache = lookup.invalidate(ele);\n if (noOtherElesUseCache) {\n for (var i = 0; i < caches.length; i++) {\n var _cache = caches[i];\n var txr = _cache.texture;\n\n // remove space from the texture it belongs to\n txr.invalidatedWidth += _cache.width;\n\n // mark the cache as invalidated\n _cache.invalidated = true;\n\n // retire the texture if its utility is low\n self.checkTextureUtility(txr);\n }\n }\n\n // remove from queue since the old req was for the old state\n self.removeFromQueue(ele);\n};\nETCp.checkTextureUtility = function (txr) {\n // invalidate all entries in the cache if the cache size is small\n if (txr.invalidatedWidth >= minUtility * txr.width) {\n this.retireTexture(txr);\n }\n};\nETCp.checkTextureFullness = function (txr) {\n // if texture has been mostly filled and passed over several times, remove\n // it from the queue so we don't need to waste time looking at it to put new things\n\n var self = this;\n var txrQ = self.getTextureQueue(txr.height);\n if (txr.usedWidth / txr.width > maxFullness && txr.fullnessChecks >= maxFullnessChecks) {\n removeFromArray(txrQ, txr);\n } else {\n txr.fullnessChecks++;\n }\n};\nETCp.retireTexture = function (txr) {\n var self = this;\n var txrH = txr.height;\n var txrQ = self.getTextureQueue(txrH);\n var lookup = this.lookup;\n\n // retire the texture from the active / searchable queue:\n\n removeFromArray(txrQ, txr);\n txr.retired = true;\n\n // remove the refs from the eles to the caches:\n\n var eleCaches = txr.eleCaches;\n for (var i = 0; i < eleCaches.length; i++) {\n var eleCache = eleCaches[i];\n lookup.deleteCache(eleCache.key, eleCache.level);\n }\n clearArray(eleCaches);\n\n // add the texture to a retired queue so it can be recycled in future:\n\n var rtxtrQ = self.getRetiredTextureQueue(txrH);\n rtxtrQ.push(txr);\n};\nETCp.addTexture = function (txrH, minW) {\n var self = this;\n var txrQ = self.getTextureQueue(txrH);\n var txr = {};\n txrQ.push(txr);\n txr.eleCaches = [];\n txr.height = txrH;\n txr.width = Math.max(defTxrWidth, minW);\n txr.usedWidth = 0;\n txr.invalidatedWidth = 0;\n txr.fullnessChecks = 0;\n txr.canvas = self.renderer.makeOffscreenCanvas(txr.width, txr.height);\n txr.context = txr.canvas.getContext('2d');\n return txr;\n};\nETCp.recycleTexture = function (txrH, minW) {\n var self = this;\n var txrQ = self.getTextureQueue(txrH);\n var rtxtrQ = self.getRetiredTextureQueue(txrH);\n for (var i = 0; i < rtxtrQ.length; i++) {\n var txr = rtxtrQ[i];\n if (txr.width >= minW) {\n txr.retired = false;\n txr.usedWidth = 0;\n txr.invalidatedWidth = 0;\n txr.fullnessChecks = 0;\n clearArray(txr.eleCaches);\n txr.context.setTransform(1, 0, 0, 1, 0, 0);\n txr.context.clearRect(0, 0, txr.width, txr.height);\n removeFromArray(rtxtrQ, txr);\n txrQ.push(txr);\n return txr;\n }\n }\n};\nETCp.queueElement = function (ele, lvl) {\n var self = this;\n var q = self.getElementQueue();\n var k2q = self.getElementKeyToQueue();\n var key = this.getKey(ele);\n var existingReq = k2q[key];\n if (existingReq) {\n // use the max lvl b/c in between lvls are cheap to make\n existingReq.level = Math.max(existingReq.level, lvl);\n existingReq.eles.merge(ele);\n existingReq.reqs++;\n q.updateItem(existingReq);\n } else {\n var req = {\n eles: ele.spawn().merge(ele),\n level: lvl,\n reqs: 1,\n key: key\n };\n q.push(req);\n k2q[key] = req;\n }\n};\nETCp.dequeue = function (pxRatio /*, extent*/) {\n var self = this;\n var q = self.getElementQueue();\n var k2q = self.getElementKeyToQueue();\n var dequeued = [];\n var lookup = self.lookup;\n for (var i = 0; i < maxDeqSize$1; i++) {\n if (q.size() > 0) {\n var req = q.pop();\n var key = req.key;\n var ele = req.eles[0]; // all eles have the same key\n var cacheExists = lookup.hasCache(ele, req.level);\n\n // clear out the key to req lookup\n k2q[key] = null;\n\n // dequeueing isn't necessary with an existing cache\n if (cacheExists) {\n continue;\n }\n dequeued.push(req);\n var bb = self.getBoundingBox(ele);\n self.getElement(ele, bb, pxRatio, req.level, getTxrReasons.dequeue);\n } else {\n break;\n }\n }\n return dequeued;\n};\nETCp.removeFromQueue = function (ele) {\n var self = this;\n var q = self.getElementQueue();\n var k2q = self.getElementKeyToQueue();\n var key = this.getKey(ele);\n var req = k2q[key];\n if (req != null) {\n if (req.eles.length === 1) {\n // remove if last ele in the req\n // bring to front of queue\n req.reqs = MAX_INT$1;\n q.updateItem(req);\n q.pop(); // remove from queue\n\n k2q[key] = null; // remove from lookup map\n } else {\n // otherwise just remove ele from req\n req.eles.unmerge(ele);\n }\n }\n};\nETCp.onDequeue = function (fn) {\n this.onDequeues.push(fn);\n};\nETCp.offDequeue = function (fn) {\n removeFromArray(this.onDequeues, fn);\n};\nETCp.setupDequeueing = defs.setupDequeueing({\n deqRedrawThreshold: deqRedrawThreshold$1,\n deqCost: deqCost$1,\n deqAvgCost: deqAvgCost$1,\n deqNoDrawCost: deqNoDrawCost$1,\n deqFastCost: deqFastCost$1,\n deq: function deq(self, pxRatio, extent) {\n return self.dequeue(pxRatio, extent);\n },\n onDeqd: function onDeqd(self, deqd) {\n for (var i = 0; i < self.onDequeues.length; i++) {\n var fn = self.onDequeues[i];\n fn(deqd);\n }\n },\n shouldRedraw: function shouldRedraw(self, deqd, pxRatio, extent) {\n for (var i = 0; i < deqd.length; i++) {\n var eles = deqd[i].eles;\n for (var j = 0; j < eles.length; j++) {\n var bb = eles[j].boundingBox();\n if (boundingBoxesIntersect(bb, extent)) {\n return true;\n }\n }\n }\n return false;\n },\n priority: function priority(self) {\n return self.renderer.beforeRenderPriorities.eleTxrDeq;\n }\n});\n\nvar defNumLayers = 1; // default number of layers to use\nvar minLvl = -4; // when scaling smaller than that we don't need to re-render\nvar maxLvl = 2; // when larger than this scale just render directly (caching is not helpful)\nvar maxZoom = 3.99; // beyond this zoom level, layered textures are not used\nvar deqRedrawThreshold = 50; // time to batch redraws together from dequeueing to allow more dequeueing calcs to happen in the meanwhile\nvar refineEleDebounceTime = 50; // time to debounce sharper ele texture updates\nvar deqCost = 0.15; // % of add'l rendering cost allowed for dequeuing ele caches each frame\nvar deqAvgCost = 0.1; // % of add'l rendering cost compared to average overall redraw time\nvar deqNoDrawCost = 0.9; // % of avg frame time that can be used for dequeueing when not drawing\nvar deqFastCost = 0.9; // % of frame time to be used when >60fps\nvar maxDeqSize = 1; // number of eles to dequeue and render at higher texture in each batch\nvar invalidThreshold = 250; // time threshold for disabling b/c of invalidations\nvar maxLayerArea = 4000 * 4000; // layers can't be bigger than this\nvar maxLayerDim = 32767; // maximum size for the width/height of layer canvases\nvar useHighQualityEleTxrReqs = true; // whether to use high quality ele txr requests (generally faster and cheaper in the longterm)\n\n// var log = function(){ console.log.apply( console, arguments ); };\n\nvar LayeredTextureCache = function LayeredTextureCache(renderer) {\n var self = this;\n var r = self.renderer = renderer;\n var cy = r.cy;\n self.layersByLevel = {}; // e.g. 2 => [ layer1, layer2, ..., layerN ]\n\n self.firstGet = true;\n self.lastInvalidationTime = performanceNow() - 2 * invalidThreshold;\n self.skipping = false;\n self.eleTxrDeqs = cy.collection();\n self.scheduleElementRefinement = debounce(function () {\n self.refineElementTextures(self.eleTxrDeqs);\n self.eleTxrDeqs.unmerge(self.eleTxrDeqs);\n }, refineEleDebounceTime);\n r.beforeRender(function (willDraw, now) {\n if (now - self.lastInvalidationTime <= invalidThreshold) {\n self.skipping = true;\n } else {\n self.skipping = false;\n }\n }, r.beforeRenderPriorities.lyrTxrSkip);\n var qSort = function qSort(a, b) {\n return b.reqs - a.reqs;\n };\n self.layersQueue = new Heap(qSort);\n self.setupDequeueing();\n};\nvar LTCp = LayeredTextureCache.prototype;\nvar layerIdPool = 0;\nvar MAX_INT = Math.pow(2, 53) - 1;\nLTCp.makeLayer = function (bb, lvl) {\n var scale = Math.pow(2, lvl);\n var w = Math.ceil(bb.w * scale);\n var h = Math.ceil(bb.h * scale);\n var canvas = this.renderer.makeOffscreenCanvas(w, h);\n var layer = {\n id: layerIdPool = ++layerIdPool % MAX_INT,\n bb: bb,\n level: lvl,\n width: w,\n height: h,\n canvas: canvas,\n context: canvas.getContext('2d'),\n eles: [],\n elesQueue: [],\n reqs: 0\n };\n\n // log('make layer %s with w %s and h %s and lvl %s', layer.id, layer.width, layer.height, layer.level);\n\n var cxt = layer.context;\n var dx = -layer.bb.x1;\n var dy = -layer.bb.y1;\n\n // do the transform on creation to save cycles (it's the same for all eles)\n cxt.scale(scale, scale);\n cxt.translate(dx, dy);\n return layer;\n};\nLTCp.getLayers = function (eles, pxRatio, lvl) {\n var self = this;\n var r = self.renderer;\n var cy = r.cy;\n var zoom = cy.zoom();\n var firstGet = self.firstGet;\n self.firstGet = false;\n\n // log('--\\nget layers with %s eles', eles.length);\n //log eles.map(function(ele){ return ele.id() }) );\n\n if (lvl == null) {\n lvl = Math.ceil(log2(zoom * pxRatio));\n if (lvl < minLvl) {\n lvl = minLvl;\n } else if (zoom >= maxZoom || lvl > maxLvl) {\n return null;\n }\n }\n self.validateLayersElesOrdering(lvl, eles);\n var layersByLvl = self.layersByLevel;\n var scale = Math.pow(2, lvl);\n var layers = layersByLvl[lvl] = layersByLvl[lvl] || [];\n var bb;\n var lvlComplete = self.levelIsComplete(lvl, eles);\n var tmpLayers;\n var checkTempLevels = function checkTempLevels() {\n var canUseAsTmpLvl = function canUseAsTmpLvl(l) {\n self.validateLayersElesOrdering(l, eles);\n if (self.levelIsComplete(l, eles)) {\n tmpLayers = layersByLvl[l];\n return true;\n }\n };\n var checkLvls = function checkLvls(dir) {\n if (tmpLayers) {\n return;\n }\n for (var l = lvl + dir; minLvl <= l && l <= maxLvl; l += dir) {\n if (canUseAsTmpLvl(l)) {\n break;\n }\n }\n };\n checkLvls(1);\n checkLvls(-1);\n\n // remove the invalid layers; they will be replaced as needed later in this function\n for (var i = layers.length - 1; i >= 0; i--) {\n var layer = layers[i];\n if (layer.invalid) {\n removeFromArray(layers, layer);\n }\n }\n };\n if (!lvlComplete) {\n // if the current level is incomplete, then use the closest, best quality layerset temporarily\n // and later queue the current layerset so we can get the proper quality level soon\n\n checkTempLevels();\n } else {\n // log('level complete, using existing layers\\n--');\n return layers;\n }\n var getBb = function getBb() {\n if (!bb) {\n bb = makeBoundingBox();\n for (var i = 0; i < eles.length; i++) {\n updateBoundingBox(bb, eles[i].boundingBox());\n }\n }\n return bb;\n };\n var makeLayer = function makeLayer(opts) {\n opts = opts || {};\n var after = opts.after;\n getBb();\n var w = Math.ceil(bb.w * scale);\n var h = Math.ceil(bb.h * scale);\n if (w > maxLayerDim || h > maxLayerDim) {\n return null;\n }\n var area = w * h;\n if (area > maxLayerArea) {\n return null;\n }\n var layer = self.makeLayer(bb, lvl);\n if (after != null) {\n var index = layers.indexOf(after) + 1;\n layers.splice(index, 0, layer);\n } else if (opts.insert === undefined || opts.insert) {\n // no after specified => first layer made so put at start\n layers.unshift(layer);\n }\n\n // if( tmpLayers ){\n //self.queueLayer( layer );\n // }\n\n return layer;\n };\n if (self.skipping && !firstGet) {\n // log('skip layers');\n return null;\n }\n\n // log('do layers');\n\n var layer = null;\n var maxElesPerLayer = eles.length / defNumLayers;\n var allowLazyQueueing = !firstGet;\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n var rs = ele._private.rscratch;\n var caches = rs.imgLayerCaches = rs.imgLayerCaches || {};\n\n // log('look at ele', ele.id());\n\n var existingLayer = caches[lvl];\n if (existingLayer) {\n // reuse layer for later eles\n // log('reuse layer for', ele.id());\n layer = existingLayer;\n continue;\n }\n if (!layer || layer.eles.length >= maxElesPerLayer || !boundingBoxInBoundingBox(layer.bb, ele.boundingBox())) {\n // log('make new layer for ele %s', ele.id());\n\n layer = makeLayer({\n insert: true,\n after: layer\n });\n\n // if now layer can be built then we can't use layers at this level\n if (!layer) {\n return null;\n }\n\n // log('new layer with id %s', layer.id);\n }\n if (tmpLayers || allowLazyQueueing) {\n // log('queue ele %s in layer %s', ele.id(), layer.id);\n self.queueLayer(layer, ele);\n } else {\n // log('draw ele %s in layer %s', ele.id(), layer.id);\n self.drawEleInLayer(layer, ele, lvl, pxRatio);\n }\n layer.eles.push(ele);\n caches[lvl] = layer;\n }\n\n // log('--');\n\n if (tmpLayers) {\n // then we only queued the current layerset and can't draw it yet\n return tmpLayers;\n }\n if (allowLazyQueueing) {\n // log('lazy queue level', lvl);\n return null;\n }\n return layers;\n};\n\n// a layer may want to use an ele cache of a higher level to avoid blurriness\n// so the layer level might not equal the ele level\nLTCp.getEleLevelForLayerLevel = function (lvl, pxRatio) {\n return lvl;\n};\nLTCp.drawEleInLayer = function (layer, ele, lvl, pxRatio) {\n var self = this;\n var r = this.renderer;\n var context = layer.context;\n var bb = ele.boundingBox();\n if (bb.w === 0 || bb.h === 0 || !ele.visible()) {\n return;\n }\n lvl = self.getEleLevelForLayerLevel(lvl, pxRatio);\n {\n r.setImgSmoothing(context, false);\n }\n {\n r.drawCachedElement(context, ele, null, null, lvl, useHighQualityEleTxrReqs);\n }\n {\n r.setImgSmoothing(context, true);\n }\n};\nLTCp.levelIsComplete = function (lvl, eles) {\n var self = this;\n var layers = self.layersByLevel[lvl];\n if (!layers || layers.length === 0) {\n return false;\n }\n var numElesInLayers = 0;\n for (var i = 0; i < layers.length; i++) {\n var layer = layers[i];\n\n // if there are any eles needed to be drawn yet, the level is not complete\n if (layer.reqs > 0) {\n return false;\n }\n\n // if the layer is invalid, the level is not complete\n if (layer.invalid) {\n return false;\n }\n numElesInLayers += layer.eles.length;\n }\n\n // we should have exactly the number of eles passed in to be complete\n if (numElesInLayers !== eles.length) {\n return false;\n }\n return true;\n};\nLTCp.validateLayersElesOrdering = function (lvl, eles) {\n var layers = this.layersByLevel[lvl];\n if (!layers) {\n return;\n }\n\n // if in a layer the eles are not in the same order, then the layer is invalid\n // (i.e. there is an ele in between the eles in the layer)\n\n for (var i = 0; i < layers.length; i++) {\n var layer = layers[i];\n var offset = -1;\n\n // find the offset\n for (var j = 0; j < eles.length; j++) {\n if (layer.eles[0] === eles[j]) {\n offset = j;\n break;\n }\n }\n if (offset < 0) {\n // then the layer has nonexistent elements and is invalid\n this.invalidateLayer(layer);\n continue;\n }\n\n // the eles in the layer must be in the same continuous order, else the layer is invalid\n\n var o = offset;\n for (var j = 0; j < layer.eles.length; j++) {\n if (layer.eles[j] !== eles[o + j]) {\n // log('invalidate based on ordering', layer.id);\n\n this.invalidateLayer(layer);\n break;\n }\n }\n }\n};\nLTCp.updateElementsInLayers = function (eles, update) {\n var self = this;\n var isEles = element(eles[0]);\n\n // collect udpated elements (cascaded from the layers) and update each\n // layer itself along the way\n for (var i = 0; i < eles.length; i++) {\n var req = isEles ? null : eles[i];\n var ele = isEles ? eles[i] : eles[i].ele;\n var rs = ele._private.rscratch;\n var caches = rs.imgLayerCaches = rs.imgLayerCaches || {};\n for (var l = minLvl; l <= maxLvl; l++) {\n var layer = caches[l];\n if (!layer) {\n continue;\n }\n\n // if update is a request from the ele cache, then it affects only\n // the matching level\n if (req && self.getEleLevelForLayerLevel(layer.level) !== req.level) {\n continue;\n }\n update(layer, ele, req);\n }\n }\n};\nLTCp.haveLayers = function () {\n var self = this;\n var haveLayers = false;\n for (var l = minLvl; l <= maxLvl; l++) {\n var layers = self.layersByLevel[l];\n if (layers && layers.length > 0) {\n haveLayers = true;\n break;\n }\n }\n return haveLayers;\n};\nLTCp.invalidateElements = function (eles) {\n var self = this;\n if (eles.length === 0) {\n return;\n }\n self.lastInvalidationTime = performanceNow();\n\n // log('update invalidate layer time from eles');\n\n if (eles.length === 0 || !self.haveLayers()) {\n return;\n }\n self.updateElementsInLayers(eles, function invalAssocLayers(layer, ele, req) {\n self.invalidateLayer(layer);\n });\n};\nLTCp.invalidateLayer = function (layer) {\n // log('update invalidate layer time');\n\n this.lastInvalidationTime = performanceNow();\n if (layer.invalid) {\n return;\n } // save cycles\n\n var lvl = layer.level;\n var eles = layer.eles;\n var layers = this.layersByLevel[lvl];\n\n // log('invalidate layer', layer.id );\n\n removeFromArray(layers, layer);\n // layer.eles = [];\n\n layer.elesQueue = [];\n layer.invalid = true;\n if (layer.replacement) {\n layer.replacement.invalid = true;\n }\n for (var i = 0; i < eles.length; i++) {\n var caches = eles[i]._private.rscratch.imgLayerCaches;\n if (caches) {\n caches[lvl] = null;\n }\n }\n};\nLTCp.refineElementTextures = function (eles) {\n var self = this;\n\n // log('refine', eles.length);\n\n self.updateElementsInLayers(eles, function refineEachEle(layer, ele, req) {\n var rLyr = layer.replacement;\n if (!rLyr) {\n rLyr = layer.replacement = self.makeLayer(layer.bb, layer.level);\n rLyr.replaces = layer;\n rLyr.eles = layer.eles;\n\n // log('make replacement layer %s for %s with level %s', rLyr.id, layer.id, rLyr.level);\n }\n if (!rLyr.reqs) {\n for (var i = 0; i < rLyr.eles.length; i++) {\n self.queueLayer(rLyr, rLyr.eles[i]);\n }\n\n // log('queue replacement layer refinement', rLyr.id);\n }\n });\n};\nLTCp.enqueueElementRefinement = function (ele) {\n this.eleTxrDeqs.merge(ele);\n this.scheduleElementRefinement();\n};\nLTCp.queueLayer = function (layer, ele) {\n var self = this;\n var q = self.layersQueue;\n var elesQ = layer.elesQueue;\n var hasId = elesQ.hasId = elesQ.hasId || {};\n\n // if a layer is going to be replaced, queuing is a waste of time\n if (layer.replacement) {\n return;\n }\n if (ele) {\n if (hasId[ele.id()]) {\n return;\n }\n elesQ.push(ele);\n hasId[ele.id()] = true;\n }\n if (layer.reqs) {\n layer.reqs++;\n q.updateItem(layer);\n } else {\n layer.reqs = 1;\n q.push(layer);\n }\n};\nLTCp.dequeue = function (pxRatio) {\n var self = this;\n var q = self.layersQueue;\n var deqd = [];\n var eleDeqs = 0;\n while (eleDeqs < maxDeqSize) {\n if (q.size() === 0) {\n break;\n }\n var layer = q.peek();\n\n // if a layer has been or will be replaced, then don't waste time with it\n if (layer.replacement) {\n // log('layer %s in queue skipped b/c it already has a replacement', layer.id);\n q.pop();\n continue;\n }\n\n // if this is a replacement layer that has been superceded, then forget it\n if (layer.replaces && layer !== layer.replaces.replacement) {\n // log('layer is no longer the most uptodate replacement; dequeued', layer.id)\n q.pop();\n continue;\n }\n if (layer.invalid) {\n // log('replacement layer %s is invalid; dequeued', layer.id);\n q.pop();\n continue;\n }\n var ele = layer.elesQueue.shift();\n if (ele) {\n // log('dequeue layer %s', layer.id);\n\n self.drawEleInLayer(layer, ele, layer.level, pxRatio);\n eleDeqs++;\n }\n if (deqd.length === 0) {\n // we need only one entry in deqd to queue redrawing etc\n deqd.push(true);\n }\n\n // if the layer has all its eles done, then remove from the queue\n if (layer.elesQueue.length === 0) {\n q.pop();\n layer.reqs = 0;\n\n // log('dequeue of layer %s complete', layer.id);\n\n // when a replacement layer is dequeued, it replaces the old layer in the level\n if (layer.replaces) {\n self.applyLayerReplacement(layer);\n }\n self.requestRedraw();\n }\n }\n return deqd;\n};\nLTCp.applyLayerReplacement = function (layer) {\n var self = this;\n var layersInLevel = self.layersByLevel[layer.level];\n var replaced = layer.replaces;\n var index = layersInLevel.indexOf(replaced);\n\n // if the replaced layer is not in the active list for the level, then replacing\n // refs would be a mistake (i.e. overwriting the true active layer)\n if (index < 0 || replaced.invalid) {\n // log('replacement layer would have no effect', layer.id);\n return;\n }\n layersInLevel[index] = layer; // replace level ref\n\n // replace refs in eles\n for (var i = 0; i < layer.eles.length; i++) {\n var _p = layer.eles[i]._private;\n var cache = _p.imgLayerCaches = _p.imgLayerCaches || {};\n if (cache) {\n cache[layer.level] = layer;\n }\n }\n\n // log('apply replacement layer %s over %s', layer.id, replaced.id);\n\n self.requestRedraw();\n};\nLTCp.requestRedraw = debounce(function () {\n var r = this.renderer;\n r.redrawHint('eles', true);\n r.redrawHint('drag', true);\n r.redraw();\n}, 100);\nLTCp.setupDequeueing = defs.setupDequeueing({\n deqRedrawThreshold: deqRedrawThreshold,\n deqCost: deqCost,\n deqAvgCost: deqAvgCost,\n deqNoDrawCost: deqNoDrawCost,\n deqFastCost: deqFastCost,\n deq: function deq(self, pxRatio) {\n return self.dequeue(pxRatio);\n },\n onDeqd: noop$1,\n shouldRedraw: trueify,\n priority: function priority(self) {\n return self.renderer.beforeRenderPriorities.lyrTxrDeq;\n }\n});\n\nvar CRp$b = {};\nvar impl;\nfunction polygon(context, points) {\n for (var i = 0; i < points.length; i++) {\n var pt = points[i];\n context.lineTo(pt.x, pt.y);\n }\n}\nfunction triangleBackcurve(context, points, controlPoint) {\n var firstPt;\n for (var i = 0; i < points.length; i++) {\n var pt = points[i];\n if (i === 0) {\n firstPt = pt;\n }\n context.lineTo(pt.x, pt.y);\n }\n context.quadraticCurveTo(controlPoint.x, controlPoint.y, firstPt.x, firstPt.y);\n}\nfunction triangleTee(context, trianglePoints, teePoints) {\n if (context.beginPath) {\n context.beginPath();\n }\n var triPts = trianglePoints;\n for (var i = 0; i < triPts.length; i++) {\n var pt = triPts[i];\n context.lineTo(pt.x, pt.y);\n }\n var teePts = teePoints;\n var firstTeePt = teePoints[0];\n context.moveTo(firstTeePt.x, firstTeePt.y);\n for (var i = 1; i < teePts.length; i++) {\n var pt = teePts[i];\n context.lineTo(pt.x, pt.y);\n }\n if (context.closePath) {\n context.closePath();\n }\n}\nfunction circleTriangle(context, trianglePoints, rx, ry, r) {\n if (context.beginPath) {\n context.beginPath();\n }\n context.arc(rx, ry, r, 0, Math.PI * 2, false);\n var triPts = trianglePoints;\n var firstTrPt = triPts[0];\n context.moveTo(firstTrPt.x, firstTrPt.y);\n for (var i = 0; i < triPts.length; i++) {\n var pt = triPts[i];\n context.lineTo(pt.x, pt.y);\n }\n if (context.closePath) {\n context.closePath();\n }\n}\nfunction circle$1(context, rx, ry, r) {\n context.arc(rx, ry, r, 0, Math.PI * 2, false);\n}\nCRp$b.arrowShapeImpl = function (name) {\n return (impl || (impl = {\n 'polygon': polygon,\n 'triangle-backcurve': triangleBackcurve,\n 'triangle-tee': triangleTee,\n 'circle-triangle': circleTriangle,\n 'triangle-cross': triangleTee,\n 'circle': circle$1\n }))[name];\n};\n\nvar CRp$a = {};\nCRp$a.drawElement = function (context, ele, shiftToOriginWithBb, showLabel, showOverlay, showOpacity) {\n var r = this;\n if (ele.isNode()) {\n r.drawNode(context, ele, shiftToOriginWithBb, showLabel, showOverlay, showOpacity);\n } else {\n r.drawEdge(context, ele, shiftToOriginWithBb, showLabel, showOverlay, showOpacity);\n }\n};\nCRp$a.drawElementOverlay = function (context, ele) {\n var r = this;\n if (ele.isNode()) {\n r.drawNodeOverlay(context, ele);\n } else {\n r.drawEdgeOverlay(context, ele);\n }\n};\nCRp$a.drawElementUnderlay = function (context, ele) {\n var r = this;\n if (ele.isNode()) {\n r.drawNodeUnderlay(context, ele);\n } else {\n r.drawEdgeUnderlay(context, ele);\n }\n};\nCRp$a.drawCachedElementPortion = function (context, ele, eleTxrCache, pxRatio, lvl, reason, getRotation, getOpacity) {\n var r = this;\n var bb = eleTxrCache.getBoundingBox(ele);\n if (bb.w === 0 || bb.h === 0) {\n return;\n } // ignore zero size case\n\n var eleCache = eleTxrCache.getElement(ele, bb, pxRatio, lvl, reason);\n if (eleCache != null) {\n var opacity = getOpacity(r, ele);\n if (opacity === 0) {\n return;\n }\n var theta = getRotation(r, ele);\n var x1 = bb.x1,\n y1 = bb.y1,\n w = bb.w,\n h = bb.h;\n var x, y, sx, sy, smooth;\n if (theta !== 0) {\n var rotPt = eleTxrCache.getRotationPoint(ele);\n sx = rotPt.x;\n sy = rotPt.y;\n context.translate(sx, sy);\n context.rotate(theta);\n smooth = r.getImgSmoothing(context);\n if (!smooth) {\n r.setImgSmoothing(context, true);\n }\n var off = eleTxrCache.getRotationOffset(ele);\n x = off.x;\n y = off.y;\n } else {\n x = x1;\n y = y1;\n }\n var oldGlobalAlpha;\n if (opacity !== 1) {\n oldGlobalAlpha = context.globalAlpha;\n context.globalAlpha = oldGlobalAlpha * opacity;\n }\n context.drawImage(eleCache.texture.canvas, eleCache.x, 0, eleCache.width, eleCache.height, x, y, w, h);\n if (opacity !== 1) {\n context.globalAlpha = oldGlobalAlpha;\n }\n if (theta !== 0) {\n context.rotate(-theta);\n context.translate(-sx, -sy);\n if (!smooth) {\n r.setImgSmoothing(context, false);\n }\n }\n } else {\n eleTxrCache.drawElement(context, ele); // direct draw fallback\n }\n};\nvar getZeroRotation = function getZeroRotation() {\n return 0;\n};\nvar getLabelRotation = function getLabelRotation(r, ele) {\n return r.getTextAngle(ele, null);\n};\nvar getSourceLabelRotation = function getSourceLabelRotation(r, ele) {\n return r.getTextAngle(ele, 'source');\n};\nvar getTargetLabelRotation = function getTargetLabelRotation(r, ele) {\n return r.getTextAngle(ele, 'target');\n};\nvar getOpacity = function getOpacity(r, ele) {\n return ele.effectiveOpacity();\n};\nvar getTextOpacity = function getTextOpacity(e, ele) {\n return ele.pstyle('text-opacity').pfValue * ele.effectiveOpacity();\n};\nCRp$a.drawCachedElement = function (context, ele, pxRatio, extent, lvl, requestHighQuality) {\n var r = this;\n var _r$data = r.data,\n eleTxrCache = _r$data.eleTxrCache,\n lblTxrCache = _r$data.lblTxrCache,\n slbTxrCache = _r$data.slbTxrCache,\n tlbTxrCache = _r$data.tlbTxrCache;\n var bb = ele.boundingBox();\n var reason = requestHighQuality === true ? eleTxrCache.reasons.highQuality : null;\n if (bb.w === 0 || bb.h === 0 || !ele.visible()) {\n return;\n }\n if (!extent || boundingBoxesIntersect(bb, extent)) {\n var isEdge = ele.isEdge();\n var badLine = ele.element()._private.rscratch.badLine;\n r.drawElementUnderlay(context, ele);\n r.drawCachedElementPortion(context, ele, eleTxrCache, pxRatio, lvl, reason, getZeroRotation, getOpacity);\n if (!isEdge || !badLine) {\n r.drawCachedElementPortion(context, ele, lblTxrCache, pxRatio, lvl, reason, getLabelRotation, getTextOpacity);\n }\n if (isEdge && !badLine) {\n r.drawCachedElementPortion(context, ele, slbTxrCache, pxRatio, lvl, reason, getSourceLabelRotation, getTextOpacity);\n r.drawCachedElementPortion(context, ele, tlbTxrCache, pxRatio, lvl, reason, getTargetLabelRotation, getTextOpacity);\n }\n r.drawElementOverlay(context, ele);\n }\n};\nCRp$a.drawElements = function (context, eles) {\n var r = this;\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n r.drawElement(context, ele);\n }\n};\nCRp$a.drawCachedElements = function (context, eles, pxRatio, extent) {\n var r = this;\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n r.drawCachedElement(context, ele, pxRatio, extent);\n }\n};\nCRp$a.drawCachedNodes = function (context, eles, pxRatio, extent) {\n var r = this;\n for (var i = 0; i < eles.length; i++) {\n var ele = eles[i];\n if (!ele.isNode()) {\n continue;\n }\n r.drawCachedElement(context, ele, pxRatio, extent);\n }\n};\nCRp$a.drawLayeredElements = function (context, eles, pxRatio, extent) {\n var r = this;\n var layers = r.data.lyrTxrCache.getLayers(eles, pxRatio);\n if (layers) {\n for (var i = 0; i < layers.length; i++) {\n var layer = layers[i];\n var bb = layer.bb;\n if (bb.w === 0 || bb.h === 0) {\n continue;\n }\n context.drawImage(layer.canvas, bb.x1, bb.y1, bb.w, bb.h);\n }\n } else {\n // fall back on plain caching if no layers\n r.drawCachedElements(context, eles, pxRatio, extent);\n }\n};\n\nvar CRp$9 = {};\nCRp$9.drawEdge = function (context, edge, shiftToOriginWithBb) {\n var drawLabel = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : true;\n var shouldDrawOverlay = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : true;\n var shouldDrawOpacity = arguments.length > 5 && arguments[5] !== undefined ? arguments[5] : true;\n var r = this;\n var rs = edge._private.rscratch;\n if (shouldDrawOpacity && !edge.visible()) {\n return;\n }\n\n // if bezier ctrl pts can not be calculated, then die\n if (rs.badLine || rs.allpts == null || isNaN(rs.allpts[0])) {\n // isNaN in case edge is impossible and browser bugs (e.g. safari)\n return;\n }\n var bb;\n if (shiftToOriginWithBb) {\n bb = shiftToOriginWithBb;\n context.translate(-bb.x1, -bb.y1);\n }\n var opacity = shouldDrawOpacity ? edge.pstyle('opacity').value : 1;\n var lineOpacity = shouldDrawOpacity ? edge.pstyle('line-opacity').value : 1;\n var curveStyle = edge.pstyle('curve-style').value;\n var lineStyle = edge.pstyle('line-style').value;\n var edgeWidth = edge.pstyle('width').pfValue;\n var lineCap = edge.pstyle('line-cap').value;\n var lineOutlineWidth = edge.pstyle('line-outline-width').value;\n var lineOutlineColor = edge.pstyle('line-outline-color').value;\n var effectiveLineOpacity = opacity * lineOpacity;\n // separate arrow opacity would require arrow-opacity property\n var effectiveArrowOpacity = opacity * lineOpacity;\n var drawLine = function drawLine() {\n var strokeOpacity = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : effectiveLineOpacity;\n if (curveStyle === 'straight-triangle') {\n r.eleStrokeStyle(context, edge, strokeOpacity);\n r.drawEdgeTrianglePath(edge, context, rs.allpts);\n } else {\n context.lineWidth = edgeWidth;\n context.lineCap = lineCap;\n r.eleStrokeStyle(context, edge, strokeOpacity);\n r.drawEdgePath(edge, context, rs.allpts, lineStyle);\n context.lineCap = 'butt'; // reset for other drawing functions\n }\n };\n var drawLineOutline = function drawLineOutline() {\n var strokeOpacity = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : effectiveLineOpacity;\n context.lineWidth = edgeWidth + lineOutlineWidth;\n context.lineCap = lineCap;\n if (lineOutlineWidth > 0) {\n r.colorStrokeStyle(context, lineOutlineColor[0], lineOutlineColor[1], lineOutlineColor[2], strokeOpacity);\n } else {\n // do not draw any lineOutline\n context.lineCap = 'butt'; // reset for other drawing functions\n return;\n }\n if (curveStyle === 'straight-triangle') {\n r.drawEdgeTrianglePath(edge, context, rs.allpts);\n } else {\n r.drawEdgePath(edge, context, rs.allpts, lineStyle);\n context.lineCap = 'butt'; // reset for other drawing functions\n }\n };\n var drawOverlay = function drawOverlay() {\n if (!shouldDrawOverlay) {\n return;\n }\n r.drawEdgeOverlay(context, edge);\n };\n var drawUnderlay = function drawUnderlay() {\n if (!shouldDrawOverlay) {\n return;\n }\n r.drawEdgeUnderlay(context, edge);\n };\n var drawArrows = function drawArrows() {\n var arrowOpacity = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : effectiveArrowOpacity;\n r.drawArrowheads(context, edge, arrowOpacity);\n };\n var drawText = function drawText() {\n r.drawElementText(context, edge, null, drawLabel);\n };\n context.lineJoin = 'round';\n var ghost = edge.pstyle('ghost').value === 'yes';\n if (ghost) {\n var gx = edge.pstyle('ghost-offset-x').pfValue;\n var gy = edge.pstyle('ghost-offset-y').pfValue;\n var ghostOpacity = edge.pstyle('ghost-opacity').value;\n var effectiveGhostOpacity = effectiveLineOpacity * ghostOpacity;\n context.translate(gx, gy);\n drawLine(effectiveGhostOpacity);\n drawArrows(effectiveGhostOpacity);\n context.translate(-gx, -gy);\n } else {\n drawLineOutline();\n }\n drawUnderlay();\n drawLine();\n drawArrows();\n drawOverlay();\n drawText();\n if (shiftToOriginWithBb) {\n context.translate(bb.x1, bb.y1);\n }\n};\nvar drawEdgeOverlayUnderlay = function drawEdgeOverlayUnderlay(overlayOrUnderlay) {\n if (!['overlay', 'underlay'].includes(overlayOrUnderlay)) {\n throw new Error('Invalid state');\n }\n return function (context, edge) {\n if (!edge.visible()) {\n return;\n }\n var opacity = edge.pstyle(\"\".concat(overlayOrUnderlay, \"-opacity\")).value;\n if (opacity === 0) {\n return;\n }\n var r = this;\n var usePaths = r.usePaths();\n var rs = edge._private.rscratch;\n var padding = edge.pstyle(\"\".concat(overlayOrUnderlay, \"-padding\")).pfValue;\n var width = 2 * padding;\n var color = edge.pstyle(\"\".concat(overlayOrUnderlay, \"-color\")).value;\n context.lineWidth = width;\n if (rs.edgeType === 'self' && !usePaths) {\n context.lineCap = 'butt';\n } else {\n context.lineCap = 'round';\n }\n r.colorStrokeStyle(context, color[0], color[1], color[2], opacity);\n r.drawEdgePath(edge, context, rs.allpts, 'solid');\n };\n};\nCRp$9.drawEdgeOverlay = drawEdgeOverlayUnderlay('overlay');\nCRp$9.drawEdgeUnderlay = drawEdgeOverlayUnderlay('underlay');\nCRp$9.drawEdgePath = function (edge, context, pts, type) {\n var rs = edge._private.rscratch;\n var canvasCxt = context;\n var path;\n var pathCacheHit = false;\n var usePaths = this.usePaths();\n var lineDashPattern = edge.pstyle('line-dash-pattern').pfValue;\n var lineDashOffset = edge.pstyle('line-dash-offset').pfValue;\n if (usePaths) {\n var pathCacheKey = pts.join('$');\n var keyMatches = rs.pathCacheKey && rs.pathCacheKey === pathCacheKey;\n if (keyMatches) {\n path = context = rs.pathCache;\n pathCacheHit = true;\n } else {\n path = context = new Path2D();\n rs.pathCacheKey = pathCacheKey;\n rs.pathCache = path;\n }\n }\n if (canvasCxt.setLineDash) {\n // for very outofdate browsers\n switch (type) {\n case 'dotted':\n canvasCxt.setLineDash([1, 1]);\n break;\n case 'dashed':\n canvasCxt.setLineDash(lineDashPattern);\n canvasCxt.lineDashOffset = lineDashOffset;\n break;\n case 'solid':\n canvasCxt.setLineDash([]);\n break;\n }\n }\n if (!pathCacheHit && !rs.badLine) {\n if (context.beginPath) {\n context.beginPath();\n }\n context.moveTo(pts[0], pts[1]);\n switch (rs.edgeType) {\n case 'bezier':\n case 'self':\n case 'compound':\n case 'multibezier':\n for (var i = 2; i + 3 < pts.length; i += 4) {\n context.quadraticCurveTo(pts[i], pts[i + 1], pts[i + 2], pts[i + 3]);\n }\n break;\n case 'straight':\n case 'haystack':\n for (var _i = 2; _i + 1 < pts.length; _i += 2) {\n context.lineTo(pts[_i], pts[_i + 1]);\n }\n break;\n case 'segments':\n if (rs.isRound) {\n var _iterator = _createForOfIteratorHelper(rs.roundCorners),\n _step;\n try {\n for (_iterator.s(); !(_step = _iterator.n()).done;) {\n var corner = _step.value;\n drawPreparedRoundCorner(context, corner);\n }\n } catch (err) {\n _iterator.e(err);\n } finally {\n _iterator.f();\n }\n context.lineTo(pts[pts.length - 2], pts[pts.length - 1]);\n } else {\n for (var _i2 = 2; _i2 + 1 < pts.length; _i2 += 2) {\n context.lineTo(pts[_i2], pts[_i2 + 1]);\n }\n }\n break;\n }\n }\n context = canvasCxt;\n if (usePaths) {\n context.stroke(path);\n } else {\n context.stroke();\n }\n\n // reset any line dashes\n if (context.setLineDash) {\n // for very outofdate browsers\n context.setLineDash([]);\n }\n};\nCRp$9.drawEdgeTrianglePath = function (edge, context, pts) {\n // use line stroke style for triangle fill style\n context.fillStyle = context.strokeStyle;\n var edgeWidth = edge.pstyle('width').pfValue;\n for (var i = 0; i + 1 < pts.length; i += 2) {\n var vector = [pts[i + 2] - pts[i], pts[i + 3] - pts[i + 1]];\n var length = Math.sqrt(vector[0] * vector[0] + vector[1] * vector[1]);\n var normal = [vector[1] / length, -vector[0] / length];\n var triangleHead = [normal[0] * edgeWidth / 2, normal[1] * edgeWidth / 2];\n context.beginPath();\n context.moveTo(pts[i] - triangleHead[0], pts[i + 1] - triangleHead[1]);\n context.lineTo(pts[i] + triangleHead[0], pts[i + 1] + triangleHead[1]);\n context.lineTo(pts[i + 2], pts[i + 3]);\n context.closePath();\n context.fill();\n }\n};\nCRp$9.drawArrowheads = function (context, edge, opacity) {\n var rs = edge._private.rscratch;\n var isHaystack = rs.edgeType === 'haystack';\n if (!isHaystack) {\n this.drawArrowhead(context, edge, 'source', rs.arrowStartX, rs.arrowStartY, rs.srcArrowAngle, opacity);\n }\n this.drawArrowhead(context, edge, 'mid-target', rs.midX, rs.midY, rs.midtgtArrowAngle, opacity);\n this.drawArrowhead(context, edge, 'mid-source', rs.midX, rs.midY, rs.midsrcArrowAngle, opacity);\n if (!isHaystack) {\n this.drawArrowhead(context, edge, 'target', rs.arrowEndX, rs.arrowEndY, rs.tgtArrowAngle, opacity);\n }\n};\nCRp$9.drawArrowhead = function (context, edge, prefix, x, y, angle, opacity) {\n if (isNaN(x) || x == null || isNaN(y) || y == null || isNaN(angle) || angle == null) {\n return;\n }\n var self = this;\n var arrowShape = edge.pstyle(prefix + '-arrow-shape').value;\n if (arrowShape === 'none') {\n return;\n }\n var arrowClearFill = edge.pstyle(prefix + '-arrow-fill').value === 'hollow' ? 'both' : 'filled';\n var arrowFill = edge.pstyle(prefix + '-arrow-fill').value;\n var edgeWidth = edge.pstyle('width').pfValue;\n var pArrowWidth = edge.pstyle(prefix + '-arrow-width');\n var arrowWidth = pArrowWidth.value === 'match-line' ? edgeWidth : pArrowWidth.pfValue;\n if (pArrowWidth.units === '%') arrowWidth *= edgeWidth;\n var edgeOpacity = edge.pstyle('opacity').value;\n if (opacity === undefined) {\n opacity = edgeOpacity;\n }\n var gco = context.globalCompositeOperation;\n if (opacity !== 1 || arrowFill === 'hollow') {\n // then extra clear is needed\n context.globalCompositeOperation = 'destination-out';\n self.colorFillStyle(context, 255, 255, 255, 1);\n self.colorStrokeStyle(context, 255, 255, 255, 1);\n self.drawArrowShape(edge, context, arrowClearFill, edgeWidth, arrowShape, arrowWidth, x, y, angle);\n context.globalCompositeOperation = gco;\n } // otherwise, the opaque arrow clears it for free :)\n\n var color = edge.pstyle(prefix + '-arrow-color').value;\n self.colorFillStyle(context, color[0], color[1], color[2], opacity);\n self.colorStrokeStyle(context, color[0], color[1], color[2], opacity);\n self.drawArrowShape(edge, context, arrowFill, edgeWidth, arrowShape, arrowWidth, x, y, angle);\n};\nCRp$9.drawArrowShape = function (edge, context, fill, edgeWidth, shape, shapeWidth, x, y, angle) {\n var r = this;\n var usePaths = this.usePaths() && shape !== 'triangle-cross';\n var pathCacheHit = false;\n var path;\n var canvasContext = context;\n var translation = {\n x: x,\n y: y\n };\n var scale = edge.pstyle('arrow-scale').value;\n var size = this.getArrowWidth(edgeWidth, scale);\n var shapeImpl = r.arrowShapes[shape];\n if (usePaths) {\n var cache = r.arrowPathCache = r.arrowPathCache || [];\n var key = hashString(shape);\n var cachedPath = cache[key];\n if (cachedPath != null) {\n path = context = cachedPath;\n pathCacheHit = true;\n } else {\n path = context = new Path2D();\n cache[key] = path;\n }\n }\n if (!pathCacheHit) {\n if (context.beginPath) {\n context.beginPath();\n }\n if (usePaths) {\n // store in the path cache with values easily manipulated later\n shapeImpl.draw(context, 1, 0, {\n x: 0,\n y: 0\n }, 1);\n } else {\n shapeImpl.draw(context, size, angle, translation, edgeWidth);\n }\n if (context.closePath) {\n context.closePath();\n }\n }\n context = canvasContext;\n if (usePaths) {\n // set transform to arrow position/orientation\n context.translate(x, y);\n context.rotate(angle);\n context.scale(size, size);\n }\n if (fill === 'filled' || fill === 'both') {\n if (usePaths) {\n context.fill(path);\n } else {\n context.fill();\n }\n }\n if (fill === 'hollow' || fill === 'both') {\n context.lineWidth = shapeWidth / (usePaths ? size : 1);\n context.lineJoin = 'miter';\n if (usePaths) {\n context.stroke(path);\n } else {\n context.stroke();\n }\n }\n if (usePaths) {\n // reset transform by applying inverse\n context.scale(1 / size, 1 / size);\n context.rotate(-angle);\n context.translate(-x, -y);\n }\n};\n\nvar CRp$8 = {};\nCRp$8.safeDrawImage = function (context, img, ix, iy, iw, ih, x, y, w, h) {\n // detect problematic cases for old browsers with bad images (cheaper than try-catch)\n if (iw <= 0 || ih <= 0 || w <= 0 || h <= 0) {\n return;\n }\n try {\n context.drawImage(img, ix, iy, iw, ih, x, y, w, h);\n } catch (e) {\n warn(e);\n }\n};\nCRp$8.drawInscribedImage = function (context, img, node, index, nodeOpacity) {\n var r = this;\n var pos = node.position();\n var nodeX = pos.x;\n var nodeY = pos.y;\n var styleObj = node.cy().style();\n var getIndexedStyle = styleObj.getIndexedStyle.bind(styleObj);\n var fit = getIndexedStyle(node, 'background-fit', 'value', index);\n var repeat = getIndexedStyle(node, 'background-repeat', 'value', index);\n var nodeW = node.width();\n var nodeH = node.height();\n var paddingX2 = node.padding() * 2;\n var nodeTW = nodeW + (getIndexedStyle(node, 'background-width-relative-to', 'value', index) === 'inner' ? 0 : paddingX2);\n var nodeTH = nodeH + (getIndexedStyle(node, 'background-height-relative-to', 'value', index) === 'inner' ? 0 : paddingX2);\n var rs = node._private.rscratch;\n var clip = getIndexedStyle(node, 'background-clip', 'value', index);\n var shouldClip = clip === 'node';\n var imgOpacity = getIndexedStyle(node, 'background-image-opacity', 'value', index) * nodeOpacity;\n var smooth = getIndexedStyle(node, 'background-image-smoothing', 'value', index);\n var cornerRadius = node.pstyle('corner-radius').value;\n if (cornerRadius !== 'auto') cornerRadius = node.pstyle('corner-radius').pfValue;\n var imgW = img.width || img.cachedW;\n var imgH = img.height || img.cachedH;\n\n // workaround for broken browsers like ie\n if (null == imgW || null == imgH) {\n document.body.appendChild(img); // eslint-disable-line no-undef\n\n imgW = img.cachedW = img.width || img.offsetWidth;\n imgH = img.cachedH = img.height || img.offsetHeight;\n document.body.removeChild(img); // eslint-disable-line no-undef\n }\n var w = imgW;\n var h = imgH;\n if (getIndexedStyle(node, 'background-width', 'value', index) !== 'auto') {\n if (getIndexedStyle(node, 'background-width', 'units', index) === '%') {\n w = getIndexedStyle(node, 'background-width', 'pfValue', index) * nodeTW;\n } else {\n w = getIndexedStyle(node, 'background-width', 'pfValue', index);\n }\n }\n if (getIndexedStyle(node, 'background-height', 'value', index) !== 'auto') {\n if (getIndexedStyle(node, 'background-height', 'units', index) === '%') {\n h = getIndexedStyle(node, 'background-height', 'pfValue', index) * nodeTH;\n } else {\n h = getIndexedStyle(node, 'background-height', 'pfValue', index);\n }\n }\n if (w === 0 || h === 0) {\n return; // no point in drawing empty image (and chrome is broken in this case)\n }\n if (fit === 'contain') {\n var scale = Math.min(nodeTW / w, nodeTH / h);\n w *= scale;\n h *= scale;\n } else if (fit === 'cover') {\n var scale = Math.max(nodeTW / w, nodeTH / h);\n w *= scale;\n h *= scale;\n }\n var x = nodeX - nodeTW / 2; // left\n var posXUnits = getIndexedStyle(node, 'background-position-x', 'units', index);\n var posXPfVal = getIndexedStyle(node, 'background-position-x', 'pfValue', index);\n if (posXUnits === '%') {\n x += (nodeTW - w) * posXPfVal;\n } else {\n x += posXPfVal;\n }\n var offXUnits = getIndexedStyle(node, 'background-offset-x', 'units', index);\n var offXPfVal = getIndexedStyle(node, 'background-offset-x', 'pfValue', index);\n if (offXUnits === '%') {\n x += (nodeTW - w) * offXPfVal;\n } else {\n x += offXPfVal;\n }\n var y = nodeY - nodeTH / 2; // top\n var posYUnits = getIndexedStyle(node, 'background-position-y', 'units', index);\n var posYPfVal = getIndexedStyle(node, 'background-position-y', 'pfValue', index);\n if (posYUnits === '%') {\n y += (nodeTH - h) * posYPfVal;\n } else {\n y += posYPfVal;\n }\n var offYUnits = getIndexedStyle(node, 'background-offset-y', 'units', index);\n var offYPfVal = getIndexedStyle(node, 'background-offset-y', 'pfValue', index);\n if (offYUnits === '%') {\n y += (nodeTH - h) * offYPfVal;\n } else {\n y += offYPfVal;\n }\n if (rs.pathCache) {\n x -= nodeX;\n y -= nodeY;\n nodeX = 0;\n nodeY = 0;\n }\n var gAlpha = context.globalAlpha;\n context.globalAlpha = imgOpacity;\n var smoothingEnabled = r.getImgSmoothing(context);\n var isSmoothingSwitched = false;\n if (smooth === 'no' && smoothingEnabled) {\n r.setImgSmoothing(context, false);\n isSmoothingSwitched = true;\n } else if (smooth === 'yes' && !smoothingEnabled) {\n r.setImgSmoothing(context, true);\n isSmoothingSwitched = true;\n }\n if (repeat === 'no-repeat') {\n if (shouldClip) {\n context.save();\n if (rs.pathCache) {\n context.clip(rs.pathCache);\n } else {\n r.nodeShapes[r.getNodeShape(node)].draw(context, nodeX, nodeY, nodeTW, nodeTH, cornerRadius, rs);\n context.clip();\n }\n }\n r.safeDrawImage(context, img, 0, 0, imgW, imgH, x, y, w, h);\n if (shouldClip) {\n context.restore();\n }\n } else {\n var pattern = context.createPattern(img, repeat);\n context.fillStyle = pattern;\n r.nodeShapes[r.getNodeShape(node)].draw(context, nodeX, nodeY, nodeTW, nodeTH, cornerRadius, rs);\n context.translate(x, y);\n context.fill();\n context.translate(-x, -y);\n }\n context.globalAlpha = gAlpha;\n if (isSmoothingSwitched) {\n r.setImgSmoothing(context, smoothingEnabled);\n }\n};\n\nvar CRp$7 = {};\nCRp$7.eleTextBiggerThanMin = function (ele, scale) {\n if (!scale) {\n var zoom = ele.cy().zoom();\n var pxRatio = this.getPixelRatio();\n var lvl = Math.ceil(log2(zoom * pxRatio)); // the effective texture level\n\n scale = Math.pow(2, lvl);\n }\n var computedSize = ele.pstyle('font-size').pfValue * scale;\n var minSize = ele.pstyle('min-zoomed-font-size').pfValue;\n if (computedSize < minSize) {\n return false;\n }\n return true;\n};\nCRp$7.drawElementText = function (context, ele, shiftToOriginWithBb, force, prefix) {\n var useEleOpacity = arguments.length > 5 && arguments[5] !== undefined ? arguments[5] : true;\n var r = this;\n if (force == null) {\n if (useEleOpacity && !r.eleTextBiggerThanMin(ele)) {\n return;\n }\n } else if (force === false) {\n return;\n }\n if (ele.isNode()) {\n var label = ele.pstyle('label');\n if (!label || !label.value) {\n return;\n }\n var justification = r.getLabelJustification(ele);\n context.textAlign = justification;\n context.textBaseline = 'bottom';\n } else {\n var badLine = ele.element()._private.rscratch.badLine;\n var _label = ele.pstyle('label');\n var srcLabel = ele.pstyle('source-label');\n var tgtLabel = ele.pstyle('target-label');\n if (badLine || (!_label || !_label.value) && (!srcLabel || !srcLabel.value) && (!tgtLabel || !tgtLabel.value)) {\n return;\n }\n context.textAlign = 'center';\n context.textBaseline = 'bottom';\n }\n var applyRotation = !shiftToOriginWithBb;\n var bb;\n if (shiftToOriginWithBb) {\n bb = shiftToOriginWithBb;\n context.translate(-bb.x1, -bb.y1);\n }\n if (prefix == null) {\n r.drawText(context, ele, null, applyRotation, useEleOpacity);\n if (ele.isEdge()) {\n r.drawText(context, ele, 'source', applyRotation, useEleOpacity);\n r.drawText(context, ele, 'target', applyRotation, useEleOpacity);\n }\n } else {\n r.drawText(context, ele, prefix, applyRotation, useEleOpacity);\n }\n if (shiftToOriginWithBb) {\n context.translate(bb.x1, bb.y1);\n }\n};\nCRp$7.getFontCache = function (context) {\n var cache;\n this.fontCaches = this.fontCaches || [];\n for (var i = 0; i < this.fontCaches.length; i++) {\n cache = this.fontCaches[i];\n if (cache.context === context) {\n return cache;\n }\n }\n cache = {\n context: context\n };\n this.fontCaches.push(cache);\n return cache;\n};\n\n// set up canvas context with font\n// returns transformed text string\nCRp$7.setupTextStyle = function (context, ele) {\n var useEleOpacity = arguments.length > 2 && arguments[2] !== undefined ? arguments[2] : true;\n // Font style\n var labelStyle = ele.pstyle('font-style').strValue;\n var labelSize = ele.pstyle('font-size').pfValue + 'px';\n var labelFamily = ele.pstyle('font-family').strValue;\n var labelWeight = ele.pstyle('font-weight').strValue;\n var opacity = useEleOpacity ? ele.effectiveOpacity() * ele.pstyle('text-opacity').value : 1;\n var outlineOpacity = ele.pstyle('text-outline-opacity').value * opacity;\n var color = ele.pstyle('color').value;\n var outlineColor = ele.pstyle('text-outline-color').value;\n context.font = labelStyle + ' ' + labelWeight + ' ' + labelSize + ' ' + labelFamily;\n context.lineJoin = 'round'; // so text outlines aren't jagged\n\n this.colorFillStyle(context, color[0], color[1], color[2], opacity);\n this.colorStrokeStyle(context, outlineColor[0], outlineColor[1], outlineColor[2], outlineOpacity);\n};\nfunction circle(ctx, x, y, width, height) {\n var diameter = Math.min(width, height);\n var radius = diameter / 2;\n var centerX = x + width / 2;\n var centerY = y + height / 2;\n ctx.beginPath();\n ctx.arc(centerX, centerY, radius, 0, Math.PI * 2);\n ctx.closePath();\n}\nfunction roundRect(ctx, x, y, width, height) {\n var radius = arguments.length > 5 && arguments[5] !== undefined ? arguments[5] : 5;\n var r = Math.min(radius, width / 2, height / 2); // prevent overflow\n ctx.beginPath();\n ctx.moveTo(x + r, y);\n ctx.lineTo(x + width - r, y);\n ctx.quadraticCurveTo(x + width, y, x + width, y + r);\n ctx.lineTo(x + width, y + height - r);\n ctx.quadraticCurveTo(x + width, y + height, x + width - r, y + height);\n ctx.lineTo(x + r, y + height);\n ctx.quadraticCurveTo(x, y + height, x, y + height - r);\n ctx.lineTo(x, y + r);\n ctx.quadraticCurveTo(x, y, x + r, y);\n ctx.closePath();\n}\nCRp$7.getTextAngle = function (ele, prefix) {\n var theta;\n var _p = ele._private;\n var rscratch = _p.rscratch;\n var pdash = prefix ? prefix + '-' : '';\n var rotation = ele.pstyle(pdash + 'text-rotation');\n if (rotation.strValue === 'autorotate') {\n var textAngle = getPrefixedProperty(rscratch, 'labelAngle', prefix);\n theta = ele.isEdge() ? textAngle : 0;\n } else if (rotation.strValue === 'none') {\n theta = 0;\n } else {\n theta = rotation.pfValue;\n }\n return theta;\n};\nCRp$7.drawText = function (context, ele, prefix) {\n var applyRotation = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : true;\n var useEleOpacity = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : true;\n var _p = ele._private;\n var rscratch = _p.rscratch;\n var parentOpacity = useEleOpacity ? ele.effectiveOpacity() : 1;\n if (useEleOpacity && (parentOpacity === 0 || ele.pstyle('text-opacity').value === 0)) {\n return;\n }\n\n // use 'main' as an alias for the main label (i.e. null prefix)\n if (prefix === 'main') {\n prefix = null;\n }\n var textX = getPrefixedProperty(rscratch, 'labelX', prefix);\n var textY = getPrefixedProperty(rscratch, 'labelY', prefix);\n var orgTextX, orgTextY; // used for rotation\n var text = this.getLabelText(ele, prefix);\n if (text != null && text !== '' && !isNaN(textX) && !isNaN(textY)) {\n this.setupTextStyle(context, ele, useEleOpacity);\n var pdash = prefix ? prefix + '-' : '';\n var textW = getPrefixedProperty(rscratch, 'labelWidth', prefix);\n var textH = getPrefixedProperty(rscratch, 'labelHeight', prefix);\n var marginX = ele.pstyle(pdash + 'text-margin-x').pfValue;\n var marginY = ele.pstyle(pdash + 'text-margin-y').pfValue;\n var isEdge = ele.isEdge();\n var halign = ele.pstyle('text-halign').value;\n var valign = ele.pstyle('text-valign').value;\n if (isEdge) {\n halign = 'center';\n valign = 'center';\n }\n textX += marginX;\n textY += marginY;\n var theta;\n if (!applyRotation) {\n theta = 0;\n } else {\n theta = this.getTextAngle(ele, prefix);\n }\n if (theta !== 0) {\n orgTextX = textX;\n orgTextY = textY;\n context.translate(orgTextX, orgTextY);\n context.rotate(theta);\n textX = 0;\n textY = 0;\n }\n switch (valign) {\n case 'top':\n break;\n case 'center':\n textY += textH / 2;\n break;\n case 'bottom':\n textY += textH;\n break;\n }\n var backgroundOpacity = ele.pstyle('text-background-opacity').value;\n var borderOpacity = ele.pstyle('text-border-opacity').value;\n var textBorderWidth = ele.pstyle('text-border-width').pfValue;\n var backgroundPadding = ele.pstyle('text-background-padding').pfValue;\n var styleShape = ele.pstyle('text-background-shape').strValue;\n var rounded = styleShape === 'round-rectangle' || styleShape === 'roundrectangle';\n var circled = styleShape === 'circle';\n var roundRadius = 2;\n if (backgroundOpacity > 0 || textBorderWidth > 0 && borderOpacity > 0) {\n var textFill = context.fillStyle;\n var textStroke = context.strokeStyle;\n var textLineWidth = context.lineWidth;\n var textBackgroundColor = ele.pstyle('text-background-color').value;\n var textBorderColor = ele.pstyle('text-border-color').value;\n var textBorderStyle = ele.pstyle('text-border-style').value;\n var doFill = backgroundOpacity > 0;\n var doStroke = textBorderWidth > 0 && borderOpacity > 0;\n var bgX = textX - backgroundPadding;\n switch (halign) {\n case 'left':\n bgX -= textW;\n break;\n case 'center':\n bgX -= textW / 2;\n break;\n }\n var bgY = textY - textH - backgroundPadding;\n var bgW = textW + 2 * backgroundPadding;\n var bgH = textH + 2 * backgroundPadding;\n if (doFill) {\n context.fillStyle = \"rgba(\".concat(textBackgroundColor[0], \",\").concat(textBackgroundColor[1], \",\").concat(textBackgroundColor[2], \",\").concat(backgroundOpacity * parentOpacity, \")\");\n }\n if (doStroke) {\n context.strokeStyle = \"rgba(\".concat(textBorderColor[0], \",\").concat(textBorderColor[1], \",\").concat(textBorderColor[2], \",\").concat(borderOpacity * parentOpacity, \")\");\n context.lineWidth = textBorderWidth;\n if (context.setLineDash) {\n switch (textBorderStyle) {\n case 'dotted':\n context.setLineDash([1, 1]);\n break;\n case 'dashed':\n context.setLineDash([4, 2]);\n break;\n case 'double':\n context.lineWidth = textBorderWidth / 4;\n context.setLineDash([]);\n break;\n case 'solid':\n default:\n context.setLineDash([]);\n break;\n }\n }\n }\n if (rounded) {\n context.beginPath();\n roundRect(context, bgX, bgY, bgW, bgH, roundRadius);\n } else if (circled) {\n context.beginPath();\n circle(context, bgX, bgY, bgW, bgH);\n } else {\n context.beginPath();\n context.rect(bgX, bgY, bgW, bgH);\n }\n if (doFill) context.fill();\n if (doStroke) context.stroke();\n\n // Double border pass for 'double' style\n if (doStroke && textBorderStyle === 'double') {\n var whiteWidth = textBorderWidth / 2;\n context.beginPath();\n if (rounded) {\n roundRect(context, bgX + whiteWidth, bgY + whiteWidth, bgW - 2 * whiteWidth, bgH - 2 * whiteWidth, roundRadius);\n } else {\n context.rect(bgX + whiteWidth, bgY + whiteWidth, bgW - 2 * whiteWidth, bgH - 2 * whiteWidth);\n }\n context.stroke();\n }\n context.fillStyle = textFill;\n context.strokeStyle = textStroke;\n context.lineWidth = textLineWidth;\n if (context.setLineDash) context.setLineDash([]);\n }\n var lineWidth = 2 * ele.pstyle('text-outline-width').pfValue; // *2 b/c the stroke is drawn centred on the middle\n\n if (lineWidth > 0) {\n context.lineWidth = lineWidth;\n }\n if (ele.pstyle('text-wrap').value === 'wrap') {\n var lines = getPrefixedProperty(rscratch, 'labelWrapCachedLines', prefix);\n var lineHeight = getPrefixedProperty(rscratch, 'labelLineHeight', prefix);\n var halfTextW = textW / 2;\n var justification = this.getLabelJustification(ele);\n if (justification === 'auto') ; else if (halign === 'left') {\n // auto justification : right\n if (justification === 'left') {\n textX += -textW;\n } else if (justification === 'center') {\n textX += -halfTextW;\n } // else same as auto\n } else if (halign === 'center') {\n // auto justfication : center\n if (justification === 'left') {\n textX += -halfTextW;\n } else if (justification === 'right') {\n textX += halfTextW;\n } // else same as auto\n } else if (halign === 'right') {\n // auto justification : left\n if (justification === 'center') {\n textX += halfTextW;\n } else if (justification === 'right') {\n textX += textW;\n } // else same as auto\n }\n switch (valign) {\n case 'top':\n textY -= (lines.length - 1) * lineHeight;\n break;\n case 'center':\n case 'bottom':\n textY -= (lines.length - 1) * lineHeight;\n break;\n }\n for (var l = 0; l < lines.length; l++) {\n if (lineWidth > 0) {\n context.strokeText(lines[l], textX, textY);\n }\n context.fillText(lines[l], textX, textY);\n textY += lineHeight;\n }\n } else {\n if (lineWidth > 0) {\n context.strokeText(text, textX, textY);\n }\n context.fillText(text, textX, textY);\n }\n if (theta !== 0) {\n context.rotate(-theta);\n context.translate(-orgTextX, -orgTextY);\n }\n }\n};\n\n/* global Path2D */\n\nvar CRp$6 = {};\nCRp$6.drawNode = function (context, node, shiftToOriginWithBb) {\n var drawLabel = arguments.length > 3 && arguments[3] !== undefined ? arguments[3] : true;\n var shouldDrawOverlay = arguments.length > 4 && arguments[4] !== undefined ? arguments[4] : true;\n var shouldDrawOpacity = arguments.length > 5 && arguments[5] !== undefined ? arguments[5] : true;\n var r = this;\n var nodeWidth, nodeHeight;\n var _p = node._private;\n var rs = _p.rscratch;\n var pos = node.position();\n if (!number$1(pos.x) || !number$1(pos.y)) {\n return; // can't draw node with undefined position\n }\n if (shouldDrawOpacity && !node.visible()) {\n return;\n }\n var eleOpacity = shouldDrawOpacity ? node.effectiveOpacity() : 1;\n var usePaths = r.usePaths();\n var path;\n var pathCacheHit = false;\n var padding = node.padding();\n nodeWidth = node.width() + 2 * padding;\n nodeHeight = node.height() + 2 * padding;\n\n //\n // setup shift\n\n var bb;\n if (shiftToOriginWithBb) {\n bb = shiftToOriginWithBb;\n context.translate(-bb.x1, -bb.y1);\n }\n\n //\n // load bg image\n\n var bgImgProp = node.pstyle('background-image');\n var urls = bgImgProp.value;\n var urlDefined = new Array(urls.length);\n var image = new Array(urls.length);\n var numImages = 0;\n for (var i = 0; i < urls.length; i++) {\n var url = urls[i];\n var defd = urlDefined[i] = url != null && url !== 'none';\n if (defd) {\n var bgImgCrossOrigin = node.cy().style().getIndexedStyle(node, 'background-image-crossorigin', 'value', i);\n numImages++;\n\n // get image, and if not loaded then ask to redraw when later loaded\n image[i] = r.getCachedImage(url, bgImgCrossOrigin, function () {\n _p.backgroundTimestamp = Date.now();\n node.emitAndNotify('background');\n });\n }\n }\n\n //\n // setup styles\n\n var darkness = node.pstyle('background-blacken').value;\n var borderWidth = node.pstyle('border-width').pfValue;\n var bgOpacity = node.pstyle('background-opacity').value * eleOpacity;\n var borderColor = node.pstyle('border-color').value;\n var borderStyle = node.pstyle('border-style').value;\n var borderJoin = node.pstyle('border-join').value;\n var borderCap = node.pstyle('border-cap').value;\n var borderPosition = node.pstyle('border-position').value;\n var borderPattern = node.pstyle('border-dash-pattern').pfValue;\n var borderOffset = node.pstyle('border-dash-offset').pfValue;\n var borderOpacity = node.pstyle('border-opacity').value * eleOpacity;\n var outlineWidth = node.pstyle('outline-width').pfValue;\n var outlineColor = node.pstyle('outline-color').value;\n var outlineStyle = node.pstyle('outline-style').value;\n var outlineOpacity = node.pstyle('outline-opacity').value * eleOpacity;\n var outlineOffset = node.pstyle('outline-offset').value;\n var cornerRadius = node.pstyle('corner-radius').value;\n if (cornerRadius !== 'auto') cornerRadius = node.pstyle('corner-radius').pfValue;\n var setupShapeColor = function setupShapeColor() {\n var bgOpy = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : bgOpacity;\n r.eleFillStyle(context, node, bgOpy);\n };\n var setupBorderColor = function setupBorderColor() {\n var bdrOpy = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : borderOpacity;\n r.colorStrokeStyle(context, borderColor[0], borderColor[1], borderColor[2], bdrOpy);\n };\n var setupOutlineColor = function setupOutlineColor() {\n var otlnOpy = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : outlineOpacity;\n r.colorStrokeStyle(context, outlineColor[0], outlineColor[1], outlineColor[2], otlnOpy);\n };\n\n //\n // setup shape\n\n var getPath = function getPath(width, height, shape, points) {\n var pathCache = r.nodePathCache = r.nodePathCache || [];\n var key = hashStrings(shape === 'polygon' ? shape + ',' + points.join(',') : shape, '' + height, '' + width, '' + cornerRadius);\n var cachedPath = pathCache[key];\n var path;\n var cacheHit = false;\n if (cachedPath != null) {\n path = cachedPath;\n cacheHit = true;\n rs.pathCache = path;\n } else {\n path = new Path2D();\n pathCache[key] = rs.pathCache = path;\n }\n return {\n path: path,\n cacheHit: cacheHit\n };\n };\n var styleShape = node.pstyle('shape').strValue;\n var shapePts = node.pstyle('shape-polygon-points').pfValue;\n if (usePaths) {\n context.translate(pos.x, pos.y);\n var shapePath = getPath(nodeWidth, nodeHeight, styleShape, shapePts);\n path = shapePath.path;\n pathCacheHit = shapePath.cacheHit;\n }\n var drawShape = function drawShape() {\n if (!pathCacheHit) {\n var npos = pos;\n if (usePaths) {\n npos = {\n x: 0,\n y: 0\n };\n }\n r.nodeShapes[r.getNodeShape(node)].draw(path || context, npos.x, npos.y, nodeWidth, nodeHeight, cornerRadius, rs);\n }\n if (usePaths) {\n context.fill(path);\n } else {\n context.fill();\n }\n };\n var drawImages = function drawImages() {\n var nodeOpacity = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : eleOpacity;\n var inside = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : true;\n var prevBging = _p.backgrounding;\n var totalCompleted = 0;\n for (var _i = 0; _i < image.length; _i++) {\n var bgContainment = node.cy().style().getIndexedStyle(node, 'background-image-containment', 'value', _i);\n if (inside && bgContainment === 'over' || !inside && bgContainment === 'inside') {\n totalCompleted++;\n continue;\n }\n if (urlDefined[_i] && image[_i].complete && !image[_i].error) {\n totalCompleted++;\n r.drawInscribedImage(context, image[_i], node, _i, nodeOpacity);\n }\n }\n _p.backgrounding = !(totalCompleted === numImages);\n if (prevBging !== _p.backgrounding) {\n // update style b/c :backgrounding state changed\n node.updateStyle(false);\n }\n };\n var drawPie = function drawPie() {\n var redrawShape = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : false;\n var pieOpacity = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : eleOpacity;\n if (r.hasPie(node)) {\n r.drawPie(context, node, pieOpacity);\n\n // redraw/restore path if steps after pie need it\n if (redrawShape) {\n if (!usePaths) {\n r.nodeShapes[r.getNodeShape(node)].draw(context, pos.x, pos.y, nodeWidth, nodeHeight, cornerRadius, rs);\n }\n }\n }\n };\n var drawStripe = function drawStripe() {\n var redrawShape = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : false;\n var stripeOpacity = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : eleOpacity;\n if (r.hasStripe(node)) {\n context.save();\n if (usePaths) {\n context.clip(rs.pathCache);\n } else {\n r.nodeShapes[r.getNodeShape(node)].draw(context, pos.x, pos.y, nodeWidth, nodeHeight, cornerRadius, rs);\n context.clip();\n }\n r.drawStripe(context, node, stripeOpacity);\n context.restore();\n\n // redraw/restore path if steps after stripes need it\n if (redrawShape) {\n if (!usePaths) {\n r.nodeShapes[r.getNodeShape(node)].draw(context, pos.x, pos.y, nodeWidth, nodeHeight, cornerRadius, rs);\n }\n }\n }\n };\n var darken = function darken() {\n var darkenOpacity = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : eleOpacity;\n var opacity = (darkness > 0 ? darkness : -darkness) * darkenOpacity;\n var c = darkness > 0 ? 0 : 255;\n if (darkness !== 0) {\n r.colorFillStyle(context, c, c, c, opacity);\n if (usePaths) {\n context.fill(path);\n } else {\n context.fill();\n }\n }\n };\n var drawBorder = function drawBorder() {\n if (borderWidth > 0) {\n context.lineWidth = borderWidth;\n context.lineCap = borderCap;\n context.lineJoin = borderJoin;\n if (context.setLineDash) {\n // for very outofdate browsers\n switch (borderStyle) {\n case 'dotted':\n context.setLineDash([1, 1]);\n break;\n case 'dashed':\n context.setLineDash(borderPattern);\n context.lineDashOffset = borderOffset;\n break;\n case 'solid':\n case 'double':\n context.setLineDash([]);\n break;\n }\n }\n if (borderPosition !== 'center') {\n context.save();\n context.lineWidth *= 2;\n if (borderPosition === 'inside') {\n usePaths ? context.clip(path) : context.clip();\n } else {\n var region = new Path2D();\n region.rect(-nodeWidth / 2 - borderWidth, -nodeHeight / 2 - borderWidth, nodeWidth + 2 * borderWidth, nodeHeight + 2 * borderWidth);\n region.addPath(path);\n context.clip(region, 'evenodd');\n }\n usePaths ? context.stroke(path) : context.stroke();\n context.restore();\n } else {\n usePaths ? context.stroke(path) : context.stroke();\n }\n if (borderStyle === 'double') {\n context.lineWidth = borderWidth / 3;\n var gco = context.globalCompositeOperation;\n context.globalCompositeOperation = 'destination-out';\n if (usePaths) {\n context.stroke(path);\n } else {\n context.stroke();\n }\n context.globalCompositeOperation = gco;\n }\n\n // reset in case we changed the border style\n if (context.setLineDash) {\n // for very outofdate browsers\n context.setLineDash([]);\n }\n }\n };\n var drawOutline = function drawOutline() {\n if (outlineWidth > 0) {\n context.lineWidth = outlineWidth;\n context.lineCap = 'butt';\n if (context.setLineDash) {\n // for very outofdate browsers\n switch (outlineStyle) {\n case 'dotted':\n context.setLineDash([1, 1]);\n break;\n case 'dashed':\n context.setLineDash([4, 2]);\n break;\n case 'solid':\n case 'double':\n context.setLineDash([]);\n break;\n }\n }\n var npos = pos;\n if (usePaths) {\n npos = {\n x: 0,\n y: 0\n };\n }\n var shape = r.getNodeShape(node);\n var bWidth = borderWidth;\n if (borderPosition === 'inside') bWidth = 0;\n if (borderPosition === 'outside') bWidth *= 2;\n var scaleX = (nodeWidth + bWidth + (outlineWidth + outlineOffset)) / nodeWidth;\n var scaleY = (nodeHeight + bWidth + (outlineWidth + outlineOffset)) / nodeHeight;\n var sWidth = nodeWidth * scaleX;\n var sHeight = nodeHeight * scaleY;\n var points = r.nodeShapes[shape].points;\n var _path;\n if (usePaths) {\n var outlinePath = getPath(sWidth, sHeight, shape, points);\n _path = outlinePath.path;\n }\n\n // draw the outline path, either by using expanded points or by scaling \n // the dimensions, depending on shape\n if (shape === \"ellipse\") {\n r.drawEllipsePath(_path || context, npos.x, npos.y, sWidth, sHeight);\n } else if (['round-diamond', 'round-heptagon', 'round-hexagon', 'round-octagon', 'round-pentagon', 'round-polygon', 'round-triangle', 'round-tag'].includes(shape)) {\n var sMult = 0;\n var offsetX = 0;\n var offsetY = 0;\n if (shape === 'round-diamond') {\n sMult = (bWidth + outlineOffset + outlineWidth) * 1.4;\n } else if (shape === 'round-heptagon') {\n sMult = (bWidth + outlineOffset + outlineWidth) * 1.075;\n offsetY = -(bWidth / 2 + outlineOffset + outlineWidth) / 35;\n } else if (shape === 'round-hexagon') {\n sMult = (bWidth + outlineOffset + outlineWidth) * 1.12;\n } else if (shape === 'round-pentagon') {\n sMult = (bWidth + outlineOffset + outlineWidth) * 1.13;\n offsetY = -(bWidth / 2 + outlineOffset + outlineWidth) / 15;\n } else if (shape === 'round-tag') {\n sMult = (bWidth + outlineOffset + outlineWidth) * 1.12;\n offsetX = (bWidth / 2 + outlineWidth + outlineOffset) * .07;\n } else if (shape === 'round-triangle') {\n sMult = (bWidth + outlineOffset + outlineWidth) * (Math.PI / 2);\n offsetY = -(bWidth + outlineOffset / 2 + outlineWidth) / Math.PI;\n }\n if (sMult !== 0) {\n scaleX = (nodeWidth + sMult) / nodeWidth;\n sWidth = nodeWidth * scaleX;\n if (!['round-hexagon', 'round-tag'].includes(shape)) {\n scaleY = (nodeHeight + sMult) / nodeHeight;\n sHeight = nodeHeight * scaleY;\n }\n }\n cornerRadius = cornerRadius === 'auto' ? getRoundPolygonRadius(sWidth, sHeight) : cornerRadius;\n var halfW = sWidth / 2;\n var halfH = sHeight / 2;\n var radius = cornerRadius + (bWidth + outlineWidth + outlineOffset) / 2;\n var p = new Array(points.length / 2);\n var corners = new Array(points.length / 2);\n for (var _i2 = 0; _i2 < points.length / 2; _i2++) {\n p[_i2] = {\n x: npos.x + offsetX + halfW * points[_i2 * 2],\n y: npos.y + offsetY + halfH * points[_i2 * 2 + 1]\n };\n }\n var _i3,\n p1,\n p2,\n p3,\n len = p.length;\n p1 = p[len - 1];\n // for each point\n for (_i3 = 0; _i3 < len; _i3++) {\n p2 = p[_i3 % len];\n p3 = p[(_i3 + 1) % len];\n corners[_i3] = getRoundCorner(p1, p2, p3, radius);\n p1 = p2;\n p2 = p3;\n }\n r.drawRoundPolygonPath(_path || context, npos.x + offsetX, npos.y + offsetY, nodeWidth * scaleX, nodeHeight * scaleY, points, corners);\n } else if (['roundrectangle', 'round-rectangle'].includes(shape)) {\n cornerRadius = cornerRadius === 'auto' ? getRoundRectangleRadius(sWidth, sHeight) : cornerRadius;\n r.drawRoundRectanglePath(_path || context, npos.x, npos.y, sWidth, sHeight, cornerRadius + (bWidth + outlineWidth + outlineOffset) / 2);\n } else if (['cutrectangle', 'cut-rectangle'].includes(shape)) {\n cornerRadius = cornerRadius === 'auto' ? getCutRectangleCornerLength() : cornerRadius;\n r.drawCutRectanglePath(_path || context, npos.x, npos.y, sWidth, sHeight, null, cornerRadius + (bWidth + outlineWidth + outlineOffset) / 4);\n } else if (['bottomroundrectangle', 'bottom-round-rectangle'].includes(shape)) {\n cornerRadius = cornerRadius === 'auto' ? getRoundRectangleRadius(sWidth, sHeight) : cornerRadius;\n r.drawBottomRoundRectanglePath(_path || context, npos.x, npos.y, sWidth, sHeight, cornerRadius + (bWidth + outlineWidth + outlineOffset) / 2);\n } else if (shape === \"barrel\") {\n r.drawBarrelPath(_path || context, npos.x, npos.y, sWidth, sHeight);\n } else if (shape.startsWith(\"polygon\") || ['rhomboid', 'right-rhomboid', 'round-tag', 'tag', 'vee'].includes(shape)) {\n var pad = (bWidth + outlineWidth + outlineOffset) / nodeWidth;\n points = joinLines(expandPolygon(points, pad));\n r.drawPolygonPath(_path || context, npos.x, npos.y, nodeWidth, nodeHeight, points);\n } else {\n var _pad = (bWidth + outlineWidth + outlineOffset) / nodeWidth;\n points = joinLines(expandPolygon(points, -_pad));\n r.drawPolygonPath(_path || context, npos.x, npos.y, nodeWidth, nodeHeight, points);\n }\n if (usePaths) {\n context.stroke(_path);\n } else {\n context.stroke();\n }\n if (outlineStyle === 'double') {\n context.lineWidth = bWidth / 3;\n var gco = context.globalCompositeOperation;\n context.globalCompositeOperation = 'destination-out';\n if (usePaths) {\n context.stroke(_path);\n } else {\n context.stroke();\n }\n context.globalCompositeOperation = gco;\n }\n\n // reset in case we changed the border style\n if (context.setLineDash) {\n // for very outofdate browsers\n context.setLineDash([]);\n }\n }\n };\n var drawOverlay = function drawOverlay() {\n if (shouldDrawOverlay) {\n r.drawNodeOverlay(context, node, pos, nodeWidth, nodeHeight);\n }\n };\n var drawUnderlay = function drawUnderlay() {\n if (shouldDrawOverlay) {\n r.drawNodeUnderlay(context, node, pos, nodeWidth, nodeHeight);\n }\n };\n var drawText = function drawText() {\n r.drawElementText(context, node, null, drawLabel);\n };\n var ghost = node.pstyle('ghost').value === 'yes';\n if (ghost) {\n var gx = node.pstyle('ghost-offset-x').pfValue;\n var gy = node.pstyle('ghost-offset-y').pfValue;\n var ghostOpacity = node.pstyle('ghost-opacity').value;\n var effGhostOpacity = ghostOpacity * eleOpacity;\n context.translate(gx, gy);\n setupOutlineColor();\n drawOutline();\n setupShapeColor(ghostOpacity * bgOpacity);\n drawShape();\n drawImages(effGhostOpacity, true);\n setupBorderColor(ghostOpacity * borderOpacity);\n drawBorder();\n drawPie(darkness !== 0 || borderWidth !== 0);\n drawStripe(darkness !== 0 || borderWidth !== 0);\n drawImages(effGhostOpacity, false);\n darken(effGhostOpacity);\n context.translate(-gx, -gy);\n }\n if (usePaths) {\n context.translate(-pos.x, -pos.y);\n }\n drawUnderlay();\n if (usePaths) {\n context.translate(pos.x, pos.y);\n }\n setupOutlineColor();\n drawOutline();\n setupShapeColor();\n drawShape();\n drawImages(eleOpacity, true);\n setupBorderColor();\n drawBorder();\n drawPie(darkness !== 0 || borderWidth !== 0);\n drawStripe(darkness !== 0 || borderWidth !== 0);\n drawImages(eleOpacity, false);\n darken();\n if (usePaths) {\n context.translate(-pos.x, -pos.y);\n }\n drawText();\n drawOverlay();\n\n //\n // clean up shift\n\n if (shiftToOriginWithBb) {\n context.translate(bb.x1, bb.y1);\n }\n};\nvar drawNodeOverlayUnderlay = function drawNodeOverlayUnderlay(overlayOrUnderlay) {\n if (!['overlay', 'underlay'].includes(overlayOrUnderlay)) {\n throw new Error('Invalid state');\n }\n return function (context, node, pos, nodeWidth, nodeHeight) {\n var r = this;\n if (!node.visible()) {\n return;\n }\n var padding = node.pstyle(\"\".concat(overlayOrUnderlay, \"-padding\")).pfValue;\n var opacity = node.pstyle(\"\".concat(overlayOrUnderlay, \"-opacity\")).value;\n var color = node.pstyle(\"\".concat(overlayOrUnderlay, \"-color\")).value;\n var shape = node.pstyle(\"\".concat(overlayOrUnderlay, \"-shape\")).value;\n var radius = node.pstyle(\"\".concat(overlayOrUnderlay, \"-corner-radius\")).value;\n if (opacity > 0) {\n pos = pos || node.position();\n if (nodeWidth == null || nodeHeight == null) {\n var _padding = node.padding();\n nodeWidth = node.width() + 2 * _padding;\n nodeHeight = node.height() + 2 * _padding;\n }\n r.colorFillStyle(context, color[0], color[1], color[2], opacity);\n r.nodeShapes[shape].draw(context, pos.x, pos.y, nodeWidth + padding * 2, nodeHeight + padding * 2, radius);\n context.fill();\n }\n };\n};\nCRp$6.drawNodeOverlay = drawNodeOverlayUnderlay('overlay');\nCRp$6.drawNodeUnderlay = drawNodeOverlayUnderlay('underlay');\n\n// does the node have at least one pie piece?\nCRp$6.hasPie = function (node) {\n node = node[0]; // ensure ele ref\n\n return node._private.hasPie;\n};\nCRp$6.hasStripe = function (node) {\n node = node[0]; // ensure ele ref\n\n return node._private.hasStripe;\n};\nCRp$6.drawPie = function (context, node, nodeOpacity, pos) {\n node = node[0]; // ensure ele ref\n pos = pos || node.position();\n var cyStyle = node.cy().style();\n var pieSize = node.pstyle('pie-size');\n var hole = node.pstyle('pie-hole');\n var overallStartAngle = node.pstyle('pie-start-angle').pfValue;\n var x = pos.x;\n var y = pos.y;\n var nodeW = node.width();\n var nodeH = node.height();\n var radius = Math.min(nodeW, nodeH) / 2; // must fit in node\n var holeRadius;\n var lastPercent = 0; // what % to continue drawing pie slices from on [0, 1]\n var usePaths = this.usePaths();\n if (usePaths) {\n x = 0;\n y = 0;\n }\n if (pieSize.units === '%') {\n radius = radius * pieSize.pfValue;\n } else if (pieSize.pfValue !== undefined) {\n radius = pieSize.pfValue / 2; // diameter in pixels => radius\n }\n if (hole.units === '%') {\n holeRadius = radius * hole.pfValue;\n } else if (hole.pfValue !== undefined) {\n holeRadius = hole.pfValue / 2; // diameter in pixels => radius\n }\n if (holeRadius >= radius) {\n return; // the pie would be invisible anyway\n }\n for (var i = 1; i <= cyStyle.pieBackgroundN; i++) {\n // 1..N\n var size = node.pstyle('pie-' + i + '-background-size').value;\n var color = node.pstyle('pie-' + i + '-background-color').value;\n var opacity = node.pstyle('pie-' + i + '-background-opacity').value * nodeOpacity;\n var percent = size / 100; // map integer range [0, 100] to [0, 1]\n\n // percent can't push beyond 1\n if (percent + lastPercent > 1) {\n percent = 1 - lastPercent;\n }\n var angleStart = 1.5 * Math.PI + 2 * Math.PI * lastPercent; // start at 12 o'clock and go clockwise\n angleStart += overallStartAngle; // shift by the overall pie start angle\n var angleDelta = 2 * Math.PI * percent;\n var angleEnd = angleStart + angleDelta;\n\n // ignore if\n // - zero size\n // - we're already beyond the full circle\n // - adding the current slice would go beyond the full circle\n if (size === 0 || lastPercent >= 1 || lastPercent + percent > 1) {\n continue;\n }\n if (holeRadius === 0) {\n // make a pie slice\n context.beginPath();\n context.moveTo(x, y);\n context.arc(x, y, radius, angleStart, angleEnd);\n context.closePath();\n } else {\n // make a pie slice that's like the above but with a hole in the middle\n context.beginPath();\n context.arc(x, y, radius, angleStart, angleEnd);\n context.arc(x, y, holeRadius, angleEnd, angleStart, true); // true for anticlockwise\n context.closePath();\n }\n this.colorFillStyle(context, color[0], color[1], color[2], opacity);\n context.fill();\n lastPercent += percent;\n }\n};\nCRp$6.drawStripe = function (context, node, nodeOpacity, pos) {\n node = node[0]; // ensure ele ref\n pos = pos || node.position();\n var cyStyle = node.cy().style();\n var x = pos.x;\n var y = pos.y;\n var nodeW = node.width();\n var nodeH = node.height();\n var lastPercent = 0; // what % to continue drawing pie slices from on [0, 1]\n var usePaths = this.usePaths();\n context.save();\n var direction = node.pstyle('stripe-direction').value;\n var stripeSize = node.pstyle('stripe-size');\n switch (direction) {\n case 'vertical':\n break;\n // default\n case 'righward':\n context.rotate(-Math.PI / 2);\n break;\n }\n var stripeW = nodeW;\n var stripeH = nodeH;\n if (stripeSize.units === '%') {\n stripeW = stripeW * stripeSize.pfValue;\n stripeH = stripeH * stripeSize.pfValue;\n } else if (stripeSize.pfValue !== undefined) {\n stripeW = stripeSize.pfValue;\n stripeH = stripeSize.pfValue;\n }\n if (usePaths) {\n x = 0;\n y = 0;\n }\n\n // shift up from the centre of the node to the top-left corner\n y -= stripeW / 2;\n x -= stripeH / 2;\n for (var i = 1; i <= cyStyle.stripeBackgroundN; i++) {\n // 1..N\n var size = node.pstyle('stripe-' + i + '-background-size').value;\n var color = node.pstyle('stripe-' + i + '-background-color').value;\n var opacity = node.pstyle('stripe-' + i + '-background-opacity').value * nodeOpacity;\n var percent = size / 100; // map integer range [0, 100] to [0, 1]\n\n // percent can't push beyond 1\n if (percent + lastPercent > 1) {\n percent = 1 - lastPercent;\n }\n\n // ignore if\n // - zero size\n // - we're already beyond the full chart\n // - adding the current slice would go beyond the full chart\n if (size === 0 || lastPercent >= 1 || lastPercent + percent > 1) {\n continue;\n }\n\n // draw rect for the current stripe\n context.beginPath();\n context.rect(x, y + stripeH * lastPercent, stripeW, stripeH * percent);\n context.closePath();\n this.colorFillStyle(context, color[0], color[1], color[2], opacity);\n context.fill();\n lastPercent += percent;\n }\n context.restore();\n};\n\nvar CRp$5 = {};\nvar motionBlurDelay = 100;\n\n// var isFirefox = typeof InstallTrigger !== 'undefined';\n\nCRp$5.getPixelRatio = function () {\n var context = this.data.contexts[0];\n if (this.forcedPixelRatio != null) {\n return this.forcedPixelRatio;\n }\n var containerWindow = this.cy.window();\n var backingStore = context.backingStorePixelRatio || context.webkitBackingStorePixelRatio || context.mozBackingStorePixelRatio || context.msBackingStorePixelRatio || context.oBackingStorePixelRatio || context.backingStorePixelRatio || 1;\n return (containerWindow.devicePixelRatio || 1) / backingStore; // eslint-disable-line no-undef\n};\nCRp$5.paintCache = function (context) {\n var caches = this.paintCaches = this.paintCaches || [];\n var needToCreateCache = true;\n var cache;\n for (var i = 0; i < caches.length; i++) {\n cache = caches[i];\n if (cache.context === context) {\n needToCreateCache = false;\n break;\n }\n }\n if (needToCreateCache) {\n cache = {\n context: context\n };\n caches.push(cache);\n }\n return cache;\n};\nCRp$5.createGradientStyleFor = function (context, shapeStyleName, ele, fill, opacity) {\n var gradientStyle;\n var usePaths = this.usePaths();\n var colors = ele.pstyle(shapeStyleName + '-gradient-stop-colors').value,\n positions = ele.pstyle(shapeStyleName + '-gradient-stop-positions').pfValue;\n if (fill === 'radial-gradient') {\n if (ele.isEdge()) {\n var start = ele.sourceEndpoint(),\n end = ele.targetEndpoint(),\n mid = ele.midpoint();\n var d1 = dist(start, mid);\n var d2 = dist(end, mid);\n gradientStyle = context.createRadialGradient(mid.x, mid.y, 0, mid.x, mid.y, Math.max(d1, d2));\n } else {\n var pos = usePaths ? {\n x: 0,\n y: 0\n } : ele.position(),\n width = ele.paddedWidth(),\n height = ele.paddedHeight();\n gradientStyle = context.createRadialGradient(pos.x, pos.y, 0, pos.x, pos.y, Math.max(width, height));\n }\n } else {\n if (ele.isEdge()) {\n var _start = ele.sourceEndpoint(),\n _end = ele.targetEndpoint();\n gradientStyle = context.createLinearGradient(_start.x, _start.y, _end.x, _end.y);\n } else {\n var _pos = usePaths ? {\n x: 0,\n y: 0\n } : ele.position(),\n _width = ele.paddedWidth(),\n _height = ele.paddedHeight(),\n halfWidth = _width / 2,\n halfHeight = _height / 2;\n var direction = ele.pstyle('background-gradient-direction').value;\n switch (direction) {\n case 'to-bottom':\n gradientStyle = context.createLinearGradient(_pos.x, _pos.y - halfHeight, _pos.x, _pos.y + halfHeight);\n break;\n case 'to-top':\n gradientStyle = context.createLinearGradient(_pos.x, _pos.y + halfHeight, _pos.x, _pos.y - halfHeight);\n break;\n case 'to-left':\n gradientStyle = context.createLinearGradient(_pos.x + halfWidth, _pos.y, _pos.x - halfWidth, _pos.y);\n break;\n case 'to-right':\n gradientStyle = context.createLinearGradient(_pos.x - halfWidth, _pos.y, _pos.x + halfWidth, _pos.y);\n break;\n case 'to-bottom-right':\n case 'to-right-bottom':\n gradientStyle = context.createLinearGradient(_pos.x - halfWidth, _pos.y - halfHeight, _pos.x + halfWidth, _pos.y + halfHeight);\n break;\n case 'to-top-right':\n case 'to-right-top':\n gradientStyle = context.createLinearGradient(_pos.x - halfWidth, _pos.y + halfHeight, _pos.x + halfWidth, _pos.y - halfHeight);\n break;\n case 'to-bottom-left':\n case 'to-left-bottom':\n gradientStyle = context.createLinearGradient(_pos.x + halfWidth, _pos.y - halfHeight, _pos.x - halfWidth, _pos.y + halfHeight);\n break;\n case 'to-top-left':\n case 'to-left-top':\n gradientStyle = context.createLinearGradient(_pos.x + halfWidth, _pos.y + halfHeight, _pos.x - halfWidth, _pos.y - halfHeight);\n break;\n }\n }\n }\n if (!gradientStyle) return null; // invalid gradient style\n\n var hasPositions = positions.length === colors.length;\n var length = colors.length;\n for (var i = 0; i < length; i++) {\n gradientStyle.addColorStop(hasPositions ? positions[i] : i / (length - 1), 'rgba(' + colors[i][0] + ',' + colors[i][1] + ',' + colors[i][2] + ',' + opacity + ')');\n }\n return gradientStyle;\n};\nCRp$5.gradientFillStyle = function (context, ele, fill, opacity) {\n var gradientStyle = this.createGradientStyleFor(context, 'background', ele, fill, opacity);\n if (!gradientStyle) return null; // error\n context.fillStyle = gradientStyle;\n};\nCRp$5.colorFillStyle = function (context, r, g, b, a) {\n context.fillStyle = 'rgba(' + r + ',' + g + ',' + b + ',' + a + ')';\n // turn off for now, seems context does its own caching\n\n // var cache = this.paintCache(context);\n\n // var fillStyle = 'rgba(' + r + ',' + g + ',' + b + ',' + a + ')';\n\n // if( cache.fillStyle !== fillStyle ){\n // context.fillStyle = cache.fillStyle = fillStyle;\n // }\n};\nCRp$5.eleFillStyle = function (context, ele, opacity) {\n var backgroundFill = ele.pstyle('background-fill').value;\n if (backgroundFill === 'linear-gradient' || backgroundFill === 'radial-gradient') {\n this.gradientFillStyle(context, ele, backgroundFill, opacity);\n } else {\n var backgroundColor = ele.pstyle('background-color').value;\n this.colorFillStyle(context, backgroundColor[0], backgroundColor[1], backgroundColor[2], opacity);\n }\n};\nCRp$5.gradientStrokeStyle = function (context, ele, fill, opacity) {\n var gradientStyle = this.createGradientStyleFor(context, 'line', ele, fill, opacity);\n if (!gradientStyle) return null; // error\n context.strokeStyle = gradientStyle;\n};\nCRp$5.colorStrokeStyle = function (context, r, g, b, a) {\n context.strokeStyle = 'rgba(' + r + ',' + g + ',' + b + ',' + a + ')';\n // turn off for now, seems context does its own caching\n\n // var cache = this.paintCache(context);\n\n // var strokeStyle = 'rgba(' + r + ',' + g + ',' + b + ',' + a + ')';\n\n // if( cache.strokeStyle !== strokeStyle ){\n // context.strokeStyle = cache.strokeStyle = strokeStyle;\n // }\n};\nCRp$5.eleStrokeStyle = function (context, ele, opacity) {\n var lineFill = ele.pstyle('line-fill').value;\n if (lineFill === 'linear-gradient' || lineFill === 'radial-gradient') {\n this.gradientStrokeStyle(context, ele, lineFill, opacity);\n } else {\n var lineColor = ele.pstyle('line-color').value;\n this.colorStrokeStyle(context, lineColor[0], lineColor[1], lineColor[2], opacity);\n }\n};\n\n// Resize canvas\nCRp$5.matchCanvasSize = function (container) {\n var r = this;\n var data = r.data;\n var bb = r.findContainerClientCoords();\n var width = bb[2];\n var height = bb[3];\n var pixelRatio = r.getPixelRatio();\n var mbPxRatio = r.motionBlurPxRatio;\n if (container === r.data.bufferCanvases[r.MOTIONBLUR_BUFFER_NODE] || container === r.data.bufferCanvases[r.MOTIONBLUR_BUFFER_DRAG]) {\n pixelRatio = mbPxRatio;\n }\n var canvasWidth = width * pixelRatio;\n var canvasHeight = height * pixelRatio;\n var canvas;\n if (canvasWidth === r.canvasWidth && canvasHeight === r.canvasHeight) {\n return; // save cycles if same\n }\n r.fontCaches = null; // resizing resets the style\n\n var canvasContainer = data.canvasContainer;\n canvasContainer.style.width = width + 'px';\n canvasContainer.style.height = height + 'px';\n for (var i = 0; i < r.CANVAS_LAYERS; i++) {\n canvas = data.canvases[i];\n canvas.width = canvasWidth;\n canvas.height = canvasHeight;\n canvas.style.width = width + 'px';\n canvas.style.height = height + 'px';\n }\n for (var i = 0; i < r.BUFFER_COUNT; i++) {\n canvas = data.bufferCanvases[i];\n canvas.width = canvasWidth;\n canvas.height = canvasHeight;\n canvas.style.width = width + 'px';\n canvas.style.height = height + 'px';\n }\n r.textureMult = 1;\n if (pixelRatio <= 1) {\n canvas = data.bufferCanvases[r.TEXTURE_BUFFER];\n r.textureMult = 2;\n canvas.width = canvasWidth * r.textureMult;\n canvas.height = canvasHeight * r.textureMult;\n }\n r.canvasWidth = canvasWidth;\n r.canvasHeight = canvasHeight;\n r.pixelRatio = pixelRatio;\n};\nCRp$5.renderTo = function (cxt, zoom, pan, pxRatio) {\n this.render({\n forcedContext: cxt,\n forcedZoom: zoom,\n forcedPan: pan,\n drawAllLayers: true,\n forcedPxRatio: pxRatio\n });\n};\nCRp$5.clearCanvas = function () {\n var r = this;\n var data = r.data;\n function clear(context) {\n context.clearRect(0, 0, r.canvasWidth, r.canvasHeight);\n }\n clear(data.contexts[r.NODE]);\n clear(data.contexts[r.DRAG]);\n};\nCRp$5.render = function (options) {\n var r = this;\n options = options || staticEmptyObject();\n var cy = r.cy;\n var forcedContext = options.forcedContext;\n var drawAllLayers = options.drawAllLayers;\n var drawOnlyNodeLayer = options.drawOnlyNodeLayer;\n var forcedZoom = options.forcedZoom;\n var forcedPan = options.forcedPan;\n var pixelRatio = options.forcedPxRatio === undefined ? this.getPixelRatio() : options.forcedPxRatio;\n var data = r.data;\n var needDraw = data.canvasNeedsRedraw;\n var textureDraw = r.textureOnViewport && !forcedContext && (r.pinching || r.hoverData.dragging || r.swipePanning || r.data.wheelZooming);\n var motionBlur = options.motionBlur !== undefined ? options.motionBlur : r.motionBlur;\n var mbPxRatio = r.motionBlurPxRatio;\n var hasCompoundNodes = cy.hasCompoundNodes();\n var inNodeDragGesture = r.hoverData.draggingEles;\n var inBoxSelection = r.hoverData.selecting || r.touchData.selecting ? true : false;\n motionBlur = motionBlur && !forcedContext && r.motionBlurEnabled && !inBoxSelection;\n var motionBlurFadeEffect = motionBlur;\n if (!forcedContext) {\n if (r.prevPxRatio !== pixelRatio) {\n r.invalidateContainerClientCoordsCache();\n r.matchCanvasSize(r.container);\n r.redrawHint('eles', true);\n r.redrawHint('drag', true);\n }\n r.prevPxRatio = pixelRatio;\n }\n if (!forcedContext && r.motionBlurTimeout) {\n clearTimeout(r.motionBlurTimeout);\n }\n if (motionBlur) {\n if (r.mbFrames == null) {\n r.mbFrames = 0;\n }\n r.mbFrames++;\n if (r.mbFrames < 3) {\n // need several frames before even high quality motionblur\n motionBlurFadeEffect = false;\n }\n\n // go to lower quality blurry frames when several m/b frames have been rendered (avoids flashing)\n if (r.mbFrames > r.minMbLowQualFrames) {\n //r.fullQualityMb = false;\n r.motionBlurPxRatio = r.mbPxRBlurry;\n }\n }\n if (r.clearingMotionBlur) {\n r.motionBlurPxRatio = 1;\n }\n\n // b/c drawToContext() may be async w.r.t. redraw(), keep track of last texture frame\n // because a rogue async texture frame would clear needDraw\n if (r.textureDrawLastFrame && !textureDraw) {\n needDraw[r.NODE] = true;\n needDraw[r.SELECT_BOX] = true;\n }\n var style = cy.style();\n var zoom = cy.zoom();\n var effectiveZoom = forcedZoom !== undefined ? forcedZoom : zoom;\n var pan = cy.pan();\n var effectivePan = {\n x: pan.x,\n y: pan.y\n };\n var vp = {\n zoom: zoom,\n pan: {\n x: pan.x,\n y: pan.y\n }\n };\n var prevVp = r.prevViewport;\n var viewportIsDiff = prevVp === undefined || vp.zoom !== prevVp.zoom || vp.pan.x !== prevVp.pan.x || vp.pan.y !== prevVp.pan.y;\n\n // we want the low quality motionblur only when the viewport is being manipulated etc (where it's not noticed)\n if (!viewportIsDiff && !(inNodeDragGesture && !hasCompoundNodes)) {\n r.motionBlurPxRatio = 1;\n }\n if (forcedPan) {\n effectivePan = forcedPan;\n }\n\n // apply pixel ratio\n\n effectiveZoom *= pixelRatio;\n effectivePan.x *= pixelRatio;\n effectivePan.y *= pixelRatio;\n var eles = r.getCachedZSortedEles();\n function mbclear(context, x, y, w, h) {\n var gco = context.globalCompositeOperation;\n context.globalCompositeOperation = 'destination-out';\n r.colorFillStyle(context, 255, 255, 255, r.motionBlurTransparency);\n context.fillRect(x, y, w, h);\n context.globalCompositeOperation = gco;\n }\n function setContextTransform(context, clear) {\n var ePan, eZoom, w, h;\n if (!r.clearingMotionBlur && (context === data.bufferContexts[r.MOTIONBLUR_BUFFER_NODE] || context === data.bufferContexts[r.MOTIONBLUR_BUFFER_DRAG])) {\n ePan = {\n x: pan.x * mbPxRatio,\n y: pan.y * mbPxRatio\n };\n eZoom = zoom * mbPxRatio;\n w = r.canvasWidth * mbPxRatio;\n h = r.canvasHeight * mbPxRatio;\n } else {\n ePan = effectivePan;\n eZoom = effectiveZoom;\n w = r.canvasWidth;\n h = r.canvasHeight;\n }\n context.setTransform(1, 0, 0, 1, 0, 0);\n if (clear === 'motionBlur') {\n mbclear(context, 0, 0, w, h);\n } else if (!forcedContext && (clear === undefined || clear)) {\n context.clearRect(0, 0, w, h);\n }\n if (!drawAllLayers) {\n context.translate(ePan.x, ePan.y);\n context.scale(eZoom, eZoom);\n }\n if (forcedPan) {\n context.translate(forcedPan.x, forcedPan.y);\n }\n if (forcedZoom) {\n context.scale(forcedZoom, forcedZoom);\n }\n }\n if (!textureDraw) {\n r.textureDrawLastFrame = false;\n }\n if (textureDraw) {\n r.textureDrawLastFrame = true;\n if (!r.textureCache) {\n r.textureCache = {};\n r.textureCache.bb = cy.mutableElements().boundingBox();\n r.textureCache.texture = r.data.bufferCanvases[r.TEXTURE_BUFFER];\n var cxt = r.data.bufferContexts[r.TEXTURE_BUFFER];\n cxt.setTransform(1, 0, 0, 1, 0, 0);\n cxt.clearRect(0, 0, r.canvasWidth * r.textureMult, r.canvasHeight * r.textureMult);\n r.render({\n forcedContext: cxt,\n drawOnlyNodeLayer: true,\n forcedPxRatio: pixelRatio * r.textureMult\n });\n var vp = r.textureCache.viewport = {\n zoom: cy.zoom(),\n pan: cy.pan(),\n width: r.canvasWidth,\n height: r.canvasHeight\n };\n vp.mpan = {\n x: (0 - vp.pan.x) / vp.zoom,\n y: (0 - vp.pan.y) / vp.zoom\n };\n }\n needDraw[r.DRAG] = false;\n needDraw[r.NODE] = false;\n var context = data.contexts[r.NODE];\n var texture = r.textureCache.texture;\n var vp = r.textureCache.viewport;\n context.setTransform(1, 0, 0, 1, 0, 0);\n if (motionBlur) {\n mbclear(context, 0, 0, vp.width, vp.height);\n } else {\n context.clearRect(0, 0, vp.width, vp.height);\n }\n var outsideBgColor = style.core('outside-texture-bg-color').value;\n var outsideBgOpacity = style.core('outside-texture-bg-opacity').value;\n r.colorFillStyle(context, outsideBgColor[0], outsideBgColor[1], outsideBgColor[2], outsideBgOpacity);\n context.fillRect(0, 0, vp.width, vp.height);\n var zoom = cy.zoom();\n setContextTransform(context, false);\n context.clearRect(vp.mpan.x, vp.mpan.y, vp.width / vp.zoom / pixelRatio, vp.height / vp.zoom / pixelRatio);\n context.drawImage(texture, vp.mpan.x, vp.mpan.y, vp.width / vp.zoom / pixelRatio, vp.height / vp.zoom / pixelRatio);\n } else if (r.textureOnViewport && !forcedContext) {\n // clear the cache since we don't need it\n r.textureCache = null;\n }\n var extent = cy.extent();\n var vpManip = r.pinching || r.hoverData.dragging || r.swipePanning || r.data.wheelZooming || r.hoverData.draggingEles || r.cy.animated();\n var hideEdges = r.hideEdgesOnViewport && vpManip;\n var needMbClear = [];\n needMbClear[r.NODE] = !needDraw[r.NODE] && motionBlur && !r.clearedForMotionBlur[r.NODE] || r.clearingMotionBlur;\n if (needMbClear[r.NODE]) {\n r.clearedForMotionBlur[r.NODE] = true;\n }\n needMbClear[r.DRAG] = !needDraw[r.DRAG] && motionBlur && !r.clearedForMotionBlur[r.DRAG] || r.clearingMotionBlur;\n if (needMbClear[r.DRAG]) {\n r.clearedForMotionBlur[r.DRAG] = true;\n }\n if (needDraw[r.NODE] || drawAllLayers || drawOnlyNodeLayer || needMbClear[r.NODE]) {\n var useBuffer = motionBlur && !needMbClear[r.NODE] && mbPxRatio !== 1;\n var context = forcedContext || (useBuffer ? r.data.bufferContexts[r.MOTIONBLUR_BUFFER_NODE] : data.contexts[r.NODE]);\n var clear = motionBlur && !useBuffer ? 'motionBlur' : undefined;\n setContextTransform(context, clear);\n if (hideEdges) {\n r.drawCachedNodes(context, eles.nondrag, pixelRatio, extent);\n } else {\n r.drawLayeredElements(context, eles.nondrag, pixelRatio, extent);\n }\n if (r.debug) {\n r.drawDebugPoints(context, eles.nondrag);\n }\n if (!drawAllLayers && !motionBlur) {\n needDraw[r.NODE] = false;\n }\n }\n if (!drawOnlyNodeLayer && (needDraw[r.DRAG] || drawAllLayers || needMbClear[r.DRAG])) {\n var useBuffer = motionBlur && !needMbClear[r.DRAG] && mbPxRatio !== 1;\n var context = forcedContext || (useBuffer ? r.data.bufferContexts[r.MOTIONBLUR_BUFFER_DRAG] : data.contexts[r.DRAG]);\n setContextTransform(context, motionBlur && !useBuffer ? 'motionBlur' : undefined);\n if (hideEdges) {\n r.drawCachedNodes(context, eles.drag, pixelRatio, extent);\n } else {\n r.drawCachedElements(context, eles.drag, pixelRatio, extent);\n }\n if (r.debug) {\n r.drawDebugPoints(context, eles.drag);\n }\n if (!drawAllLayers && !motionBlur) {\n needDraw[r.DRAG] = false;\n }\n }\n this.drawSelectionRectangle(options, setContextTransform);\n\n // motionblur: blit rendered blurry frames\n if (motionBlur && mbPxRatio !== 1) {\n var cxtNode = data.contexts[r.NODE];\n var txtNode = r.data.bufferCanvases[r.MOTIONBLUR_BUFFER_NODE];\n var cxtDrag = data.contexts[r.DRAG];\n var txtDrag = r.data.bufferCanvases[r.MOTIONBLUR_BUFFER_DRAG];\n var drawMotionBlur = function drawMotionBlur(cxt, txt, needClear) {\n cxt.setTransform(1, 0, 0, 1, 0, 0);\n if (needClear || !motionBlurFadeEffect) {\n cxt.clearRect(0, 0, r.canvasWidth, r.canvasHeight);\n } else {\n mbclear(cxt, 0, 0, r.canvasWidth, r.canvasHeight);\n }\n var pxr = mbPxRatio;\n cxt.drawImage(txt,\n // img\n 0, 0,\n // sx, sy\n r.canvasWidth * pxr, r.canvasHeight * pxr,\n // sw, sh\n 0, 0,\n // x, y\n r.canvasWidth, r.canvasHeight // w, h\n );\n };\n if (needDraw[r.NODE] || needMbClear[r.NODE]) {\n drawMotionBlur(cxtNode, txtNode, needMbClear[r.NODE]);\n needDraw[r.NODE] = false;\n }\n if (needDraw[r.DRAG] || needMbClear[r.DRAG]) {\n drawMotionBlur(cxtDrag, txtDrag, needMbClear[r.DRAG]);\n needDraw[r.DRAG] = false;\n }\n }\n r.prevViewport = vp;\n if (r.clearingMotionBlur) {\n r.clearingMotionBlur = false;\n r.motionBlurCleared = true;\n r.motionBlur = true;\n }\n if (motionBlur) {\n r.motionBlurTimeout = setTimeout(function () {\n r.motionBlurTimeout = null;\n r.clearedForMotionBlur[r.NODE] = false;\n r.clearedForMotionBlur[r.DRAG] = false;\n r.motionBlur = false;\n r.clearingMotionBlur = !textureDraw;\n r.mbFrames = 0;\n needDraw[r.NODE] = true;\n needDraw[r.DRAG] = true;\n r.redraw();\n }, motionBlurDelay);\n }\n if (!forcedContext) {\n cy.emit('render');\n }\n};\nvar fpsHeight;\nCRp$5.drawSelectionRectangle = function (options, setContextTransform) {\n var r = this;\n var cy = r.cy;\n var data = r.data;\n var style = cy.style();\n var drawOnlyNodeLayer = options.drawOnlyNodeLayer;\n var drawAllLayers = options.drawAllLayers;\n var needDraw = data.canvasNeedsRedraw;\n var forcedContext = options.forcedContext;\n if (r.showFps || !drawOnlyNodeLayer && needDraw[r.SELECT_BOX] && !drawAllLayers) {\n var context = forcedContext || data.contexts[r.SELECT_BOX];\n setContextTransform(context);\n if (r.selection[4] == 1 && (r.hoverData.selecting || r.touchData.selecting)) {\n var zoom = r.cy.zoom();\n var borderWidth = style.core('selection-box-border-width').value / zoom;\n context.lineWidth = borderWidth;\n context.fillStyle = 'rgba(' + style.core('selection-box-color').value[0] + ',' + style.core('selection-box-color').value[1] + ',' + style.core('selection-box-color').value[2] + ',' + style.core('selection-box-opacity').value + ')';\n context.fillRect(r.selection[0], r.selection[1], r.selection[2] - r.selection[0], r.selection[3] - r.selection[1]);\n if (borderWidth > 0) {\n context.strokeStyle = 'rgba(' + style.core('selection-box-border-color').value[0] + ',' + style.core('selection-box-border-color').value[1] + ',' + style.core('selection-box-border-color').value[2] + ',' + style.core('selection-box-opacity').value + ')';\n context.strokeRect(r.selection[0], r.selection[1], r.selection[2] - r.selection[0], r.selection[3] - r.selection[1]);\n }\n }\n if (data.bgActivePosistion && !r.hoverData.selecting) {\n var zoom = r.cy.zoom();\n var pos = data.bgActivePosistion;\n context.fillStyle = 'rgba(' + style.core('active-bg-color').value[0] + ',' + style.core('active-bg-color').value[1] + ',' + style.core('active-bg-color').value[2] + ',' + style.core('active-bg-opacity').value + ')';\n context.beginPath();\n context.arc(pos.x, pos.y, style.core('active-bg-size').pfValue / zoom, 0, 2 * Math.PI);\n context.fill();\n }\n var timeToRender = r.lastRedrawTime;\n if (r.showFps && timeToRender) {\n timeToRender = Math.round(timeToRender);\n var fps = Math.round(1000 / timeToRender);\n var text = '1 frame = ' + timeToRender + ' ms = ' + fps + ' fps';\n context.setTransform(1, 0, 0, 1, 0, 0);\n context.fillStyle = 'rgba(255, 0, 0, 0.75)';\n context.strokeStyle = 'rgba(255, 0, 0, 0.75)';\n // context.lineWidth = 1;\n context.font = '30px Arial';\n if (!fpsHeight) {\n var dims = context.measureText(text);\n fpsHeight = dims.actualBoundingBoxAscent;\n }\n context.fillText(text, 0, fpsHeight);\n var maxFps = 60;\n context.strokeRect(0, fpsHeight + 10, 250, 20);\n context.fillRect(0, fpsHeight + 10, 250 * Math.min(fps / maxFps, 1), 20);\n }\n if (!drawAllLayers) {\n needDraw[r.SELECT_BOX] = false;\n }\n }\n};\n\n/**\n * Notes:\n * - All colors have premultiplied alpha. Very important for textues and \n * blending to work correctly.\n */\n\nfunction compileShader(gl, type, source) {\n var shader = gl.createShader(type);\n gl.shaderSource(shader, source);\n gl.compileShader(shader);\n if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) {\n throw new Error(gl.getShaderInfoLog(shader));\n }\n // console.log(gl.getShaderInfoLog(shader));\n return shader;\n}\nfunction createProgram(gl, vertexSource, fragementSource) {\n var vertexShader = compileShader(gl, gl.VERTEX_SHADER, vertexSource);\n var fragmentShader = compileShader(gl, gl.FRAGMENT_SHADER, fragementSource);\n var program = gl.createProgram();\n gl.attachShader(program, vertexShader);\n gl.attachShader(program, fragmentShader);\n gl.linkProgram(program);\n if (!gl.getProgramParameter(program, gl.LINK_STATUS)) {\n throw new Error('Could not initialize shaders');\n }\n return program;\n}\n\n/**\n * Creates an offscren canvas with a 2D context, for the\n * canvas renderer to use for drawing textures.\n */\nfunction createTextureCanvas(r, width, height) {\n if (height === undefined) {\n height = width;\n }\n var canvas = r.makeOffscreenCanvas(width, height);\n var ctx = canvas.context = canvas.getContext('2d');\n canvas.clear = function () {\n return ctx.clearRect(0, 0, canvas.width, canvas.height);\n };\n canvas.clear();\n return canvas;\n}\n\n/**\n * Returns the current pan & zoom values, scaled by the pixel ratio.\n */\nfunction getEffectivePanZoom(r) {\n var pixelRatio = r.pixelRatio;\n var zoom = r.cy.zoom();\n var pan = r.cy.pan();\n return {\n zoom: zoom * pixelRatio,\n pan: {\n x: pan.x * pixelRatio,\n y: pan.y * pixelRatio\n }\n };\n}\n\n/**\n * Returns the zoom value, scaled by the pixel ratio.\n */\nfunction getEffectiveZoom(r) {\n var pixelRatio = r.pixelRatio;\n var zoom = r.cy.zoom();\n return zoom * pixelRatio;\n}\nfunction modelToRenderedPosition(r, pan, zoom, x, y) {\n var rx = x * zoom + pan.x;\n var ry = y * zoom + pan.y;\n ry = Math.round(r.canvasHeight - ry); // adjust for webgl\n return [rx, ry];\n}\nfunction isSimpleShape(node, renderTarget) {\n // the actual shape is checked in ElementDrawingWebGL._getVertTypeForShape()\n // no need to check it twice, this just checks other visual properties\n if (renderTarget.picking) {\n // We don't care about the border or background style for picking\n return true;\n } else {\n if (node.pstyle('background-fill').value !== 'solid') return false;\n if (node.pstyle('background-image').strValue !== 'none') return false;\n if (node.pstyle('border-width').value === 0) return true;\n if (node.pstyle('border-opacity').value === 0) return true;\n // we have a border but it must be simple\n if (node.pstyle('border-style').value !== 'solid') return false;\n // TODO ignoring 'border-cap', 'border-join' and 'border-position' for now\n return true;\n }\n}\nfunction arrayEqual(a1, a2) {\n if (a1.length !== a2.length) {\n return false;\n }\n for (var i = 0; i < a1.length; i++) {\n if (a1[i] !== a2[i]) {\n return false;\n }\n }\n return true;\n}\n\n/**\n * Takes color & opacity style values and converts them to WebGL format. \n * Alpha is premultiplied.\n */\nfunction toWebGLColor(color, opacity, outArray) {\n var r = color[0] / 255;\n var g = color[1] / 255;\n var b = color[2] / 255;\n var a = opacity;\n var arr = outArray || new Array(4);\n arr[0] = r * a;\n arr[1] = g * a;\n arr[2] = b * a;\n arr[3] = a;\n return arr;\n}\nfunction indexToVec4(index, outArray) {\n var arr = outArray || new Array(4);\n arr[0] = (index >> 0 & 0xFF) / 0xFF;\n arr[1] = (index >> 8 & 0xFF) / 0xFF;\n arr[2] = (index >> 16 & 0xFF) / 0xFF;\n arr[3] = (index >> 24 & 0xFF) / 0xFF;\n return arr;\n}\nfunction vec4ToIndex(vec4) {\n return vec4[0] + (vec4[1] << 8) + (vec4[2] << 16) + (vec4[3] << 24);\n}\nfunction createTexture(gl, debugID) {\n var texture = gl.createTexture();\n texture.buffer = function (offscreenCanvas) {\n gl.bindTexture(gl.TEXTURE_2D, texture);\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR_MIPMAP_NEAREST);\n\n // very important, this tells webgl to premultiply colors by the alpha channel\n gl.pixelStorei(gl.UNPACK_PREMULTIPLY_ALPHA_WEBGL, true);\n gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, offscreenCanvas);\n gl.generateMipmap(gl.TEXTURE_2D);\n gl.bindTexture(gl.TEXTURE_2D, null);\n };\n texture.deleteTexture = function () {\n gl.deleteTexture(texture);\n };\n return texture;\n}\nfunction getTypeInfo(gl, glslType) {\n switch (glslType) {\n case 'float':\n return [1, gl.FLOAT, 4];\n case 'vec2':\n return [2, gl.FLOAT, 4];\n case 'vec3':\n return [3, gl.FLOAT, 4];\n case 'vec4':\n return [4, gl.FLOAT, 4];\n case 'int':\n return [1, gl.INT, 4];\n case 'ivec2':\n return [2, gl.INT, 4];\n }\n}\nfunction createTypedArray(gl, glType, dataOrSize) {\n switch (glType) {\n case gl.FLOAT:\n return new Float32Array(dataOrSize);\n case gl.INT:\n return new Int32Array(dataOrSize);\n }\n}\nfunction createTypedArrayView(gl, glType, array, stride, size, i) {\n switch (glType) {\n case gl.FLOAT:\n return new Float32Array(array.buffer, i * stride, size);\n case gl.INT:\n return new Int32Array(array.buffer, i * stride, size);\n }\n}\n\n/** @param {WebGLRenderingContext} gl */\nfunction createBufferStaticDraw(gl, type, attributeLoc, dataArray) {\n var _getTypeInfo = getTypeInfo(gl, type),\n _getTypeInfo2 = _slicedToArray(_getTypeInfo, 2),\n size = _getTypeInfo2[0],\n glType = _getTypeInfo2[1];\n var data = createTypedArray(gl, glType, dataArray);\n var buffer = gl.createBuffer();\n gl.bindBuffer(gl.ARRAY_BUFFER, buffer);\n gl.bufferData(gl.ARRAY_BUFFER, data, gl.STATIC_DRAW);\n if (glType === gl.FLOAT) {\n gl.vertexAttribPointer(attributeLoc, size, glType, false, 0, 0);\n } else if (glType === gl.INT) {\n gl.vertexAttribIPointer(attributeLoc, size, glType, 0, 0);\n }\n gl.enableVertexAttribArray(attributeLoc);\n gl.bindBuffer(gl.ARRAY_BUFFER, null);\n return buffer;\n}\n\n/** \n * Creates a float buffer with gl.DYNAMIC_DRAW.\n * The returned buffer object contains functions to easily set instance data and buffer the data before a draw call.\n * @param {WebGLRenderingContext} gl \n */\nfunction createBufferDynamicDraw(gl, instances, type, attributeLoc) {\n var _getTypeInfo3 = getTypeInfo(gl, type),\n _getTypeInfo4 = _slicedToArray(_getTypeInfo3, 3),\n size = _getTypeInfo4[0],\n glType = _getTypeInfo4[1],\n bytes = _getTypeInfo4[2];\n var dataArray = createTypedArray(gl, glType, instances * size);\n var stride = size * bytes;\n var buffer = gl.createBuffer();\n gl.bindBuffer(gl.ARRAY_BUFFER, buffer);\n gl.bufferData(gl.ARRAY_BUFFER, instances * stride, gl.DYNAMIC_DRAW);\n gl.enableVertexAttribArray(attributeLoc);\n if (glType === gl.FLOAT) {\n gl.vertexAttribPointer(attributeLoc, size, glType, false, stride, 0);\n } else if (glType === gl.INT) {\n gl.vertexAttribIPointer(attributeLoc, size, glType, stride, 0);\n }\n gl.vertexAttribDivisor(attributeLoc, 1);\n gl.bindBuffer(gl.ARRAY_BUFFER, null);\n\n // use array views to set values directly into the buffer array\n var views = new Array(instances);\n for (var i = 0; i < instances; i++) {\n views[i] = createTypedArrayView(gl, glType, dataArray, stride, size, i);\n }\n buffer.dataArray = dataArray;\n buffer.stride = stride;\n buffer.size = size;\n buffer.getView = function (i) {\n return views[i];\n };\n buffer.setPoint = function (i, x, y) {\n var view = views[i];\n view[0] = x;\n view[1] = y;\n };\n buffer.bufferSubData = function (count) {\n gl.bindBuffer(gl.ARRAY_BUFFER, buffer);\n if (count) {\n gl.bufferSubData(gl.ARRAY_BUFFER, 0, dataArray, 0, count * size);\n } else {\n gl.bufferSubData(gl.ARRAY_BUFFER, 0, dataArray);\n }\n };\n return buffer;\n}\n\n/** \n * Creates a buffer of 3x3 matrix data for use as attribute data.\n * @param {WebGLRenderingContext} gl \n */\nfunction create3x3MatrixBufferDynamicDraw(gl, instances, attributeLoc) {\n var matrixSize = 9; // 3x3 matrix\n var matrixData = new Float32Array(instances * matrixSize);\n\n // use matrix views to set values directly into the matrixData array\n var matrixViews = new Array(instances);\n for (var i = 0; i < instances; i++) {\n var byteOffset = i * matrixSize * 4; // 4 bytes per float\n matrixViews[i] = new Float32Array(matrixData.buffer, byteOffset, matrixSize); // array view\n }\n var buffer = gl.createBuffer();\n gl.bindBuffer(gl.ARRAY_BUFFER, buffer);\n gl.bufferData(gl.ARRAY_BUFFER, matrixData.byteLength, gl.DYNAMIC_DRAW);\n\n // each row of the matrix needs to be a separate attribute\n for (var _i = 0; _i < 3; _i++) {\n var loc = attributeLoc + _i;\n gl.enableVertexAttribArray(loc);\n gl.vertexAttribPointer(loc, 3, gl.FLOAT, false, 3 * 12, _i * 12);\n gl.vertexAttribDivisor(loc, 1);\n }\n gl.bindBuffer(gl.ARRAY_BUFFER, null);\n buffer.getMatrixView = function (i) {\n return matrixViews[i];\n };\n\n // TODO this is too slow, use getMatrixView and pass the view directly to the glmatrix library\n buffer.setData = function (matrix, i) {\n matrixViews[i].set(matrix, 0);\n };\n buffer.bufferSubData = function () {\n gl.bindBuffer(gl.ARRAY_BUFFER, buffer);\n gl.bufferSubData(gl.ARRAY_BUFFER, 0, matrixData);\n };\n return buffer;\n}\n\n/** \n * Creates a Frame Buffer to use for offscreen rendering.\n * @param {WebGLRenderingContext} gl \n */\nfunction createPickingFrameBuffer(gl) {\n // Create and bind the framebuffer\n var fb = gl.createFramebuffer();\n gl.bindFramebuffer(gl.FRAMEBUFFER, fb);\n\n // Create a texture to render to\n var targetTexture = gl.createTexture();\n gl.bindTexture(gl.TEXTURE_2D, targetTexture);\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);\n gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);\n\n // attach the texture as the first color attachment\n gl.framebufferTexture2D(gl.FRAMEBUFFER, gl.COLOR_ATTACHMENT0, gl.TEXTURE_2D, targetTexture, 0);\n gl.bindFramebuffer(gl.FRAMEBUFFER, null);\n fb.setFramebufferAttachmentSizes = function (width, height) {\n gl.bindTexture(gl.TEXTURE_2D, targetTexture);\n gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, width, height, 0, gl.RGBA, gl.UNSIGNED_BYTE, null);\n };\n return fb;\n}\n\n/**\n * Common utilities\n * @module glMatrix\n */\n// Configuration Constants\nvar ARRAY_TYPE = typeof Float32Array !== 'undefined' ? Float32Array : Array;\nif (!Math.hypot) Math.hypot = function () {\n var y = 0,\n i = arguments.length;\n\n while (i--) {\n y += arguments[i] * arguments[i];\n }\n\n return Math.sqrt(y);\n};\n\n/**\n * 3x3 Matrix\n * @module mat3\n */\n\n/**\n * Creates a new identity mat3\n *\n * @returns {mat3} a new 3x3 matrix\n */\n\nfunction create() {\n var out = new ARRAY_TYPE(9);\n\n if (ARRAY_TYPE != Float32Array) {\n out[1] = 0;\n out[2] = 0;\n out[3] = 0;\n out[5] = 0;\n out[6] = 0;\n out[7] = 0;\n }\n\n out[0] = 1;\n out[4] = 1;\n out[8] = 1;\n return out;\n}\n/**\n * Set a mat3 to the identity matrix\n *\n * @param {mat3} out the receiving matrix\n * @returns {mat3} out\n */\n\nfunction identity(out) {\n out[0] = 1;\n out[1] = 0;\n out[2] = 0;\n out[3] = 0;\n out[4] = 1;\n out[5] = 0;\n out[6] = 0;\n out[7] = 0;\n out[8] = 1;\n return out;\n}\n/**\n * Multiplies two mat3's\n *\n * @param {mat3} out the receiving matrix\n * @param {ReadonlyMat3} a the first operand\n * @param {ReadonlyMat3} b the second operand\n * @returns {mat3} out\n */\n\nfunction multiply(out, a, b) {\n var a00 = a[0],\n a01 = a[1],\n a02 = a[2];\n var a10 = a[3],\n a11 = a[4],\n a12 = a[5];\n var a20 = a[6],\n a21 = a[7],\n a22 = a[8];\n var b00 = b[0],\n b01 = b[1],\n b02 = b[2];\n var b10 = b[3],\n b11 = b[4],\n b12 = b[5];\n var b20 = b[6],\n b21 = b[7],\n b22 = b[8];\n out[0] = b00 * a00 + b01 * a10 + b02 * a20;\n out[1] = b00 * a01 + b01 * a11 + b02 * a21;\n out[2] = b00 * a02 + b01 * a12 + b02 * a22;\n out[3] = b10 * a00 + b11 * a10 + b12 * a20;\n out[4] = b10 * a01 + b11 * a11 + b12 * a21;\n out[5] = b10 * a02 + b11 * a12 + b12 * a22;\n out[6] = b20 * a00 + b21 * a10 + b22 * a20;\n out[7] = b20 * a01 + b21 * a11 + b22 * a21;\n out[8] = b20 * a02 + b21 * a12 + b22 * a22;\n return out;\n}\n/**\n * Translate a mat3 by the given vector\n *\n * @param {mat3} out the receiving matrix\n * @param {ReadonlyMat3} a the matrix to translate\n * @param {ReadonlyVec2} v vector to translate by\n * @returns {mat3} out\n */\n\nfunction translate(out, a, v) {\n var a00 = a[0],\n a01 = a[1],\n a02 = a[2],\n a10 = a[3],\n a11 = a[4],\n a12 = a[5],\n a20 = a[6],\n a21 = a[7],\n a22 = a[8],\n x = v[0],\n y = v[1];\n out[0] = a00;\n out[1] = a01;\n out[2] = a02;\n out[3] = a10;\n out[4] = a11;\n out[5] = a12;\n out[6] = x * a00 + y * a10 + a20;\n out[7] = x * a01 + y * a11 + a21;\n out[8] = x * a02 + y * a12 + a22;\n return out;\n}\n/**\n * Rotates a mat3 by the given angle\n *\n * @param {mat3} out the receiving matrix\n * @param {ReadonlyMat3} a the matrix to rotate\n * @param {Number} rad the angle to rotate the matrix by\n * @returns {mat3} out\n */\n\nfunction rotate(out, a, rad) {\n var a00 = a[0],\n a01 = a[1],\n a02 = a[2],\n a10 = a[3],\n a11 = a[4],\n a12 = a[5],\n a20 = a[6],\n a21 = a[7],\n a22 = a[8],\n s = Math.sin(rad),\n c = Math.cos(rad);\n out[0] = c * a00 + s * a10;\n out[1] = c * a01 + s * a11;\n out[2] = c * a02 + s * a12;\n out[3] = c * a10 - s * a00;\n out[4] = c * a11 - s * a01;\n out[5] = c * a12 - s * a02;\n out[6] = a20;\n out[7] = a21;\n out[8] = a22;\n return out;\n}\n/**\n * Scales the mat3 by the dimensions in the given vec2\n *\n * @param {mat3} out the receiving matrix\n * @param {ReadonlyMat3} a the matrix to rotate\n * @param {ReadonlyVec2} v the vec2 to scale the matrix by\n * @returns {mat3} out\n **/\n\nfunction scale(out, a, v) {\n var x = v[0],\n y = v[1];\n out[0] = x * a[0];\n out[1] = x * a[1];\n out[2] = x * a[2];\n out[3] = y * a[3];\n out[4] = y * a[4];\n out[5] = y * a[5];\n out[6] = a[6];\n out[7] = a[7];\n out[8] = a[8];\n return out;\n}\n/**\n * Generates a 2D projection matrix with the given bounds\n *\n * @param {mat3} out mat3 frustum matrix will be written into\n * @param {number} width Width of your gl context\n * @param {number} height Height of gl context\n * @returns {mat3} out\n */\n\nfunction projection(out, width, height) {\n out[0] = 2 / width;\n out[1] = 0;\n out[2] = 0;\n out[3] = 0;\n out[4] = -2 / height;\n out[5] = 0;\n out[6] = -1;\n out[7] = 1;\n out[8] = 1;\n return out;\n}\n\n// A \"texture atlas\" is a big canvas, and sections of it are used as textures for nodes/labels.\n\n/**\n * A single square texture atlas (also known as a \"sprite sheet\").\n */\nvar Atlas = /*#__PURE__*/function () {\n function Atlas(r, texSize, texRows, createTextureCanvas) {\n _classCallCheck(this, Atlas);\n this.debugID = Math.floor(Math.random() * 10000);\n this.r = r;\n this.texSize = texSize;\n this.texRows = texRows;\n this.texHeight = Math.floor(texSize / texRows);\n this.enableWrapping = true; // hardcoded for now, can be made an option\n\n this.locked = false; // once an atlas is locked it can no longer be drawn to\n this.texture = null; // WebGLTexture object\n this.needsBuffer = true;\n\n // a \"location\" is an pointer into the atlas with a 'row' and 'x' fields\n this.freePointer = {\n x: 0,\n row: 0\n };\n\n // map from the style key to the row/x where the texture starts\n // if the texture wraps then there's a second location\n this.keyToLocation = new Map(); // styleKey -> [ location, location ]\n\n this.canvas = createTextureCanvas(r, texSize, texSize);\n this.scratch = createTextureCanvas(r, texSize, this.texHeight, 'scratch');\n }\n return _createClass(Atlas, [{\n key: \"lock\",\n value: function lock() {\n this.locked = true;\n }\n }, {\n key: \"getKeys\",\n value: function getKeys() {\n return new Set(this.keyToLocation.keys());\n }\n }, {\n key: \"getScale\",\n value: function getScale(_ref) {\n var w = _ref.w,\n h = _ref.h;\n var texHeight = this.texHeight,\n maxTexWidth = this.texSize;\n // try to fit to the height of a row\n var scale = texHeight / h; // TODO what about pixelRatio?\n var texW = w * scale;\n var texH = h * scale;\n // if the scaled width is too wide then scale to fit max width instead\n if (texW > maxTexWidth) {\n scale = maxTexWidth / w;\n texW = w * scale;\n texH = h * scale;\n }\n return {\n scale: scale,\n texW: texW,\n texH: texH\n };\n }\n }, {\n key: \"draw\",\n value: function draw(key, bb, doDrawing) {\n var _this = this;\n if (this.locked) throw new Error('can\\'t draw, atlas is locked');\n var texSize = this.texSize,\n texRows = this.texRows,\n texHeight = this.texHeight;\n var _this$getScale = this.getScale(bb),\n scale = _this$getScale.scale,\n texW = _this$getScale.texW,\n texH = _this$getScale.texH;\n var drawAt = function drawAt(location, canvas) {\n if (doDrawing && canvas) {\n var context = canvas.context;\n var x = location.x,\n row = location.row;\n var xOffset = x;\n var yOffset = texHeight * row;\n context.save();\n context.translate(xOffset, yOffset);\n context.scale(scale, scale);\n doDrawing(context, bb);\n context.restore();\n }\n };\n var locations = [null, null];\n var drawNormal = function drawNormal() {\n // don't need to wrap, draw directly on the canvas\n drawAt(_this.freePointer, _this.canvas);\n locations[0] = {\n x: _this.freePointer.x,\n y: _this.freePointer.row * texHeight,\n w: texW,\n h: texH\n };\n locations[1] = {\n // create a second location with a width of 0, for convenience\n x: _this.freePointer.x + texW,\n y: _this.freePointer.row * texHeight,\n w: 0,\n h: texH\n };\n\n // move the pointer to the end of the texture\n _this.freePointer.x += texW;\n if (_this.freePointer.x == texSize) {\n _this.freePointer.x = 0;\n _this.freePointer.row++;\n }\n };\n var drawWrapped = function drawWrapped() {\n var scratch = _this.scratch,\n canvas = _this.canvas;\n\n // Draw to the scratch canvas\n scratch.clear();\n drawAt({\n x: 0,\n row: 0\n }, scratch);\n var firstTexW = texSize - _this.freePointer.x;\n var secondTexW = texW - firstTexW;\n var h = texHeight;\n {\n // copy first part of scratch to the first texture\n var dx = _this.freePointer.x;\n var dy = _this.freePointer.row * texHeight;\n var w = firstTexW;\n canvas.context.drawImage(scratch, 0, 0, w, h, dx, dy, w, h);\n locations[0] = {\n x: dx,\n y: dy,\n w: w,\n h: texH\n };\n }\n {\n // copy second part of scratch to the second texture\n var sx = firstTexW;\n var _dy = (_this.freePointer.row + 1) * texHeight;\n var _w = secondTexW;\n if (canvas) {\n canvas.context.drawImage(scratch, sx, 0, _w, h, 0, _dy, _w, h);\n }\n locations[1] = {\n x: 0,\n y: _dy,\n w: _w,\n h: texH\n };\n }\n _this.freePointer.x = secondTexW;\n _this.freePointer.row++;\n };\n var moveToStartOfNextRow = function moveToStartOfNextRow() {\n _this.freePointer.x = 0;\n _this.freePointer.row++;\n };\n if (this.freePointer.x + texW <= texSize) {\n // There's enough space in the current row\n drawNormal();\n } else if (this.freePointer.row >= texRows - 1) {\n // Need to move to the next row, but there are no more rows, atlas is full.\n return false;\n } else if (this.freePointer.x === texSize) {\n // happen to be right at end of current row\n moveToStartOfNextRow();\n drawNormal();\n } else if (this.enableWrapping) {\n // draw part of the texture to the end of the curent row, then wrap to the next row\n drawWrapped();\n } else {\n // move to the start of the next row, then draw normally\n moveToStartOfNextRow();\n drawNormal();\n }\n this.keyToLocation.set(key, locations);\n this.needsBuffer = true;\n return locations;\n }\n }, {\n key: \"getOffsets\",\n value: function getOffsets(key) {\n return this.keyToLocation.get(key);\n }\n }, {\n key: \"isEmpty\",\n value: function isEmpty() {\n return this.freePointer.x === 0 && this.freePointer.row === 0;\n }\n }, {\n key: \"canFit\",\n value: function canFit(bb) {\n if (this.locked) return false;\n var texSize = this.texSize,\n texRows = this.texRows;\n var _this$getScale2 = this.getScale(bb),\n texW = _this$getScale2.texW;\n if (this.freePointer.x + texW > texSize) {\n // need to wrap\n return this.freePointer.row < texRows - 1; // return true if there's a row to wrap to\n }\n return true;\n }\n\n // called on every frame\n }, {\n key: \"bufferIfNeeded\",\n value: function bufferIfNeeded(gl) {\n if (!this.texture) {\n this.texture = createTexture(gl, this.debugID);\n }\n if (this.needsBuffer) {\n this.texture.buffer(this.canvas);\n this.needsBuffer = false;\n if (this.locked) {\n this.canvas = null;\n this.scratch = null;\n }\n }\n }\n }, {\n key: \"dispose\",\n value: function dispose() {\n if (this.texture) {\n this.texture.deleteTexture();\n this.texture = null;\n }\n this.canvas = null;\n this.scratch = null;\n this.locked = true;\n }\n }]);\n}();\n\n/**\n * A collection of texture atlases, all of the same \"render type\". \n * ('node-body' is an example of a render type.)\n * An AtlasCollection can also be notified when a texture is no longer needed, \n * and it can garbage collect the unused textures.\n */\nvar AtlasCollection = /*#__PURE__*/function () {\n function AtlasCollection(r, texSize, texRows, createTextureCanvas) {\n _classCallCheck(this, AtlasCollection);\n this.r = r;\n this.texSize = texSize;\n this.texRows = texRows;\n this.createTextureCanvas = createTextureCanvas;\n this.atlases = [];\n this.styleKeyToAtlas = new Map();\n this.markedKeys = new Set(); // marked for garbage collection\n }\n return _createClass(AtlasCollection, [{\n key: \"getKeys\",\n value: function getKeys() {\n return new Set(this.styleKeyToAtlas.keys());\n }\n }, {\n key: \"_createAtlas\",\n value: function _createAtlas() {\n var r = this.r,\n texSize = this.texSize,\n texRows = this.texRows,\n createTextureCanvas = this.createTextureCanvas;\n return new Atlas(r, texSize, texRows, createTextureCanvas);\n }\n }, {\n key: \"_getScratchCanvas\",\n value: function _getScratchCanvas() {\n if (!this.scratch) {\n var r = this.r,\n texSize = this.texSize,\n texRows = this.texRows,\n createTextureCanvas = this.createTextureCanvas;\n var texHeight = Math.floor(texSize / texRows);\n this.scratch = createTextureCanvas(r, texSize, texHeight, 'scratch');\n }\n return this.scratch;\n }\n }, {\n key: \"draw\",\n value: function draw(key, bb, doDrawing) {\n var atlas = this.styleKeyToAtlas.get(key);\n if (!atlas) {\n // check for space at the end of the last atlas\n atlas = this.atlases[this.atlases.length - 1];\n if (!atlas || !atlas.canFit(bb)) {\n if (atlas) atlas.lock();\n // create a new atlas\n atlas = this._createAtlas();\n this.atlases.push(atlas);\n }\n atlas.draw(key, bb, doDrawing);\n this.styleKeyToAtlas.set(key, atlas);\n }\n return atlas;\n }\n }, {\n key: \"getAtlas\",\n value: function getAtlas(key) {\n return this.styleKeyToAtlas.get(key);\n }\n }, {\n key: \"hasAtlas\",\n value: function hasAtlas(key) {\n return this.styleKeyToAtlas.has(key);\n }\n }, {\n key: \"markKeyForGC\",\n value: function markKeyForGC(key) {\n this.markedKeys.add(key);\n }\n }, {\n key: \"gc\",\n value: function gc() {\n var _this2 = this;\n var markedKeys = this.markedKeys;\n if (markedKeys.size === 0) {\n console.log('nothing to garbage collect');\n return;\n }\n var newAtlases = [];\n var newStyleKeyToAtlas = new Map();\n var newAtlas = null;\n var _iterator = _createForOfIteratorHelper(this.atlases),\n _step;\n try {\n var _loop = function _loop() {\n var atlas = _step.value;\n var keys = atlas.getKeys();\n var keysToCollect = intersection(markedKeys, keys);\n if (keysToCollect.size === 0) {\n // this atlas can still be used\n newAtlases.push(atlas);\n keys.forEach(function (k) {\n return newStyleKeyToAtlas.set(k, atlas);\n });\n return 1; // continue\n }\n if (!newAtlas) {\n newAtlas = _this2._createAtlas();\n newAtlases.push(newAtlas);\n }\n var _iterator2 = _createForOfIteratorHelper(keys),\n _step2;\n try {\n for (_iterator2.s(); !(_step2 = _iterator2.n()).done;) {\n var key = _step2.value;\n if (!keysToCollect.has(key)) {\n var _atlas$getOffsets = atlas.getOffsets(key),\n _atlas$getOffsets2 = _slicedToArray(_atlas$getOffsets, 2),\n s1 = _atlas$getOffsets2[0],\n s2 = _atlas$getOffsets2[1];\n if (!newAtlas.canFit({\n w: s1.w + s2.w,\n h: s1.h\n })) {\n newAtlas.lock();\n newAtlas = _this2._createAtlas();\n newAtlases.push(newAtlas);\n }\n if (atlas.canvas) {\n // if the texture can't be copied then it will have to be redrawn on the next frame\n _this2._copyTextureToNewAtlas(key, atlas, newAtlas);\n newStyleKeyToAtlas.set(key, newAtlas);\n }\n }\n }\n } catch (err) {\n _iterator2.e(err);\n } finally {\n _iterator2.f();\n }\n atlas.dispose();\n };\n for (_iterator.s(); !(_step = _iterator.n()).done;) {\n if (_loop()) continue;\n }\n } catch (err) {\n _iterator.e(err);\n } finally {\n _iterator.f();\n }\n this.atlases = newAtlases;\n this.styleKeyToAtlas = newStyleKeyToAtlas;\n this.markedKeys = new Set();\n }\n }, {\n key: \"_copyTextureToNewAtlas\",\n value: function _copyTextureToNewAtlas(key, oldAtlas, newAtlas) {\n var _oldAtlas$getOffsets = oldAtlas.getOffsets(key),\n _oldAtlas$getOffsets2 = _slicedToArray(_oldAtlas$getOffsets, 2),\n s1 = _oldAtlas$getOffsets2[0],\n s2 = _oldAtlas$getOffsets2[1];\n if (s2.w === 0) {\n // the texture does not wrap, draw directly to new atlas\n newAtlas.draw(key, s1, function (context) {\n context.drawImage(oldAtlas.canvas, s1.x, s1.y, s1.w, s1.h, 0, 0, s1.w, s1.h);\n });\n } else {\n // the texture wraps, first draw both parts to a scratch canvas\n var scratch = this._getScratchCanvas();\n scratch.clear();\n scratch.context.drawImage(oldAtlas.canvas, s1.x, s1.y, s1.w, s1.h, 0, 0, s1.w, s1.h);\n scratch.context.drawImage(oldAtlas.canvas, s2.x, s2.y, s2.w, s2.h, s1.w, 0, s2.w, s2.h);\n\n // now draw the scratch to the new atlas\n var w = s1.w + s2.w;\n var h = s1.h;\n newAtlas.draw(key, {\n w: w,\n h: h\n }, function (context) {\n context.drawImage(scratch, 0, 0, w, h, 0, 0, w, h // the destination context has already been translated to the correct position\n );\n });\n }\n }\n }, {\n key: \"getCounts\",\n value: function getCounts() {\n return {\n keyCount: this.styleKeyToAtlas.size,\n atlasCount: new Set(this.styleKeyToAtlas.values()).size\n };\n }\n }]);\n}();\nfunction intersection(set1, set2) {\n // TODO why no Set.intersection in node 16???\n if (set1.intersection) return set1.intersection(set2);else return new Set(_toConsumableArray(set1).filter(function (x) {\n return set2.has(x);\n }));\n}\n\n/**\n * Used to manage batches of Atlases for drawing nodes and labels.\n * Supports different types of AtlasCollections for different render types,\n * for example 'node-body' and 'node-label' would be different render types.\n * Render types are kept separate because they will likely need to be garbage collected\n * separately and its not entierly guaranteed that their style keys won't collide.\n */\nvar AtlasManager = /*#__PURE__*/function () {\n function AtlasManager(r, globalOptions) {\n _classCallCheck(this, AtlasManager);\n this.r = r;\n this.globalOptions = globalOptions;\n this.atlasSize = globalOptions.webglTexSize;\n this.maxAtlasesPerBatch = globalOptions.webglTexPerBatch;\n this.renderTypes = new Map(); // renderType:string -> renderTypeOptions\n this.collections = new Map(); // collectionName:string -> AtlasCollection\n\n this.typeAndIdToKey = new Map(); // [renderType,id] => Array + + + + + + + + + + + + + \ No newline at end of file diff --git a/ontology_platform/vendored/ontocast/docs/contributing.md b/ontology_platform/vendored/ontocast/docs/contributing.md new file mode 100644 index 0000000..c1514a6 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/contributing.md @@ -0,0 +1,81 @@ +# Contributing to Suthing + +We welcome contributions to Suthing! This document provides guidelines and instructions for contributing to the project. + +## Getting Started + +1. Fork the repository on GitHub +2. Clone your fork locally +3. Install the development dependencies: + ```bash + uv sync --all-groups --extra doc-processing + ``` +4. Install pre-commit hooks: + ```bash + pre-commit install + ``` + +## Development Workflow + +1. Create a new branch for your feature or bugfix: + ```bash + git checkout -b feature/your-feature-name + ``` + +2. Make your changes and ensure tests pass: + ```bash + pytest test + ``` + +3. Commit your changes with a descriptive message: + ```bash + git commit -m "Add feature: your feature description" + ``` + +4. Push your branch to your fork: + ```bash + git push origin feature/your-feature-name + ``` + +5. Create a Pull Request on GitHub + +## Code Style + +- Follow [PEP 8](https://www.python.org/dev/peps/pep-0008/) style guidelines +- Use type hints for all function parameters and return values +- Write docstrings following the Google style +- Keep functions focused and small +- Add tests for new features + +## Documentation + +- Update relevant documentation when adding new features +- Add docstrings to all new functions and classes +- Include examples in docstrings where appropriate +- Update the changelog for significant changes + +## Testing + +- Write tests for all new features +- Ensure all tests pass before submitting a PR +- Add tests for bug fixes +- Maintain or improve test coverage + +## Pull Request Process + +1. Ensure your PR description clearly describes the problem and solution +2. Include relevant tests +3. Update documentation as needed +4. Ensure all CI checks pass +5. Request review from maintainers + +## Reporting Issues + +When reporting issues, please include: + +- Python version +- Suthing version +- Steps to reproduce +- Expected behavior +- Actual behavior +- Any relevant error messages diff --git a/ontology_platform/vendored/ontocast/docs/gen_pages.py b/ontology_platform/vendored/ontocast/docs/gen_pages.py new file mode 100644 index 0000000..929834d --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/gen_pages.py @@ -0,0 +1,26 @@ +from pathlib import Path + +import mkdocs_gen_files + +nav = mkdocs_gen_files.Nav() +pname = "ontocast" + +for path in sorted(Path(pname).rglob("*.py")): + module_path = path.relative_to(pname).with_suffix("") + doc_path = path.relative_to(pname).with_suffix(".md") + full_doc_path = Path("reference", doc_path) + + parts = list(module_path.parts) + + if parts[-1] == "__init__": + parts = parts[:-1] + if not parts: + continue + parts_str: tuple[str, ...] = tuple(parts) + nav[parts_str] = str(full_doc_path) + + with mkdocs_gen_files.open(full_doc_path, "w") as f: + ident = ".".join([pname] + parts) + f.write(f"# `{ident}`\n\n::: {ident}\n") + + mkdocs_gen_files.set_edit_path(full_doc_path, path) diff --git a/ontology_platform/vendored/ontocast/docs/getting_started/installation.md b/ontology_platform/vendored/ontocast/docs/getting_started/installation.md new file mode 100644 index 0000000..4c0fc41 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/getting_started/installation.md @@ -0,0 +1,28 @@ +# Installation + +This guide will help you install OntoCast and its dependencies. + +## System Requirements + +- Python 3.12 or higher +- uv (Python package installer) + +## Installation Steps + +```bash +uv add ontocast +``` + +or + + +```bash +pip install ontocast +``` + +## Next Steps + +After installation, you can: + +1. Read the [Quick Start](quickstart.md) guide +3. Check the [API Reference](../reference/onto.md) for detailed documentation \ No newline at end of file diff --git a/ontology_platform/vendored/ontocast/docs/getting_started/quickstart.md b/ontology_platform/vendored/ontocast/docs/getting_started/quickstart.md new file mode 100644 index 0000000..47f62bd --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/getting_started/quickstart.md @@ -0,0 +1,169 @@ +# Quick Start + +This guide will help you get started with OntoCast quickly. We'll walk through a simple example of processing a document and viewing the results. + +## Prerequisites + +- OntoCast installed (see [Installation](installation.md)) +- A sample document to process (e.g., a pdf or a markdown file) + +## Basic Example + +### Query the Server + +```bash +curl -X POST http://url:port/process -F "file=@sample.pdf" + +curl -X POST http://url:port/process -F "file=@sample.json" +``` + +`url` would be `localhost` for a locally running server, default port is 8999 + +### Running a Server + +To start an OntoCast server: + +```bash +# Backend automatically detected from .env configuration +ontocast --env-path .env + +# Process specific file +ontocast --env-path .env --input-path ./document.pdf + +# Process with chunk limit (for testing) +ontocast --env-path .env --head-chunks 5 +``` + +- Backend selection is **fully automatic** based on available configuration +- No explicit backend flags needed - just provide the required credentials/paths in .env +- All paths and directories are configured via .env file + +### Configuration + +OntoCast uses a hierarchical configuration system with environment variables. Create a `.env` file in your project directory: + +```bash +# Domain configuration (used for URI generation) +CURRENT_DOMAIN=https://example.com +PORT=8999 +LLM_TEMPERATURE=0.0 + +# LLM Configuration +LLM_PROVIDER=openai +LLM_API_KEY=your-api-key-here +LLM_MODEL_NAME=gpt-4o-mini + +# Server Configuration +MAX_VISITS=3 +BASE_RECURSION_LIMIT=1000 +ESTIMATED_CHUNKS=30 +RENDER_MODE=ontology_and_facts +ONTOLOGY_MAX_TRIPLES=50000 +PARALLEL_WORKERS=4 +PARALLEL_FACTS_RETRIES=3 +PARALLEL_ONTOLOGY_RETRIES=3 +ENABLE_ONTOLOGY_CONSOLIDATION=false + +# Backend Configuration (auto-detected) +FUSEKI_URI=http://localhost:3032/test +FUSEKI_AUTH=admin:password +ONTOCAST_WORKING_DIRECTORY=/path/to/working + +# Path Configuration (required for filesystem backends) +ONTOCAST_WORKING_DIRECTORY=/path/to/working/directory +ONTOCAST_ONTOLOGY_DIRECTORY=/path/to/ontology/files +ONTOCAST_CACHE_DIR=/path/to/cache/directory + +# Triple Store Configuration (optional) +# For Neo4j +NEO4J_URI=bolt://localhost:7687 +NEO4J_AUTH=username:password + +# For Fuseki +FUSEKI_URI=http://localhost:3030 +FUSEKI_AUTH=username:password +FUSEKI_DATASET=dataset_name +FUSEKI_ONTOLOGIES_DATASET=ontologies + +# Optional aggregation controls +AGG_EMBEDDING_MODEL=paraphrase-multilingual-MiniLM-L12-v2 +AGG_SIMILARITY_THRESHOLD=0.80 + +# Optional web-search grounding +WEB_SEARCH_ENABLED=false +WEB_SEARCH_PROVIDER=duckduckgo +WEB_SEARCH_TOP_K=3 +``` + +#### Alternative: Ollama Configuration + +```bash +# For Ollama +LLM_PROVIDER=ollama +LLM_BASE_URL=http://localhost:11434 +LLM_MODEL_NAME=granite3.3 +``` + +### CLI Parameters + +You can use these CLI parameters: + +```bash +# Use custom .env file +ontocast --env-path /path/to/custom.env + +# Process specific input file +ontocast --env-path .env --input-path /path/to/document.pdf + +# Process only first 5 chunks (for testing) +ontocast --env-path .env --head-chunks 5 +``` + +**Note:** All paths and directories are configured via the `.env` file - no CLI overrides needed. + +### Receive Results + +After processing, the ontology and the facts graph are returned in turtle format + +```json +{ + "data": { + "facts": "# facts in turtle format", + "ontology": "# ontology in turtle format" + } + ... +} +``` + +## Configuration System + +OntoCast uses a hierarchical configuration system: + +- **ToolConfig**: Configuration for tools (LLM, triple stores, paths) +- **ServerConfig**: Configuration for server behavior +- **Environment Variables**: Override defaults via `.env` file or environment + +### Key Environment Variables + +| Variable | Description | Default | +|----------|-------------|---------| +| `LLM_API_KEY` | API key for LLM provider | Required | +| `LLM_PROVIDER` | LLM provider (openai, ollama) | openai | +| `LLM_MODEL_NAME` | Model name | gpt-4o-mini | +| `FUSEKI_URI` + `FUSEKI_AUTH` | Use Fuseki as main triple store | Auto-detected | +| `NEO4J_URI` + `NEO4J_AUTH` | Use Neo4j as main triple store | Auto-detected | +| `ONTOCAST_WORKING_DIRECTORY` + `ONTOCAST_ONTOLOGY_DIRECTORY` | Use filesystem as main triple store | Auto-detected | +| `ONTOCAST_ONTOLOGY_DIRECTORY` | Ontology files directory | Provide seed ontologies | +| `MAX_VISITS` | Maximum visits per node | 3 | +| `BASE_RECURSION_LIMIT` | Base recursion limit for workflow | 1000 | +| `ONTOLOGY_MAX_TRIPLES` | Maximum triples allowed in ontology graph | 50000 | +| `ENABLE_ONTOLOGY_CONSOLIDATION` | Run ontology consolidation pass | false | + +## Next Steps + +Now that you've processed your first document, you can: + +1. Try processing different types of documents (PDF, Word) +2. Configure triple stores (Neo4j, Fuseki) for persistent storage +3. Check the [API Reference](../reference/onto.md) for more details +4. Explore the [User Guide](../user_guide/concepts.md) for advanced usage diff --git a/ontology_platform/vendored/ontocast/docs/index.md b/ontology_platform/vendored/ontocast/docs/index.md new file mode 100644 index 0000000..0378dc8 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/index.md @@ -0,0 +1,279 @@ +# OntoCast Agentic Ontology Triplecast logo + +### Agentic ontology-assisted framework for semantic triple extraction + +![Python](https://img.shields.io/badge/python-3.12-blue.svg) +[![PyPI version](https://badge.fury.io/py/ontocast.svg)](https://badge.fury.io/py/ontocast) +[![PyPI Downloads](https://static.pepy.tech/badge/ontocast)](https://pepy.tech/projects/ontocast) +[![License](https://img.shields.io/badge/License-Apache_2.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) +[![pre-commit](https://github.com/growgraph/ontocast/actions/workflows/pre-commit.yml/badge.svg)](https://github.com/growgraph/ontocast/actions/workflows/pre-commit.yml) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.17796467.svg)](https://doi.org/10.5281/zenodo.17796467) + +--- + +## Overview + +OntoCast is a framework for extracting semantic triples (creating a Knowledge Graph) from documents using an agentic, ontology-driven approach. It combines ontology management, natural language processing, and knowledge graph serialization to turn unstructured text into structured, queryable data. + +--- + +## Key Features + +- **Ontology-Guided Extraction**: Ensures semantic consistency and co-evolves ontologies +- **Entity Disambiguation**: Resolves references across document chunks +- **Multi-Format Support**: Handles text, JSON, PDF, and Markdown +- **Semantic Chunking**: Splits text based on semantic similarity +- **MCP Compatibility**: Implements Model Control Protocol endpoints +- **RDF Output**: Produces standardized RDF/Turtle +- **Triple Store Integration**: Supports Neo4j (n10s) and Apache Fuseki +- **Automatic LLM Caching**: Built-in response caching for improved performance and cost reduction +- **GraphUpdate Operations**: Token-efficient SPARQL-based updates instead of full graph regeneration +- **Budget Tracking**: Comprehensive tracking of LLM usage and triple generation metrics +- **Ontology Versioning**: Automatic semantic versioning with hash-based lineage tracking + +--- + +## Applications + +OntoCast can be used for: + +- **Knowledge Graph Construction**: Build domain-specific or general-purpose knowledge graphs from documents +- **Semantic Search**: Power search and retrieval with structured triples +- **GraphRAG**: Enable retrieval-augmented generation over knowledge graphs (e.g., with LLMs) +- **Ontology Management**: Automate ontology creation, validation, and refinement +- **Data Integration**: Unify data from diverse sources into a semantic graph + +--- + +## Installation + +```sh +uv add ontocast +# or +pip install ontocast +``` + +--- + +## Configuration + +## Documentation + +- [Quick Start Guide](getting_started/quickstart.md) - Get started quickly +- [Configuration System](user_guide/configuration.md) - Detailed configuration guide +- [LLM Caching](user_guide/llm_caching.md) - Automatic response caching +- [Triple Store Setup](user_guide/triple_stores.md) - Triple store configuration +- [User Guide](user_guide/concepts.md) - Core concepts and workflow +- [API Reference](reference/onto.md) - Detailed API documentation + + +### Environment Variables + +Copy the example file and edit as needed: + +```bash +cp .env.example .env +# Edit with your values +``` + +**Main options:** +```bash +# LLM Configuration +# common +LLM_PROVIDER=openai # or ollama +LLM_MODEL_NAME=gpt-4o-mini # ollama model +LLM_TEMPERATURE=0.0 + +# openai +LLM_API_KEY=your_openai_api_key_here + +# ollama +LLM_BASE_URL= + +# Server +PORT=8999 +BASE_RECURSION_LIMIT=1000 +ESTIMATED_CHUNKS=30 +MAX_VISITS=3 +RENDER_MODE=ontology_and_facts +ONTOLOGY_MAX_TRIPLES=50000 +PARALLEL_WORKERS=4 +PARALLEL_FACTS_RETRIES=3 +PARALLEL_ONTOLOGY_RETRIES=3 +ENABLE_ONTOLOGY_CONSOLIDATION=false + +# Backend Configuration (auto-detected) +FUSEKI_URI=http://localhost:3032/test +FUSEKI_AUTH=admin:password +ONTOCAST_WORKING_DIRECTORY=/path/to/working + +# Optional: Triple Store Configuration (Fuseki preferred over Neo4j) +FUSEKI_URI=http://localhost:3032/test +FUSEKI_AUTH=admin/abc123-qwe +FUSEKI_DATASET=dataset_name +FUSEKI_ONTOLOGIES_DATASET=ontologies + +NEO4J_URI=bolt://localhost:7689 +NEO4J_AUTH=neo4j/test!passfortesting + +# Aggregation controls +AGG_EMBEDDING_MODEL=paraphrase-multilingual-MiniLM-L12-v2 +AGG_SIMILARITY_THRESHOLD=0.80 + +# Optional web grounding +WEB_SEARCH_ENABLED=false +WEB_SEARCH_PROVIDER=duckduckgo +``` + +--- + +## Triple Store Setup + +OntoCast supports multiple triple store backends. When both Fuseki and Neo4j are configured, **Fuseki is preferred**. + +- See [Triple Store Setup](user_guide/triple_stores.md) for detailed Docker Compose instructions and sample `.env.example` files. +- Quick summary: copy and edit the provided `.env.example` in `docker/fuseki` or `docker/neo4j`, then run `docker compose --env-file .env up -d` in the respective directory. + +--- + +## Running OntoCast Server + +```bash +# Backend automatically detected from .env configuration +ontocast --env-path .env + +# Process specific file +ontocast --env-path .env --input-path ./document.pdf + +# Process with chunk limit (for testing) +ontocast --env-path .env --head-chunks 5 +``` + +- Backend selection is **fully automatic** based on available configuration +- No explicit backend flags needed - just provide the required credentials/paths in .env +- All paths and directories are configured via .env file + +--- + +## API Usage + +- **POST /process**: Accepts `application/json` or file uploads (`multipart/form-data`). +- Returns: JSON with extracted facts (Turtle), ontology (Turtle), and processing metadata. Triples are also serialized to the configured triple store. + +**Example:** +```bash +curl -X POST http://localhost:8999/process \ + -H "Content-Type: application/json" \ + -d '{"text": "Your document text here"}' + +# Process a PDF file +curl -X POST http://url:port/process -F "file=@data/pdf/sample.pdf" + +# Process a json file +curl -X POST http://url:port/process -F "file=@test2/sample.json" +``` + +--- + +## MCP Endpoints + +- `GET /health`: Health check +- `GET /info`: Service info +- `POST /process`: Document processing +- `POST /flush`: Flush/clean triple store data (optional `dataset` query parameter for Fuseki) + +--- + +## Filesystem Mode + +If no triple store is configured, OntoCast stores ontologies and facts as Turtle files in the working directory. + +--- + +## Notes + +- JSON documents must contain a `text` field, e.g.: + ```json + { "text": "abc" } + ``` +- `recursion_limit` is calculated as `max_visits * estimated_chunks` (default 30, or set via `.env`) +- Default port: 8999 + +--- + +## Docker + +To build the OntoCast Docker image: +```sh +docker buildx build -t growgraph/ontocast:0.1.4 . 2>&1 | tee build.log +``` + +--- + +## Project Structure + +``` +ontocast/ +├── agent/ # Agent workflow and orchestration +├── cli/ # CLI utilities and server +├── prompt/ # LLM prompt templates +├── stategraph/ # State graph logic +├── tool/ # Triple store, chunking, and ontology tools +├── toolbox.py # Toolbox for agent tools +├── onto.py # Ontology and RDF graph handling +├── util.py # Utilities +``` +Other directories: +- `docker/` – Docker Compose and .env.example files for triple stores +- `data/` – Example data, ontologies, and test files +- `docs/` – Documentation and user guides +- `test/` – Test suite + +--- + +## Workflow + +The extraction follows a multi-stage workflow: + +Workflow diagram + +1. **Document Preparation** + - [Optional] Convert to Markdown + - Text chunking +2. **Ontology Processing** + - Ontology selection + - Text to ontology triples + - Ontology critique +3. **Fact Extraction** + - Text to facts + - Facts critique + - Ontology sublimation +4. **Chunk Normalization** + - Chunk KG aggregation + - Entity/Property Disambiguation +5. **Storage** + - Triple/KG serialization + +--- + + +## Roadmap + +- [x] Add Jena Fuseki triple store for triple serialization +- [x] Add Neo4j n10s for triple serialization +- [ ] Replace triple prompting with a tool for local graph retrieval + +--- + +## Contributing + +Contributions are welcome! Please feel free to submit a Pull Request. + +--- + +## Acknowledgments + +- Uses RDFlib for semantic triple management +- Uses docling for pdf/pptx conversion +- Uses OpenAI language models / open models served via Ollama for fact extraction +- Uses langchain/langgraph diff --git a/ontology_platform/vendored/ontocast/docs/reference/agent/__init__.md b/ontology_platform/vendored/ontocast/docs/reference/agent/__init__.md new file mode 100644 index 0000000..4fb0eaa --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/agent/__init__.md @@ -0,0 +1,3 @@ +# `ontocast.agent` + +::: ontocast.agent diff --git a/ontology_platform/vendored/ontocast/docs/reference/agent/chunk_text.md b/ontology_platform/vendored/ontocast/docs/reference/agent/chunk_text.md new file mode 100644 index 0000000..d0e0adf --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/agent/chunk_text.md @@ -0,0 +1,3 @@ +# `ontocast.agent.chunk_text` + +::: ontocast.agent.chunk_text diff --git a/ontology_platform/vendored/ontocast/docs/reference/agent/common.md b/ontology_platform/vendored/ontocast/docs/reference/agent/common.md new file mode 100644 index 0000000..4191127 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/agent/common.md @@ -0,0 +1,3 @@ +# `ontocast.agent.common` + +::: ontocast.agent.common diff --git a/ontology_platform/vendored/ontocast/docs/reference/agent/convert_document.md b/ontology_platform/vendored/ontocast/docs/reference/agent/convert_document.md new file mode 100644 index 0000000..d2bd3ea --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/agent/convert_document.md @@ -0,0 +1,3 @@ +# `ontocast.agent.convert_document` + +::: ontocast.agent.convert_document diff --git a/ontology_platform/vendored/ontocast/docs/reference/agent/criticise_facts.md b/ontology_platform/vendored/ontocast/docs/reference/agent/criticise_facts.md new file mode 100644 index 0000000..ae7f929 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/agent/criticise_facts.md @@ -0,0 +1,3 @@ +# `ontocast.agent.criticise_facts` + +::: ontocast.agent.criticise_facts diff --git a/ontology_platform/vendored/ontocast/docs/reference/agent/criticise_ontology.md b/ontology_platform/vendored/ontocast/docs/reference/agent/criticise_ontology.md new file mode 100644 index 0000000..2352af3 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/agent/criticise_ontology.md @@ -0,0 +1,3 @@ +# `ontocast.agent.criticise_ontology` + +::: ontocast.agent.criticise_ontology diff --git a/ontology_platform/vendored/ontocast/docs/reference/agent/external_evidence.md b/ontology_platform/vendored/ontocast/docs/reference/agent/external_evidence.md new file mode 100644 index 0000000..0a07e1f --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/agent/external_evidence.md @@ -0,0 +1,3 @@ +# `ontocast.agent.external_evidence` + +::: ontocast.agent.external_evidence diff --git a/ontology_platform/vendored/ontocast/docs/reference/agent/normalize_ontology.md b/ontology_platform/vendored/ontocast/docs/reference/agent/normalize_ontology.md new file mode 100644 index 0000000..5ba88ea --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/agent/normalize_ontology.md @@ -0,0 +1,3 @@ +# `ontocast.agent.normalize_ontology` + +::: ontocast.agent.normalize_ontology diff --git a/ontology_platform/vendored/ontocast/docs/reference/agent/render_facts.md b/ontology_platform/vendored/ontocast/docs/reference/agent/render_facts.md new file mode 100644 index 0000000..95e0cb6 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/agent/render_facts.md @@ -0,0 +1,3 @@ +# `ontocast.agent.render_facts` + +::: ontocast.agent.render_facts diff --git a/ontology_platform/vendored/ontocast/docs/reference/agent/render_ontology.md b/ontology_platform/vendored/ontocast/docs/reference/agent/render_ontology.md new file mode 100644 index 0000000..33cc9dd --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/agent/render_ontology.md @@ -0,0 +1,3 @@ +# `ontocast.agent.render_ontology` + +::: ontocast.agent.render_ontology diff --git a/ontology_platform/vendored/ontocast/docs/reference/agent/select_ontology.md b/ontology_platform/vendored/ontocast/docs/reference/agent/select_ontology.md new file mode 100644 index 0000000..4d8c3d2 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/agent/select_ontology.md @@ -0,0 +1,3 @@ +# `ontocast.agent.select_ontology` + +::: ontocast.agent.select_ontology diff --git a/ontology_platform/vendored/ontocast/docs/reference/agent/serialize.md b/ontology_platform/vendored/ontocast/docs/reference/agent/serialize.md new file mode 100644 index 0000000..dd96173 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/agent/serialize.md @@ -0,0 +1,3 @@ +# `ontocast.agent.serialize` + +::: ontocast.agent.serialize diff --git a/ontology_platform/vendored/ontocast/docs/reference/agent/sublimate_ontology.md b/ontology_platform/vendored/ontocast/docs/reference/agent/sublimate_ontology.md new file mode 100644 index 0000000..5200d73 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/agent/sublimate_ontology.md @@ -0,0 +1,3 @@ +# `ontocast.agent.sublimate_ontology` + +::: ontocast.agent.sublimate_ontology diff --git a/ontology_platform/vendored/ontocast/docs/reference/cli/__init__.md b/ontology_platform/vendored/ontocast/docs/reference/cli/__init__.md new file mode 100644 index 0000000..f5925a3 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/cli/__init__.md @@ -0,0 +1,3 @@ +# `ontocast.cli` + +::: ontocast.cli diff --git a/ontology_platform/vendored/ontocast/docs/reference/cli/batch_process.md b/ontology_platform/vendored/ontocast/docs/reference/cli/batch_process.md new file mode 100644 index 0000000..0946861 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/cli/batch_process.md @@ -0,0 +1,3 @@ +# `ontocast.cli.batch_process` + +::: ontocast.cli.batch_process diff --git a/ontology_platform/vendored/ontocast/docs/reference/cli/cmp_states.md b/ontology_platform/vendored/ontocast/docs/reference/cli/cmp_states.md new file mode 100644 index 0000000..0b13e97 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/cli/cmp_states.md @@ -0,0 +1,3 @@ +# `ontocast.cli.cmp_states` + +::: ontocast.cli.cmp_states diff --git a/ontology_platform/vendored/ontocast/docs/reference/cli/merge_ontologies.md b/ontology_platform/vendored/ontocast/docs/reference/cli/merge_ontologies.md new file mode 100644 index 0000000..c4c8d95 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/cli/merge_ontologies.md @@ -0,0 +1,3 @@ +# `ontocast.cli.merge_ontologies` + +::: ontocast.cli.merge_ontologies diff --git a/ontology_platform/vendored/ontocast/docs/reference/cli/pdfs_to_markdown.md b/ontology_platform/vendored/ontocast/docs/reference/cli/pdfs_to_markdown.md new file mode 100644 index 0000000..eb78052 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/cli/pdfs_to_markdown.md @@ -0,0 +1,3 @@ +# `ontocast.cli.pdfs_to_markdown` + +::: ontocast.cli.pdfs_to_markdown diff --git a/ontology_platform/vendored/ontocast/docs/reference/cli/plot_graph.md b/ontology_platform/vendored/ontocast/docs/reference/cli/plot_graph.md new file mode 100644 index 0000000..923a5c2 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/cli/plot_graph.md @@ -0,0 +1,3 @@ +# `ontocast.cli.plot_graph` + +::: ontocast.cli.plot_graph diff --git a/ontology_platform/vendored/ontocast/docs/reference/cli/serve.md b/ontology_platform/vendored/ontocast/docs/reference/cli/serve.md new file mode 100644 index 0000000..6cb1cae --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/cli/serve.md @@ -0,0 +1,3 @@ +# `ontocast.cli.serve` + +::: ontocast.cli.serve diff --git a/ontology_platform/vendored/ontocast/docs/reference/cli/split_chunks.md b/ontology_platform/vendored/ontocast/docs/reference/cli/split_chunks.md new file mode 100644 index 0000000..8624185 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/cli/split_chunks.md @@ -0,0 +1,3 @@ +# `ontocast.cli.split_chunks` + +::: ontocast.cli.split_chunks diff --git a/ontology_platform/vendored/ontocast/docs/reference/cli/test_api.md b/ontology_platform/vendored/ontocast/docs/reference/cli/test_api.md new file mode 100644 index 0000000..dd56f43 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/cli/test_api.md @@ -0,0 +1,3 @@ +# `ontocast.cli.test_api` + +::: ontocast.cli.test_api diff --git a/ontology_platform/vendored/ontocast/docs/reference/cli/util.md b/ontology_platform/vendored/ontocast/docs/reference/cli/util.md new file mode 100644 index 0000000..a314994 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/cli/util.md @@ -0,0 +1,3 @@ +# `ontocast.cli.util` + +::: ontocast.cli.util diff --git a/ontology_platform/vendored/ontocast/docs/reference/config.md b/ontology_platform/vendored/ontocast/docs/reference/config.md new file mode 100644 index 0000000..c1a1ee3 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/config.md @@ -0,0 +1,3 @@ +# `ontocast.config` + +::: ontocast.config diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto.md b/ontology_platform/vendored/ontocast/docs/reference/onto.md new file mode 100644 index 0000000..8e9df37 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto.md @@ -0,0 +1,3 @@ +# `ontocast.onto` + +::: ontocast.onto diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto/__init__.md b/ontology_platform/vendored/ontocast/docs/reference/onto/__init__.md new file mode 100644 index 0000000..8e9df37 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto/__init__.md @@ -0,0 +1,3 @@ +# `ontocast.onto` + +::: ontocast.onto diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto/constants.md b/ontology_platform/vendored/ontocast/docs/reference/onto/constants.md new file mode 100644 index 0000000..ff0d933 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto/constants.md @@ -0,0 +1,3 @@ +# `ontocast.onto.constants` + +::: ontocast.onto.constants diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto/content_unit.md b/ontology_platform/vendored/ontocast/docs/reference/onto/content_unit.md new file mode 100644 index 0000000..c688f67 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto/content_unit.md @@ -0,0 +1,3 @@ +# `ontocast.onto.content_unit` + +::: ontocast.onto.content_unit diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto/context.md b/ontology_platform/vendored/ontocast/docs/reference/onto/context.md new file mode 100644 index 0000000..482bef6 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto/context.md @@ -0,0 +1,3 @@ +# `ontocast.onto.context` + +::: ontocast.onto.context diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto/enum.md b/ontology_platform/vendored/ontocast/docs/reference/onto/enum.md new file mode 100644 index 0000000..4d6a061 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto/enum.md @@ -0,0 +1,3 @@ +# `ontocast.onto.enum` + +::: ontocast.onto.enum diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto/model.md b/ontology_platform/vendored/ontocast/docs/reference/onto/model.md new file mode 100644 index 0000000..de64cba --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto/model.md @@ -0,0 +1,3 @@ +# `ontocast.onto.model` + +::: ontocast.onto.model diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto/null.md b/ontology_platform/vendored/ontocast/docs/reference/onto/null.md new file mode 100644 index 0000000..64e7613 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto/null.md @@ -0,0 +1,3 @@ +# `ontocast.onto.null` + +::: ontocast.onto.null diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto/ontology.md b/ontology_platform/vendored/ontocast/docs/reference/onto/ontology.md new file mode 100644 index 0000000..b2897aa --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto/ontology.md @@ -0,0 +1,3 @@ +# `ontocast.onto.ontology` + +::: ontocast.onto.ontology diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto/ontology_operations.md b/ontology_platform/vendored/ontocast/docs/reference/onto/ontology_operations.md new file mode 100644 index 0000000..12291c6 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto/ontology_operations.md @@ -0,0 +1,3 @@ +# `ontocast.onto.ontology_operations` + +::: ontocast.onto.ontology_operations diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto/rdfgraph.md b/ontology_platform/vendored/ontocast/docs/reference/onto/rdfgraph.md new file mode 100644 index 0000000..e7ef029 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto/rdfgraph.md @@ -0,0 +1,3 @@ +# `ontocast.onto.rdfgraph` + +::: ontocast.onto.rdfgraph diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto/sparql_models.md b/ontology_platform/vendored/ontocast/docs/reference/onto/sparql_models.md new file mode 100644 index 0000000..94c9d57 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto/sparql_models.md @@ -0,0 +1,3 @@ +# `ontocast.onto.sparql_models` + +::: ontocast.onto.sparql_models diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto/state.md b/ontology_platform/vendored/ontocast/docs/reference/onto/state.md new file mode 100644 index 0000000..3880ace --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto/state.md @@ -0,0 +1,3 @@ +# `ontocast.onto.state` + +::: ontocast.onto.state diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto/unit_states.md b/ontology_platform/vendored/ontocast/docs/reference/onto/unit_states.md new file mode 100644 index 0000000..b5e6aa0 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto/unit_states.md @@ -0,0 +1,3 @@ +# `ontocast.onto.unit_states` + +::: ontocast.onto.unit_states diff --git a/ontology_platform/vendored/ontocast/docs/reference/onto/util.md b/ontology_platform/vendored/ontocast/docs/reference/onto/util.md new file mode 100644 index 0000000..1273213 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/onto/util.md @@ -0,0 +1,3 @@ +# `ontocast.onto.util` + +::: ontocast.onto.util diff --git a/ontology_platform/vendored/ontocast/docs/reference/prompt/__init__.md b/ontology_platform/vendored/ontocast/docs/reference/prompt/__init__.md new file mode 100644 index 0000000..4f39dfb --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/prompt/__init__.md @@ -0,0 +1,3 @@ +# `ontocast.prompt` + +::: ontocast.prompt diff --git a/ontology_platform/vendored/ontocast/docs/reference/prompt/common.md b/ontology_platform/vendored/ontocast/docs/reference/prompt/common.md new file mode 100644 index 0000000..72b0803 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/prompt/common.md @@ -0,0 +1,3 @@ +# `ontocast.prompt.common` + +::: ontocast.prompt.common diff --git a/ontology_platform/vendored/ontocast/docs/reference/prompt/criticise_facts.md b/ontology_platform/vendored/ontocast/docs/reference/prompt/criticise_facts.md new file mode 100644 index 0000000..690dcba --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/prompt/criticise_facts.md @@ -0,0 +1,3 @@ +# `ontocast.prompt.criticise_facts` + +::: ontocast.prompt.criticise_facts diff --git a/ontology_platform/vendored/ontocast/docs/reference/prompt/criticise_ontology.md b/ontology_platform/vendored/ontocast/docs/reference/prompt/criticise_ontology.md new file mode 100644 index 0000000..3f5f733 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/prompt/criticise_ontology.md @@ -0,0 +1,3 @@ +# `ontocast.prompt.criticise_ontology` + +::: ontocast.prompt.criticise_ontology diff --git a/ontology_platform/vendored/ontocast/docs/reference/prompt/render_facts.md b/ontology_platform/vendored/ontocast/docs/reference/prompt/render_facts.md new file mode 100644 index 0000000..dd235a7 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/prompt/render_facts.md @@ -0,0 +1,3 @@ +# `ontocast.prompt.render_facts` + +::: ontocast.prompt.render_facts diff --git a/ontology_platform/vendored/ontocast/docs/reference/prompt/render_ontology.md b/ontology_platform/vendored/ontocast/docs/reference/prompt/render_ontology.md new file mode 100644 index 0000000..472af0e --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/prompt/render_ontology.md @@ -0,0 +1,3 @@ +# `ontocast.prompt.render_ontology` + +::: ontocast.prompt.render_ontology diff --git a/ontology_platform/vendored/ontocast/docs/reference/prompt/select_ontology.md b/ontology_platform/vendored/ontocast/docs/reference/prompt/select_ontology.md new file mode 100644 index 0000000..688ced8 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/prompt/select_ontology.md @@ -0,0 +1,3 @@ +# `ontocast.prompt.select_ontology` + +::: ontocast.prompt.select_ontology diff --git a/ontology_platform/vendored/ontocast/docs/reference/stategraph/__init__.md b/ontology_platform/vendored/ontocast/docs/reference/stategraph/__init__.md new file mode 100644 index 0000000..f330739 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/stategraph/__init__.md @@ -0,0 +1,3 @@ +# `ontocast.stategraph` + +::: ontocast.stategraph diff --git a/ontology_platform/vendored/ontocast/docs/reference/stategraph/atomic.md b/ontology_platform/vendored/ontocast/docs/reference/stategraph/atomic.md new file mode 100644 index 0000000..b6a8e9a --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/stategraph/atomic.md @@ -0,0 +1,3 @@ +# `ontocast.stategraph.atomic` + +::: ontocast.stategraph.atomic diff --git a/ontology_platform/vendored/ontocast/docs/reference/stategraph/create.md b/ontology_platform/vendored/ontocast/docs/reference/stategraph/create.md new file mode 100644 index 0000000..d0add48 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/stategraph/create.md @@ -0,0 +1,3 @@ +# `ontocast.stategraph.create` + +::: ontocast.stategraph.create diff --git a/ontology_platform/vendored/ontocast/docs/reference/stategraph/helpers.md b/ontology_platform/vendored/ontocast/docs/reference/stategraph/helpers.md new file mode 100644 index 0000000..f4f6aec --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/stategraph/helpers.md @@ -0,0 +1,3 @@ +# `ontocast.stategraph.helpers` + +::: ontocast.stategraph.helpers diff --git a/ontology_platform/vendored/ontocast/docs/reference/stategraph/node_factories.md b/ontology_platform/vendored/ontocast/docs/reference/stategraph/node_factories.md new file mode 100644 index 0000000..6a8f42c --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/stategraph/node_factories.md @@ -0,0 +1,3 @@ +# `ontocast.stategraph.node_factories` + +::: ontocast.stategraph.node_factories diff --git a/ontology_platform/vendored/ontocast/docs/reference/stategraph/routing.md b/ontology_platform/vendored/ontocast/docs/reference/stategraph/routing.md new file mode 100644 index 0000000..b466571 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/stategraph/routing.md @@ -0,0 +1,3 @@ +# `ontocast.stategraph.routing` + +::: ontocast.stategraph.routing diff --git a/ontology_platform/vendored/ontocast/docs/reference/stategraph/util.md b/ontology_platform/vendored/ontocast/docs/reference/stategraph/util.md new file mode 100644 index 0000000..9565dc1 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/stategraph/util.md @@ -0,0 +1,3 @@ +# `ontocast.stategraph.util` + +::: ontocast.stategraph.util diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/__init__.md b/ontology_platform/vendored/ontocast/docs/reference/tool/__init__.md new file mode 100644 index 0000000..3bd5639 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/__init__.md @@ -0,0 +1,3 @@ +# `ontocast.tool` + +::: ontocast.tool diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/agg/__init__.md b/ontology_platform/vendored/ontocast/docs/reference/tool/agg/__init__.md new file mode 100644 index 0000000..8274984 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/agg/__init__.md @@ -0,0 +1,3 @@ +# `ontocast.tool.agg` + +::: ontocast.tool.agg diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/agg/aggregate.md b/ontology_platform/vendored/ontocast/docs/reference/tool/agg/aggregate.md new file mode 100644 index 0000000..78279c0 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/agg/aggregate.md @@ -0,0 +1,3 @@ +# `ontocast.tool.agg.aggregate` + +::: ontocast.tool.agg.aggregate diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/agg/clustering.md b/ontology_platform/vendored/ontocast/docs/reference/tool/agg/clustering.md new file mode 100644 index 0000000..042fddb --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/agg/clustering.md @@ -0,0 +1,3 @@ +# `ontocast.tool.agg.clustering` + +::: ontocast.tool.agg.clustering diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/agg/normalizer.md b/ontology_platform/vendored/ontocast/docs/reference/tool/agg/normalizer.md new file mode 100644 index 0000000..d0ddd82 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/agg/normalizer.md @@ -0,0 +1,3 @@ +# `ontocast.tool.agg.normalizer` + +::: ontocast.tool.agg.normalizer diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/agg/promoter.md b/ontology_platform/vendored/ontocast/docs/reference/tool/agg/promoter.md new file mode 100644 index 0000000..01ec74a --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/agg/promoter.md @@ -0,0 +1,3 @@ +# `ontocast.tool.agg.promoter` + +::: ontocast.tool.agg.promoter diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/agg/rewriter.md b/ontology_platform/vendored/ontocast/docs/reference/tool/agg/rewriter.md new file mode 100644 index 0000000..fb9af9b --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/agg/rewriter.md @@ -0,0 +1,3 @@ +# `ontocast.tool.agg.rewriter` + +::: ontocast.tool.agg.rewriter diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/agg/uri_builder.md b/ontology_platform/vendored/ontocast/docs/reference/tool/agg/uri_builder.md new file mode 100644 index 0000000..3019d1e --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/agg/uri_builder.md @@ -0,0 +1,3 @@ +# `ontocast.tool.agg.uri_builder` + +::: ontocast.tool.agg.uri_builder diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/aggregate.md b/ontology_platform/vendored/ontocast/docs/reference/tool/aggregate.md new file mode 100644 index 0000000..d894f79 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/aggregate.md @@ -0,0 +1,3 @@ +# `ontocast.tool.aggregate` + +::: ontocast.tool.aggregate diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/atomic.md b/ontology_platform/vendored/ontocast/docs/reference/tool/atomic.md new file mode 100644 index 0000000..3a78df7 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/atomic.md @@ -0,0 +1,3 @@ +# `ontocast.tool.atomic` + +::: ontocast.tool.atomic diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/cache.md b/ontology_platform/vendored/ontocast/docs/reference/tool/cache.md new file mode 100644 index 0000000..af00567 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/cache.md @@ -0,0 +1,3 @@ +# `ontocast.tool.cache` + +::: ontocast.tool.cache diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/chunk/__init__.md b/ontology_platform/vendored/ontocast/docs/reference/tool/chunk/__init__.md new file mode 100644 index 0000000..440111a --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/chunk/__init__.md @@ -0,0 +1,3 @@ +# `ontocast.tool.chunk` + +::: ontocast.tool.chunk diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/chunk/chunker.md b/ontology_platform/vendored/ontocast/docs/reference/tool/chunk/chunker.md new file mode 100644 index 0000000..2647156 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/chunk/chunker.md @@ -0,0 +1,3 @@ +# `ontocast.tool.chunk.chunker` + +::: ontocast.tool.chunk.chunker diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/chunk/util.md b/ontology_platform/vendored/ontocast/docs/reference/tool/chunk/util.md new file mode 100644 index 0000000..3c1e048 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/chunk/util.md @@ -0,0 +1,3 @@ +# `ontocast.tool.chunk.util` + +::: ontocast.tool.chunk.util diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/converter.md b/ontology_platform/vendored/ontocast/docs/reference/tool/converter.md new file mode 100644 index 0000000..413feea --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/converter.md @@ -0,0 +1,3 @@ +# `ontocast.tool.converter` + +::: ontocast.tool.converter diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/graph_diff.md b/ontology_platform/vendored/ontocast/docs/reference/tool/graph_diff.md new file mode 100644 index 0000000..bce59cd --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/graph_diff.md @@ -0,0 +1,3 @@ +# `ontocast.tool.graph_diff` + +::: ontocast.tool.graph_diff diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/graph_version_manager.md b/ontology_platform/vendored/ontocast/docs/reference/tool/graph_version_manager.md new file mode 100644 index 0000000..356c63d --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/graph_version_manager.md @@ -0,0 +1,3 @@ +# `ontocast.tool.graph_version_manager` + +::: ontocast.tool.graph_version_manager diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/llm.md b/ontology_platform/vendored/ontocast/docs/reference/tool/llm.md new file mode 100644 index 0000000..e6e846a --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/llm.md @@ -0,0 +1,3 @@ +# `ontocast.tool.llm` + +::: ontocast.tool.llm diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/onto.md b/ontology_platform/vendored/ontocast/docs/reference/tool/onto.md new file mode 100644 index 0000000..0222925 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/onto.md @@ -0,0 +1,3 @@ +# `ontocast.tool.onto` + +::: ontocast.tool.onto diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/ontology_manager.md b/ontology_platform/vendored/ontocast/docs/reference/tool/ontology_manager.md new file mode 100644 index 0000000..e52da51 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/ontology_manager.md @@ -0,0 +1,3 @@ +# `ontocast.tool.ontology_manager` + +::: ontocast.tool.ontology_manager diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/sparql.md b/ontology_platform/vendored/ontocast/docs/reference/tool/sparql.md new file mode 100644 index 0000000..8417a56 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/sparql.md @@ -0,0 +1,3 @@ +# `ontocast.tool.sparql` + +::: ontocast.tool.sparql diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/structured_sparql.md b/ontology_platform/vendored/ontocast/docs/reference/tool/structured_sparql.md new file mode 100644 index 0000000..319c16a --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/structured_sparql.md @@ -0,0 +1,3 @@ +# `ontocast.tool.structured_sparql` + +::: ontocast.tool.structured_sparql diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/__init__.md b/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/__init__.md new file mode 100644 index 0000000..4c6457e --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/__init__.md @@ -0,0 +1,3 @@ +# `ontocast.tool.triple_manager` + +::: ontocast.tool.triple_manager diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/core.md b/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/core.md new file mode 100644 index 0000000..061c435 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/core.md @@ -0,0 +1,3 @@ +# `ontocast.tool.triple_manager.core` + +::: ontocast.tool.triple_manager.core diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/filesystem_manager.md b/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/filesystem_manager.md new file mode 100644 index 0000000..770c7de --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/filesystem_manager.md @@ -0,0 +1,3 @@ +# `ontocast.tool.triple_manager.filesystem_manager` + +::: ontocast.tool.triple_manager.filesystem_manager diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/fuseki.md b/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/fuseki.md new file mode 100644 index 0000000..a2453a8 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/fuseki.md @@ -0,0 +1,3 @@ +# `ontocast.tool.triple_manager.fuseki` + +::: ontocast.tool.triple_manager.fuseki diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/mock.md b/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/mock.md new file mode 100644 index 0000000..966c9e0 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/mock.md @@ -0,0 +1,3 @@ +# `ontocast.tool.triple_manager.mock` + +::: ontocast.tool.triple_manager.mock diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/neo4j.md b/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/neo4j.md new file mode 100644 index 0000000..80d639f --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/triple_manager/neo4j.md @@ -0,0 +1,3 @@ +# `ontocast.tool.triple_manager.neo4j` + +::: ontocast.tool.triple_manager.neo4j diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/validate.md b/ontology_platform/vendored/ontocast/docs/reference/tool/validate.md new file mode 100644 index 0000000..c7915ee --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/validate.md @@ -0,0 +1,3 @@ +# `ontocast.tool.validate` + +::: ontocast.tool.validate diff --git a/ontology_platform/vendored/ontocast/docs/reference/tool/web_search.md b/ontology_platform/vendored/ontocast/docs/reference/tool/web_search.md new file mode 100644 index 0000000..a44dc24 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/tool/web_search.md @@ -0,0 +1,3 @@ +# `ontocast.tool.web_search` + +::: ontocast.tool.web_search diff --git a/ontology_platform/vendored/ontocast/docs/reference/toolbox.md b/ontology_platform/vendored/ontocast/docs/reference/toolbox.md new file mode 100644 index 0000000..b9ab283 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/toolbox.md @@ -0,0 +1,3 @@ +# `ontocast.toolbox` + +::: ontocast.toolbox diff --git a/ontology_platform/vendored/ontocast/docs/reference/util.md b/ontology_platform/vendored/ontocast/docs/reference/util.md new file mode 100644 index 0000000..9f06d18 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/reference/util.md @@ -0,0 +1,3 @@ +# `ontocast.util` + +::: ontocast.util diff --git a/ontology_platform/vendored/ontocast/docs/user_guide/concepts.md b/ontology_platform/vendored/ontocast/docs/user_guide/concepts.md new file mode 100644 index 0000000..d254c20 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/user_guide/concepts.md @@ -0,0 +1,61 @@ +# Concepts + +Here we introduce the main concepts of OntoCast, a framework for transforming data into semantic triples. + +## Ontology Management + +OntoCast manages ontologies with automatic versioning and timestamp tracking: + +- **Semantic Versioning**: Automatic version increments (MAJOR/MINOR/PATCH) based on change analysis +- **Hash-Based Lineage**: Git-style versioning with parent hashes for tracking ontology evolution +- **Multiple Versions**: Versions stored as separate named graphs in Fuseki triple stores +- **Timestamp Tracking**: `updated_at` field tracks when ontology was last modified +- **Smart Analysis**: Analyzes ontology changes (classes, properties, instances) to determine appropriate version bump: + - **MAJOR**: Substantial breaking changes (deletions of classes/properties) + - **MINOR**: New features (new classes/properties) or any deletions + - **PATCH**: Updates to existing structures (instances, descriptions, small changes) +- **Property Syncing**: Version and timestamp are synced to the RDF graph as `owl:versionInfo` and `dcterms:modified` +- **Versioned IRIs**: Each version gets a unique IRI with hash fragment for storage organization + +## GraphUpdate System + +OntoCast uses a token-efficient GraphUpdate system for incremental graph modifications: + +- **Structured Operations**: LLM outputs `GraphUpdate` objects containing `TripleOp` operations (insert/delete) instead of full TTL graphs +- **Token Efficiency**: Only changes are generated, dramatically reducing LLM token usage compared to full graph regeneration +- **SPARQL Generation**: Operations are automatically converted to executable SPARQL queries +- **Incremental Updates**: Graph updates are applied incrementally, allowing for precise modifications +- **Operation Types**: Supports both `insert` and `delete` operations with explicit prefix declarations +- **Custom Queries**: Also supports `GenericSparqlQuery` for complex custom SPARQL operations + +### How GraphUpdate Saves Tokens + +Instead of generating the entire graph in Turtle format (which can be thousands of tokens), the LLM now outputs only the changes: + +- **Before**: Full TTL graph with all triples (e.g., 5000 tokens) +- **After**: Structured operations with only changes (e.g., 200 tokens) +- **Savings**: Typically 80-95% reduction in output tokens + +## Budget Tracking + +OntoCast provides comprehensive budget tracking for LLM usage and triple generation: + +- **LLM Statistics**: Tracks API calls, characters sent/received for cost monitoring +- **Triple Metrics**: Tracks ontology and facts triples generated per operation +- **Operation Counts**: Tracks number of update operations for both ontology and facts +- **Summary Reports**: Budget summaries logged at end of processing with format: + ``` + LLM: X calls, Y sent, Z received | Triples: A ontology, B facts + ``` +- **Integrated Tracking**: Budget tracker integrated into AgentState for clean dependency injection +- **Automatic Updates**: Budget tracker automatically updated when LLM calls are made or triples are generated + +## Key Components + +- **Ontology**: RDF graph with properties (id, title, description, version, timestamp, hash, parent_hashes) +- **AgentState**: Central state management with budget tracking and GraphUpdate operations +- **ToolBox**: Collection of tools for processing and caching +- **Triple Stores**: Support for filesystem, Fuseki, and Neo4j storage +- **GraphUpdate**: Structured representation of graph modifications as SPARQL operations +- **BudgetTracker**: Lightweight tracker for LLM usage and triple generation statistics + diff --git a/ontology_platform/vendored/ontocast/docs/user_guide/configuration.md b/ontology_platform/vendored/ontocast/docs/user_guide/configuration.md new file mode 100644 index 0000000..0ab8187 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/user_guide/configuration.md @@ -0,0 +1,149 @@ +# Configuration System + +OntoCast configuration is powered by Pydantic `BaseSettings` and is loaded from environment variables (typically via `.env`). + +## Overview + +- Typed config sections with defaults +- Environment variable parsing (including lists and booleans) +- Validation for provider/model compatibility +- Unified `Config` object shared across tools and server + +## Configuration Shape + +```python +Config +├── tool_config: ToolConfig +│ ├── llm_config: LLMConfig +│ ├── chunk_config: ChunkConfig +│ ├── path_config: PathConfig +│ ├── neo4j: Neo4jConfig +│ ├── fuseki: FusekiConfig +│ ├── domain: DomainConfig +│ ├── web_search: WebSearchConfig +│ └── aggregation: AggregationConfig +└── server: ServerConfig +``` + +## Environment Variables + +### LLM + +```bash +LLM_PROVIDER=openai # openai | ollama +LLM_MODEL_NAME=gpt-4o-mini +LLM_TEMPERATURE=0.0 +LLM_API_KEY=your_openai_api_key_here # required for openai provider +LLM_BASE_URL=http://localhost:11434 # optional (mainly for ollama) +``` + +### Server + +```bash +PORT=8999 +BASE_RECURSION_LIMIT=1000 +ESTIMATED_CHUNKS=30 +MAX_VISITS=3 # alias for max_visits_per_node +RENDER_MODE=ontology_and_facts # ontology | facts | ontology_and_facts +ONTOLOGY_MAX_TRIPLES=50000 # empty/unset for unlimited +PARALLEL_WORKERS=4 +PARALLEL_FACTS_RETRIES=3 +PARALLEL_ONTOLOGY_RETRIES=3 +ENABLE_ONTOLOGY_CONSOLIDATION=false +``` + +### Chunking + +```bash +CHUNK_BREAKPOINT_THRESHOLD_TYPE=percentile # percentile | standard_deviation | interquartile | gradient +CHUNK_BREAKPOINT_THRESHOLD_AMOUNT=95.0 +CHUNK_MIN_SIZE=3000 +CHUNK_MAX_SIZE=12000 +``` + +### Triple Stores + +```bash +# Fuseki +FUSEKI_URI=http://localhost:3030/test +FUSEKI_AUTH=admin/admin +FUSEKI_DATASET=dataset_name +FUSEKI_ONTOLOGIES_DATASET=ontologies + +# Neo4j +NEO4J_URI=bolt://localhost:7687 +NEO4J_AUTH=neo4j/test +NEO4J_PORT=7476 +NEO4J_BOLT_PORT=7689 +``` + +### Paths and Domain + +```bash +CURRENT_DOMAIN=https://example.com +ONTOCAST_WORKING_DIRECTORY=/path/to/working/directory +ONTOCAST_ONTOLOGY_DIRECTORY=/path/to/ontology/files +ONTOCAST_CACHE_DIR=/path/to/cache/directory +``` + +### Aggregation + +```bash +AGG_EMBEDDING_MODEL=paraphrase-multilingual-MiniLM-L12-v2 +AGG_SIMILARITY_THRESHOLD=0.80 +``` + +### Web Search + +```bash +WEB_SEARCH_ENABLED=false +WEB_SEARCH_PROVIDER=duckduckgo +WEB_SEARCH_TOP_K=3 +WEB_SEARCH_TIMEOUT_SECONDS=8.0 +WEB_SEARCH_MAX_SNIPPET_CHARS=400 +WEB_SEARCH_MAX_TOTAL_CHARS=1800 +WEB_SEARCH_ONTOLOGY_RENDER_ENABLED=true +WEB_SEARCH_ONTOLOGY_CRITIC_ENABLED=true +WEB_SEARCH_FACTS_RENDER_ENABLED=false +WEB_SEARCH_FACTS_CRITIC_ENABLED=false +WEB_SEARCH_PLANNER_ENABLED=true +WEB_SEARCH_PLANNER_MAX_QUERIES=3 +WEB_SEARCH_PLANNER_MIN_QUERY_CHARS=12 +WEB_SEARCH_PLANNER_MIN_CONFIDENCE=0.35 +WEB_SEARCH_REUSE_EVIDENCE_ACROSS_ATTEMPT=true +WEB_SEARCH_MIN_SNIPPET_CHARS=40 +WEB_SEARCH_ALLOWED_DOMAINS= # comma-separated +WEB_SEARCH_BLOCKED_DOMAINS= # comma-separated +WEB_SEARCH_REGION=wt-wt +WEB_SEARCH_SAFESEARCH=moderate +``` + +Search is "search-later": nodes run without search first, and only request external evidence when needed. + +## Usage + +```python +from ontocast.config import Config + +config = Config() +tool_config = config.get_tool_config() + +print(config.server.port) +print(config.server.max_visits_per_node) +print(tool_config.llm_config.provider) +print(tool_config.path_config.cache_dir) +``` + +## Validation Notes + +- `LLM_PROVIDER=openai` requires `LLM_API_KEY`. +- `LLM_MODEL_NAME` must match the selected provider family. +- `MAX_VISITS` is supported as an alias for `max_visits_per_node`. +- `WEB_SEARCH_ALLOWED_DOMAINS` and `WEB_SEARCH_BLOCKED_DOMAINS` accept comma-separated values. + +## Recommended Workflow + +1. Copy `.env.example` to `.env`. +2. Fill in LLM credentials and backend settings. +3. Start with defaults for chunking/search/aggregation. +4. Tune only after inspecting extraction quality and runtime. diff --git a/ontology_platform/vendored/ontocast/docs/user_guide/llm_caching.md b/ontology_platform/vendored/ontocast/docs/user_guide/llm_caching.md new file mode 100644 index 0000000..fd19fc5 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/user_guide/llm_caching.md @@ -0,0 +1,429 @@ +# LLM Caching + +OntoCast includes automatic LLM response caching to improve performance, reduce API costs, and enable offline testing capabilities. + +--- + +## Overview + +The LLM caching system automatically caches responses from language model providers, ensuring that identical queries return cached results instead of making new API calls. This provides several benefits: + +- **Performance**: Cached responses return instantly +- **Cost Reduction**: Avoids duplicate API calls +- **Offline Testing**: Tests can run without API access +- **Transparency**: No configuration required - works automatically + +--- + +## Shared Caching Architecture + +OntoCast uses a **shared caching architecture** where: + +- **Single Cacher Instance**: One `Cacher` object manages all caching for all tools +- **Tool-Specific Subdirectories**: Each tool gets its own subdirectory within the shared cache +- **Dependency Injection**: Tools receive the shared Cacher instance through their constructors +- **Organized Storage**: Cache files are organized by tool type (llm/, converter/, chunker/) + +### Benefits + +1. **Memory Efficiency**: Single cache instance instead of multiple +2. **Consistent Configuration**: All tools use the same cache directory settings +3. **Centralized Management**: Easy to clear, monitor, and manage all caches +4. **Better Organization**: Clear separation of cache files by tool type + +--- + +## How It Works + +### Shared Caching + +OntoCast uses a shared caching system where all tools share a single Cacher instance: + +```python +from ontocast.tool.llm import LLMTool +from ontocast.config import LLMConfig +from ontocast.tool.cache import Cacher + +# Create shared cache instance +shared_cache = Cacher() + +# Create LLM tool with shared cache +llm_config = LLMConfig( + provider="openai", + model_name="gpt-4o-mini", + api_key="your-api-key" +) + +llm_tool = LLMTool.create(config=llm_config, cache=shared_cache) + +# First call - hits API and caches response +response1 = llm_tool("What is the capital of France?") + +# Second call - returns cached response instantly +response2 = llm_tool("What is the capital of France?") +``` + +### Cache Key Generation + +Cache keys are generated based on: +- LLM provider and model +- Prompt text +- Temperature and other parameters +- API endpoint URL + +This ensures that different configurations or parameters result in separate cache entries. + +--- + +## Cache Locations + +### Default Locations + +The system automatically selects appropriate cache directories: + +- **Tests**: `.test_cache/llm/` in the current working directory +- **Windows**: `%USERPROFILE%\AppData\Local\ontocast\llm\` +- **Unix/Linux**: `~/.cache/ontocast/llm/` (or `$XDG_CACHE_HOME/ontocast/llm/`) + +### Environment Variables + +Set the cache directory via environment variables: + +```bash +# OntoCast cache directory (recommended) +export ONTOCAST_CACHE_DIR=/path/to/custom/cache + +# Or use XDG cache home (affects all XDG-compliant applications) +export XDG_CACHE_HOME=/path/to/custom/cache +``` + +### CLI Parameter + +Specify cache directory via command line: + +```bash +ontocast --env-path .env --working-directory ./work --cache-dir /custom/cache/path +``` + +--- + +## Cache Management + +### Cache Structure + +The cache directory contains organized subdirectories: + +``` +cache_dir/ +├── openai/ +│ ├── gpt-4o-mini/ +│ │ ├── prompt_hash_1.json +│ │ └── prompt_hash_2.json +│ └── gpt-4/ +│ └── prompt_hash_3.json +└── ollama/ + └── llama2/ + └── prompt_hash_4.json +``` + +### Cache Files + +Each cached response is stored as a JSON file containing: +- Original prompt and parameters +- Response content +- Metadata (timestamp, model info) +- Cache key hash + +--- + +## Testing with Caching + +### Offline Testing + +Cached responses enable offline testing: + +```python +# First run - with API access +pytest test_llm_functionality.py + +# Subsequent runs - offline (uses cached responses) +pytest test_llm_functionality.py +``` + +### Test Isolation + +Each test run uses a separate cache directory (`.test_cache/llm/`) to avoid interference between tests. + +--- + +## Performance Benefits + +### Speed Improvements + +- **First Call**: Normal API response time +- **Cached Calls**: Near-instant response (< 1ms) +- **Batch Processing**: Significant speedup for repeated operations + +### Cost Savings + +- **Development**: Avoid repeated API calls during development +- **Testing**: Run tests without API costs +- **Production**: Reduce API usage for common queries + +--- + +## Best Practices + +### Development + +1. **Use Default Locations**: Let the system choose appropriate cache directories +2. **Version Control**: Add cache directories to `.gitignore` +3. **Cleanup**: Periodically clean old cache files + +### Production + +1. **Persistent Storage**: Use persistent cache directories +2. **Monitoring**: Monitor cache hit rates +3. **Maintenance**: Implement cache cleanup strategies + +### Testing + +1. **Isolated Caches**: Each test run gets its own cache +2. **Deterministic**: Cached responses ensure consistent test results +3. **Offline Capability**: Tests can run without API access + +--- + +## Troubleshooting + +### Common Issues + +1. **Cache Not Working**: Check directory permissions +2. **Stale Responses**: Clear cache directory +3. **Disk Space**: Monitor cache directory size + +### Debug Cache + +```python +from ontocast.tool.llm import LLMTool + +# Check cache directory +llm_tool = LLMTool.create(config=llm_config) +print(f"Cache directory: {llm_tool.cache.tool_cache_dir}") + +# List cached files +cache_files = list(llm_tool.cache.tool_cache_dir.glob("**/*.json")) +print(f"Cached responses: {len(cache_files)}") +``` + +### Clear Cache + +```python +import shutil +from pathlib import Path + +# Clear entire cache +cache_dir = Path.home() / ".cache" / "ontocast" / "llm" +if cache_dir.exists(): + shutil.rmtree(cache_dir) + print("Cache cleared!") +``` + +--- + +## Advanced Usage + +### Custom Cache Implementation + +For advanced use cases, you can implement custom caching by extending the Cacher class: + +```python +from ontocast.tool.llm import LLMTool +from ontocast.tool.cache import Cacher +from pathlib import Path + +class CustomLLMTool(LLMTool): + def __init__(self, config, **kwargs): + super().__init__(config, **kwargs) + # Override with custom cache + self.cache = Cacher(subdirectory="llm", cache_dir=Path("/custom/cache")) +``` + +### Cache Statistics + +```python +from ontocast.tool.llm import LLMTool + +# Get cache statistics +llm_tool = LLMTool.create(config=llm_config) +stats = llm_tool.cache.get_cache_stats() +print(f"Cache stats: {stats}") +``` + +--- + +## Integration with Other Tools + +### ToolBox Integration + +Caching works seamlessly with the ToolBox through a shared Cacher instance: + +```python +from ontocast.toolbox import ToolBox +from ontocast.config import Config + +# ToolBox automatically creates and uses a shared Cacher +config = Config() +tools = ToolBox(config) + +# All tools (LLM, Converter, Chunker) share the same cache instance +result = tools.llm("Process this document") +converted = tools.converter(document_file) +chunks = tools.chunker(text) +``` + +### Server Integration + +The server automatically uses caching for all LLM operations: + +```bash +# Start server with automatic caching +ontocast --env-path .env --working-directory /data/working +``` + +--- + +## Security Considerations + +### Sensitive Data + +- Cache files may contain sensitive prompt data +- Ensure proper file permissions on cache directories +- Consider encryption for sensitive deployments + +### Access Control + +- Restrict access to cache directories +- Use appropriate file system permissions +- Consider network security for shared cache directories + + +--- + +## Converter and Chunker Caching + +In addition to LLM response caching, OntoCast also includes caching for document conversion and text chunking operations. This helps avoid redundant processing when the same documents or text are processed multiple times. + +### Converter Caching + +The `ConverterTool` automatically caches document conversion results based on the input file content. This means: + +- **PDF files**: If the same PDF is processed multiple times, the conversion to markdown is cached +- **Other documents**: PowerPoint, Word documents, etc. are also cached after conversion +- **Plain text**: Text input is not cached as it doesn't require conversion + +### Chunker Caching + +The `ChunkerTool` caches chunking results based on: +- **Input text content**: The exact text being chunked +- **Chunking configuration**: All chunking parameters (max_size, min_size, model, etc.) +- **Chunking mode**: Whether semantic or naive chunking is used + +This ensures that identical text with identical chunking parameters will return cached results. + +### Cache Organization + +Caching is organized in subdirectories: + +``` +~/.cache/ontocast/ +├── llm/ # LLM response cache +├── converter/ # Document conversion cache +└── chunker/ # Text chunking cache +``` + +### Cache Benefits + +1. **Faster Processing**: Repeated operations return instantly from cache +2. **Cost Reduction**: Avoids redundant LLM API calls and processing +3. **Consistency**: Identical inputs always produce identical outputs +4. **Offline Capability**: Cached operations work without API access + +### Cache Management + +You can access cache statistics and management through the tool instances: + +```python +from ontocast.tool.converter import ConverterTool +from ontocast.tool.chunk.chunker import ChunkerTool + +# Get cache statistics +converter = ConverterTool() +stats = converter.cache.get_cache_stats() +print(f"Converter cache: {stats['total_files']} files, {stats['total_size_bytes']} bytes") + +# Clear cache if needed +converter.cache.clear() + +# Chunker cache management +chunker = ChunkerTool() +chunker.cache.clear() # Clear chunker cache +``` + +### Custom Cache Directories + +You can specify custom cache directories in several ways: + +#### 1. Environment Variables + +```bash +# OntoCast cache directory (recommended) +export ONTOCAST_CACHE_DIR=/custom/cache/path + +# Or use XDG cache home (affects all XDG-compliant applications) +export XDG_CACHE_HOME=/custom/cache/path +``` + +#### 2. CLI Parameter + +```bash +ontocast --env-path .env --working-directory ./work --cache-dir /custom/cache/path +``` + +#### 3. Programmatic Configuration + +```python +from ontocast.toolbox import ToolBox +from ontocast.config import Config +from pathlib import Path + +# Create config and set cache directory +config = Config() +config.tool_config.path_config.cache_dir = Path("/custom/cache/path") + +# Create ToolBox with config (cache directory is automatically used) +tools = ToolBox(config) + +# All tools will use the same custom cache directory +result = tools.llm("Process this document") +converted = tools.converter(document_file) +chunks = tools.chunker(text) +``` + +### Cache Key Generation + +Cache keys are generated based on: +- **Content hash**: SHA256 hash of the input content +- **Configuration**: All relevant parameters that affect the output +- **Tool-specific parameters**: Model names, chunking modes, etc. + +This ensures that different configurations produce different cache entries, even for the same input content. + +### Best Practices + +1. **Let caching work automatically**: No configuration needed for basic usage +2. **Monitor cache size**: Check cache statistics periodically +3. **Clear cache when needed**: If you change tool configurations significantly +4. **Use custom directories**: For testing or specific deployment scenarios +5. **Cache persistence**: Caches persist between runs for maximum benefit + diff --git a/ontology_platform/vendored/ontocast/docs/user_guide/triple_stores.md b/ontology_platform/vendored/ontocast/docs/user_guide/triple_stores.md new file mode 100644 index 0000000..3deada5 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/user_guide/triple_stores.md @@ -0,0 +1,269 @@ +# Triple Store Configuration + +OntoCast supports multiple triple store backends for storing and managing RDF data. This guide covers the setup and configuration of supported triple stores. + +--- + +## Overview + +OntoCast supports the following triple store backends: + +1. **Apache Fuseki** (Recommended) - Native RDF triple store with SPARQL support +2. **Neo4j with n10s plugin** - Graph database with RDF capabilities +3. **Filesystem** - Local file-based storage (fallback) + +When multiple triple stores are configured, OntoCast uses the following priority order: +1. Fuseki (if `FUSEKI_URI` and `FUSEKI_AUTH` are set) +2. Neo4j (if `NEO4J_URI` and `NEO4J_AUTH` are set) +3. Filesystem (default fallback) + +--- + +## Configuration + +### Environment Variables + +Configure your triple store connection using environment variables in your `.env` file: + +```bash +# Fuseki Configuration (Preferred) +FUSEKI_URI=http://localhost:3032/test +FUSEKI_AUTH=admin:password +FUSEKI_DATASET=dataset_name + +# Neo4j Configuration (Alternative) +NEO4J_URI=bolt://localhost:7689 +NEO4J_AUTH=neo4j:password + +``` + +### Configuration Hierarchy + +The new configuration system provides better organization: + +```python +from ontocast.config import Config + +config = Config() + +# Access triple store configuration +tool_config = config.get_tool_config() + +# Check which triple store is configured +if tool_config.fuseki.uri and tool_config.fuseki.auth: + print("Using Fuseki triple store") +elif tool_config.neo4j.uri and tool_config.neo4j.auth: + print("Using Neo4j triple store") +else: + print("Using filesystem storage") +``` + +--- + +## Apache Fuseki Setup + +Sample configurations are provided here: [ontocast/docker](https://github.com/growgraph/ontocast/tree/main/docker). + +**1. Prepare the environment file:** +```bash +cd docker/fuseki +cp .env.example .env +# Edit with your values +``` + +**Example `docker/fuseki/.env.example`:** +```bash +IMAGE_VERSION=secoresearch/fuseki:5.1.0 +SPEC=test +CONTAINER_NAME="${SPEC}.fuseki" +STORE_FOLDER="$HOME/tmp/${CONTAINER_NAME}" +TS_PORT=3032 +TS_PASSWORD="abc123-qwe" +TS_USERNAME="admin" +UID=1000 +GID=1000 +``` + +**2. Start/Stop Fuseki:** +```bash +# Start +cd docker/fuseki +docker compose --env-file .env fuseki up -d + +# Stop +# (use the container name from your .env, e.g. test.fuseki) +docker compose stop test.fuseki +``` + +**3. Access Fuseki:** + +- Web interface: http://localhost:3032 +- Default dataset: `/test` +- SPARQL endpoint: http://localhost:3032/test/sparql + +**4. Configure OntoCast for Fuseki:** + +```bash +# In your .env file +FUSEKI_URI=http://localhost:3032/test +FUSEKI_AUTH=admin:abc123-qwe +FUSEKI_DATASET=dataset_name +``` + +--- + +## Neo4j with n10s Plugin Setup + +**1. Prepare the environment file:** +```bash +cd docker/neo4j +cp .env.example .env +# Edit with your values +``` + +**Example `docker/neo4j/.env.example`:** +```bash +IMAGE_VERSION=neo4j:5.20 +SPEC=test +CONTAINER_NAME="${SPEC}.sem.neo4j" +NEO4J_PORT=7476 +NEO4J_BOLT_PORT=7689 +STORE_FOLDER="$HOME/tmp/${CONTAINER_NAME}" +NEO4J_PLUGINS='["apoc", "graph-data-science", "n10s"]' +NEO4J_AUTH="neo4j/test!passfortesting" +``` + +**2. Start/Stop Neo4j:** +```bash +# Start +cd docker/neo4j +docker compose --env-file .env neo4j up -d + +# Stop +docker compose stop neo4j +``` + +**3. Access Neo4j:** + +- Browser: http://localhost:7476 +- Username: `neo4j` +- Password: `test!passfortesting` +- Bolt: bolt://localhost:7689 + +**4. Configure OntoCast for Neo4j:** + +```bash +# In your .env file +NEO4J_URI=bolt://localhost:7689 +NEO4J_AUTH=neo4j:test!passfortesting +``` + +--- + +## Filesystem Storage (Fallback) + +If neither Fuseki nor Neo4j is configured, OntoCast will store ontologies and facts as Turtle files in the working directory. + +**No setup required - works out of the box.** + +--- + +## Triple Store Comparison + +| Feature | Fuseki | Neo4j + n10s | Filesystem | +|---------|--------|--------------|------------| +| **RDF Native** | ✅ Yes | ⚠️ Via plugin | ✅ Yes | +| **SPARQL** | ✅ Full 1.1 | ❌ Limited | ❌ No | +| **Setup Complexity** | ✅ Simple | ⚠️ Moderate | ✅ Very Simple | +| **Visualization** | ⚠️ Basic | ✅ Excellent | ❌ None | +| **Production Ready** | ✅ Yes | ✅ Yes | ❌ No | +| **Configuration** | ✅ Environment vars | ✅ Environment vars | ✅ Automatic | + +--- + +## Best Practices + +- Use **Filesystem** for quick setup and testing +- Use **Fuseki** for RDF-focused or production deployments +- Use **Neo4j** if you need advanced graph analytics or visualization +- Monitor triple store performance and logs +- Backup your data regularly +- Use the `/flush` API endpoint to clean triple stores when needed (see below) + +--- + +## Troubleshooting + +### Fuseki +```bash +# Check if Fuseki is running +curl http://localhost:3032/$/ping + +# Restart Fuseki +docker compose restart fuseki + +# Check dataset exists +curl http://localhost:3032/$/datasets +``` + +### Neo4j +```bash +# Check if Neo4j is running +curl http://localhost:7476 + +# Check n10s plugin +cypher-shell -u neo4j -p test!passfortesting "CALL n10s.graphconfig.show()" +``` + +### Common Problems +- **Connection Refused**: Triple store not running +- **Authentication Failed**: Incorrect credentials in environment variables +- **Dataset Not Found**: Dataset not created in Fuseki +- **Plugin Not Loaded**: n10s plugin not installed in Neo4j +- **Configuration Not Loaded**: Check `.env` file and environment variable names + +--- + +## Flushing Triple Store Data + +You can clean/flush data from the triple store using the `/flush` API endpoint. This endpoint allows you to explicitly delete data when needed. + +### Using the Flush Endpoint + +```bash +# Clean all datasets (Fuseki) or entire database (Neo4j) +curl -X POST http://localhost:8999/flush + +# Clean specific Fuseki dataset +curl -X POST "http://localhost:8999/flush?dataset=my_dataset" +``` + +**For Fuseki:** +- If no `dataset` parameter is provided, both the main dataset and ontologies dataset are cleaned +- If a `dataset` parameter is provided, only that specific dataset is cleaned + +**For Neo4j:** +- The `dataset` parameter is ignored (Neo4j doesn't support datasets) +- All nodes and relationships are deleted + +**Warning:** This operation is irreversible and will delete all data. Use with caution in production environments! + +--- + +## Migration from Previous Versions + +If you're upgrading from a previous version of OntoCast: + +1. **Update Environment Variables**: The configuration system has been refactored +2. **Check Triple Store Settings**: Ensure your triple store configuration is properly set +3. **Test Configuration**: Use the new configuration system to verify your setup + +```python +# Test your configuration +from ontocast.config import Config + +config = Config() +print("Configuration loaded successfully!") +print(f"LLM Provider: {config.tool_config.llm_config.provider}") +print(f"Working Directory: {config.tool_config.path_config.working_directory}") +``` diff --git a/ontology_platform/vendored/ontocast/docs/user_guide/user_instructions.md b/ontology_platform/vendored/ontocast/docs/user_guide/user_instructions.md new file mode 100644 index 0000000..4d48918 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/user_guide/user_instructions.md @@ -0,0 +1,314 @@ +# User Instructions + +User instructions allow you to provide specific guidance to OntoCast about what to focus on during ontology and facts extraction. This feature is particularly useful when you want to direct the AI's attention to specific types of entities, relationships, or concepts in your documents. + +--- + +## Overview + +User instructions work by injecting custom instructions into the AI prompts used during: + +- **Ontology Extraction**: When the system extracts domain concepts and relationships +- **Facts Extraction**: When the system extracts specific facts from your documents + +This allows you to customize the extraction process based on your specific needs and domain requirements. + +--- + +## How User Instructions Work + +### 1. Ontology User Instructions + +Ontology user instructions guide the AI when extracting domain concepts and relationships from your documents. These instructions help focus on specific types of entities or relationships. + +**Example:** +``` +Focus on extracting geographical locations, organizations, and their relationships. Pay special attention to company mergers, acquisitions, and partnerships. +``` + +### 2. Facts User Instructions + +Facts user instructions guide the AI when extracting specific facts and instances from your documents. These instructions help focus on particular types of facts or data points. + +**Example:** +``` +Extract financial data, dates, and numerical values. Focus on revenue, profit, and growth metrics. Include all monetary amounts with proper currency information. +``` + +--- + +## Usage Methods + +### 1. JSON API Requests + +When sending JSON requests to the API, include user instructions in your payload: + +```json +{ + "text": "Your document text here...", + "ontology_user_instruction": "Focus on extracting geographical locations and organizations", + "facts_user_instruction": "Extract financial data and numerical values with proper currency information" +} +``` + +### 2. Form Data (Multipart) + +When using multipart form data, include user instructions as form fields: + +```bash +curl -X POST http://localhost:8999/process \ + -F "file=@document.pdf" \ + -F "ontology_user_instruction=Focus on extracting geographical locations and organizations" \ + -F "facts_user_instruction=Extract financial data and numerical values" +``` + +### 3. Programmatic Usage + +When using OntoCast programmatically, set user instructions in the AgentState: + +```python +from ontocast.onto.state import AgentState + +# Create state with user instructions +state = AgentState( + input_text="Your document text...", + ontology_user_instruction="Focus on extracting geographical locations and organizations", + facts_user_instruction="Extract financial data and numerical values" +) +``` + +--- + +## Best Practices + +### 1. Be Specific and Clear + +**Good:** +``` +Focus on extracting company names, financial metrics, and business relationships. Pay special attention to revenue, profit, and growth data. +``` + +**Avoid:** +``` +Extract everything important. +``` + +### 2. Use Domain-Specific Language + +**Good:** +``` +Extract medical diagnoses, symptoms, treatments, and patient information. Focus on ICD-10 codes and medical terminology. +``` + +**Avoid:** +``` +Extract medical stuff. +``` + +### 3. Provide Context + +**Good:** +``` +Extract legal entities, court cases, and legal relationships. Focus on case numbers, dates, and legal precedents mentioned in the document. +``` + +**Avoid:** +``` +Extract legal information. +``` + +### 4. Specify Data Types + +**Good:** +``` +Extract numerical data with proper units (currency, percentages, measurements). Include dates in ISO format and geographical coordinates. +``` + +**Avoid:** +``` +Extract numbers and dates. +``` + +--- + +## Common Use Cases + +### 1. Financial Documents + +**Ontology Instruction:** +``` +Focus on extracting financial concepts, business entities, and economic relationships. Pay attention to revenue streams, cost structures, and financial metrics. +``` + +**Facts Instruction:** +``` +Extract all monetary amounts with currency codes, percentages, and financial ratios. Include dates for financial periods and growth rates. +``` + +### 2. Medical Documents + +**Ontology Instruction:** +``` +Focus on extracting medical conditions, treatments, symptoms, and healthcare relationships. Pay attention to medical terminology and clinical concepts. +``` + +**Facts Instruction:** +``` +Extract patient information, medical codes (ICD-10, CPT), dosages, and treatment timelines. Include all medical measurements and lab values. +``` + +### 3. Legal Documents + +**Ontology Instruction:** +``` +Focus on extracting legal entities, court cases, legal relationships, and regulatory frameworks. Pay attention to legal terminology and precedents. +``` + +**Facts Instruction:** +``` +Extract case numbers, court dates, legal citations, and regulatory compliance information. Include all legal references and precedents. +``` + +### 4. Scientific Papers + +**Ontology Instruction:** +``` +Focus on extracting scientific concepts, methodologies, and research relationships. Pay attention to scientific terminology and theoretical frameworks. +``` + +**Facts Instruction:** +``` +Extract experimental data, measurements, statistical results, and research findings. Include all numerical data with proper units and significance levels. +``` + +--- + +## Advanced Examples + +### 1. Multi-Domain Extraction + +```json +{ + "text": "Your document text...", + "ontology_user_instruction": "Extract both business and technical concepts. Focus on companies, products, technologies, and their relationships.", + "facts_user_instruction": "Extract business metrics, technical specifications, and performance data. Include all numerical values with proper context." +} +``` + +### 2. Temporal Focus + +```json +{ + "text": "Your document text...", + "ontology_user_instruction": "Focus on extracting entities and relationships that are time-sensitive or have temporal aspects.", + "facts_user_instruction": "Extract all dates, time periods, and temporal relationships. Pay special attention to historical events and chronological data." +} +``` + +### 3. Geographic Focus + +```json +{ + "text": "Your document text...", + "ontology_user_instruction": "Focus on extracting geographical entities, locations, and spatial relationships.", + "facts_user_instruction": "Extract all geographical coordinates, addresses, and location-specific data. Include all spatial and geographical information." +} +``` + +--- + +## Integration with Workflow + +User instructions are integrated into the OntoCast workflow at specific points: + +1. **Document Processing**: Instructions are extracted from JSON input during document conversion +2. **Ontology Extraction**: Instructions guide the AI when extracting domain concepts +3. **Facts Extraction**: Instructions guide the AI when extracting specific facts +4. **Critique Phase**: Instructions are used during the critique and improvement phases + +--- + +## Troubleshooting + +### Common Issues + +1. **Instructions Not Applied**: Ensure instructions are properly formatted in your JSON payload +2. **Vague Results**: Make instructions more specific and detailed +3. **Missing Data**: Check if instructions are too restrictive or unclear + +### Debug Tips + +1. **Check Logs**: Look for debug messages about user instructions in the server logs +2. **Test with Simple Instructions**: Start with basic instructions and refine +3. **Validate JSON**: Ensure your JSON payload is properly formatted + +### Example Debug Output + +``` +DEBUG - Set ontology user instruction: Focus on extracting geographical locations and organizations +DEBUG - Set facts user instruction: Extract financial data and numerical values +``` + +--- + +## API Reference + +### Request Format + +```json +{ + "text": "string", + "ontology_user_instruction": "string (optional)", + "facts_user_instruction": "string (optional)" +} +``` + +### Response Format + +The response includes the extracted ontology and facts, with user instructions influencing the extraction process: + +```json +{ + "status": "success", + "ontology": "...", + "facts": "...", + "metadata": { + "ontology_user_instruction": "Focus on extracting geographical locations and organizations", + "facts_user_instruction": "Extract financial data and numerical values" + } +} +``` + +--- + +## Best Practices Summary + +1. **Be Specific**: Provide clear, detailed instructions +2. **Use Domain Language**: Include relevant terminology +3. **Provide Context**: Explain what you're looking for +4. **Test and Refine**: Start simple and improve based on results +5. **Document Your Instructions**: Keep track of what works best for your use case + +--- + +## Examples by Domain + +### Healthcare +- **Ontology**: "Focus on medical conditions, treatments, and healthcare relationships" +- **Facts**: "Extract patient data, medical codes, and clinical measurements" + +### Finance +- **Ontology**: "Focus on financial entities, business relationships, and economic concepts" +- **Facts**: "Extract monetary amounts, financial ratios, and economic indicators" + +### Legal +- **Ontology**: "Focus on legal entities, court cases, and regulatory frameworks" +- **Facts**: "Extract case numbers, legal citations, and compliance information" + +### Scientific +- **Ontology**: "Focus on scientific concepts, methodologies, and research relationships" +- **Facts**: "Extract experimental data, measurements, and research findings" + +### Technical +- **Ontology**: "Focus on technical concepts, systems, and technological relationships" +- **Facts**: "Extract technical specifications, performance metrics, and system data" diff --git a/ontology_platform/vendored/ontocast/docs/user_guide/workflow.md b/ontology_platform/vendored/ontocast/docs/user_guide/workflow.md new file mode 100644 index 0000000..0e81ef9 --- /dev/null +++ b/ontology_platform/vendored/ontocast/docs/user_guide/workflow.md @@ -0,0 +1,107 @@ +# OntoCast Workflow + +This document describes the workflow of OntoCast's document processing pipeline. + +## Overview + +The OntoCast workflow consists of several stages that transform input documents into structured knowledge: + +1. **Document Conversion** + - Input documents are converted to markdown format + - Supports various input formats (PDF, DOCX, TXT, MD) + +2. **Text Chunking** + - Documents are split into manageable chunks + - Chunks are processed sequentially + - Head chunks are processed first to establish context + +3. **Ontology Processing** + - **Selection**: Choose appropriate ontology for content + - **Extraction**: Extract ontological concepts from text using GraphUpdate operations + - **GraphUpdate**: LLM outputs structured SPARQL operations (insert/delete) instead of full TTL + - **Update Application**: GraphUpdate operations are applied incrementally to the ontology graph + - **Sublimation**: Refine and enhance the ontology + - **Criticism**: Validate ontology structure and relationships + - **Versioning**: Automatic semantic version increment based on changes (MAJOR/MINOR/PATCH) + - **Timestamp**: Tracks last update time with `updated_at` field + +4. **Fact Processing** + - **Extraction**: Extract factual information from text using GraphUpdate operations + - **GraphUpdate**: LLM outputs structured SPARQL operations for facts updates + - **Update Application**: GraphUpdate operations are applied incrementally to the facts graph + - **Criticism**: Validate extracted facts + - **Aggregation**: Combine facts from all chunks + +## Detailed Flow + +### 1. Document Input +- Accepts text or file input +- Converts to markdown format +- Preserves document structure + +### 2. Text Processing +- Splits text into chunks +- Processes head chunks first +- Maintains context between chunks + +### 3. Ontology Management +- Selects relevant ontology +- Extracts new concepts using GraphUpdate operations (token-efficient) +- Applies incremental updates to ontology graph +- Validates relationships +- Refines structure +- Automatically increments version based on change analysis (MAJOR/MINOR/PATCH) +- Updates timestamp when ontology is modified +- Tracks version lineage with hash-based identifiers + +### 4. Fact Extraction +- Identifies entities +- Extracts relationships using GraphUpdate operations (token-efficient) +- Applies incremental updates to facts graph +- Validates facts +- Combines information from all chunks + +### 5. Output Generation +- Produces RDF graph +- Generates ontology with version and timestamp +- Provides extracted facts +- Reports budget usage (LLM calls, characters sent/received, triples generated) +- Logs budget summary at end of processing + +## Configuration Options + +The workflow can be configured through command-line parameters: + +- `--head-chunks`: Number of chunks to process first +- `--max-visits`: Maximum visits per node + +## Best Practices + +1. **Chunk Size** + - Keep chunks manageable + - Consider context preservation + - Balance between detail and processing time + +2. **Ontology Selection** + - Choose appropriate ontology + - Consider domain specificity + - Allow for ontology evolution + - Monitor version increments to track evolution + +3. **Fact Validation** + - Validate extracted facts + - Check for consistency + - Handle contradictions + +4. **Resource Management** + - Monitor memory usage + - Control processing time + - Handle large documents + - Review budget summaries to track LLM usage and costs + - Use budget metrics to estimate processing costs for large documents + - GraphUpdate operations significantly reduce token usage compared to full graph generation + - Monitor triple generation metrics to understand graph growth + +## Next Steps + +- Check [API Reference](../reference/onto.md) \ No newline at end of file diff --git a/ontology_platform/vendored/ontocast/graph.mmd b/ontology_platform/vendored/ontocast/graph.mmd new file mode 100644 index 0000000..8f11b82 --- /dev/null +++ b/ontology_platform/vendored/ontocast/graph.mmd @@ -0,0 +1,46 @@ +--- +config: + flowchart: + curve: linear + htmlLabels: true + useMaxWidth: true + look: handDrawn + theme: base + themeVariables: + fontFamily: '''Architects Daughter'', cursive' + fontSize: 20px + lineColor: '#FFAB91' + primaryBorderColor: '#143642' + primaryColor: '#FFF3E0' + primaryTextColor: '#372237' +--- +graph TD; + __start__([

__start__

]):::first + Convert\20to\20Markdown(Convert to Markdown) + Chunk\20Text(Chunk Text) + Select\20Ontology(Select Ontology) + Bootstrap\20Ontology(Bootstrap Ontology) + Update\20Ontology(Update Ontology) + Normalize\20Ontology\20Updates(Normalize Ontology Updates) + Consolidate\20Ontology(Consolidate Ontology) + Render\20Facts(Render Facts) + Merge\20Facts(Merge Facts) + Serialize(Serialize) + __end__([

__end__

]):::last + Bootstrap\20Ontology --> Update\20Ontology; + Chunk\20Text --> Select\20Ontology; + Consolidate\20Ontology -.-> Render\20Facts; + Consolidate\20Ontology -.-> Serialize; + Convert\20to\20Markdown --> Chunk\20Text; + Merge\20Facts --> Serialize; + Normalize\20Ontology\20Updates --> Consolidate\20Ontology; + Render\20Facts --> Merge\20Facts; + Select\20Ontology -.-> Bootstrap\20Ontology; + Select\20Ontology -.-> Render\20Facts; + Select\20Ontology -.-> Update\20Ontology; + Update\20Ontology --> Normalize\20Ontology\20Updates; + __start__ --> Convert\20to\20Markdown; + Serialize --> __end__; + classDef default fill:#f2f0ff,line-height:1.2 + classDef first fill-opacity:0 + classDef last fill:#bfb6fc diff --git a/ontology_platform/vendored/ontocast/logging.debug.conf b/ontology_platform/vendored/ontocast/logging.debug.conf new file mode 100644 index 0000000..85478e3 --- /dev/null +++ b/ontology_platform/vendored/ontocast/logging.debug.conf @@ -0,0 +1,22 @@ +[loggers] +keys=root + +[handlers] +keys=consoleHandler + +[formatters] +keys=consoleFormatter + +[logger_root] +level=DEBUG +handlers=consoleHandler + +[handler_consoleHandler] +class=StreamHandler +level=DEBUG +formatter=consoleFormatter +args=(sys.stdout,) + +[formatter_consoleFormatter] +format=%(asctime)s - %(levelname)s - %(name)s - %(funcName)s : %(message)s +datefmt=%Y-%m-%d %H:%M:%S diff --git a/ontology_platform/vendored/ontocast/logging.info.conf b/ontology_platform/vendored/ontocast/logging.info.conf new file mode 100644 index 0000000..cecc5e6 --- /dev/null +++ b/ontology_platform/vendored/ontocast/logging.info.conf @@ -0,0 +1,22 @@ +[loggers] +keys=root + +[handlers] +keys=consoleHandler + +[formatters] +keys=consoleFormatter + +[logger_root] +level=INFO +handlers=consoleHandler + +[handler_consoleHandler] +class=StreamHandler +level=INFO +formatter=consoleFormatter +args=(sys.stdout,) + +[formatter_consoleFormatter] +format=%(asctime)s - %(levelname)s - %(name)s - %(funcName)s : %(message)s +datefmt=%Y-%m-%d %H:%M:%S diff --git a/ontology_platform/vendored/ontocast/logging.warning.conf b/ontology_platform/vendored/ontocast/logging.warning.conf new file mode 100644 index 0000000..a4ea3f1 --- /dev/null +++ b/ontology_platform/vendored/ontocast/logging.warning.conf @@ -0,0 +1,22 @@ +[loggers] +keys=root + +[handlers] +keys=consoleHandler + +[formatters] +keys=consoleFormatter + +[logger_root] +level=WARNING +handlers=consoleHandler + +[handler_consoleHandler] +class=StreamHandler +level=WARNING +formatter=consoleFormatter +args=(sys.stdout,) + +[formatter_consoleFormatter] +format=%(asctime)s - %(levelname)s - %(name)s - %(funcName)s : %(message)s +datefmt=%Y-%m-%d %H:%M:%S diff --git a/ontology_platform/vendored/ontocast/mkdocs.yml b/ontology_platform/vendored/ontocast/mkdocs.yml new file mode 100644 index 0000000..8c95b00 --- /dev/null +++ b/ontology_platform/vendored/ontocast/mkdocs.yml @@ -0,0 +1,106 @@ +site_name: OntoCast +site_description: A tool for converting documents into RDF graphs +site_url: https://growgraph.github.io/ontocast +repo_url: https://github.com/growgraph/ontocast +repo_name: growgraph/ontocast +copyright: Copyright © 2025 GrowGraph + +theme: + name: material + user_color_mode_toggle: true + palette: + - scheme: default + primary: orange + accent: brown + toggle: + icon: material/toggle-switch-off-outline + name: Switch to dark mode + - scheme: slate + primary: brown + accent: lime + toggle: + icon: material/toggle-switch + name: Switch to light mode + logo: assets/project_logo.png + icon: + repo: fontawesome/brands/github + favicon: assets/favicon.ico + +plugins: +- search +- glightbox +- gen-files: + scripts: + - docs/gen_pages.py +- literate-nav +- mkdocstrings: + default_handler: python + handlers: + python: + options: + extra: + show_root_full_path: true + show_docstring: true + show_source: true + show_root_toc_entry: false + docstring_style: google + show_submodules: true + show_category_heading: true + show_if_no_docstring: true + setup_commands: + - "import sys" + - "sys.path.append('.')" + +nav: +- Home: index.md +- Getting Started: + - Installation: getting_started/installation.md + - Quick Start: getting_started/quickstart.md +- User Guide: + - Core Concepts: user_guide/concepts.md + - Configuration System: user_guide/configuration.md + - Triple Store Configuration: user_guide/triple_stores.md + - Workflow: user_guide/workflow.md + - User Instructions: user_guide/user_instructions.md +- API Reference: reference/ +- Contributing: contributing.md + +markdown_extensions: +- pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true +- def_list +- pymdownx.tasklist: + custom_checkbox: true +- pymdownx.inlinehilite +- pymdownx.snippets +- pymdownx.superfences +- attr_list +- md_in_html +- admonition +- codehilite +- footnotes +- meta +- pymdownx.blocks.caption +- admonition +- pymdownx.superfences +- pymdownx.details +- pymdownx.tabbed +- pymdownx.highlight +- pymdownx.emoji +- pymdownx.tasklist +- pymdownx.keys +- pymdownx.snippets +- pymdownx.magiclink +- pymdownx.betterem +- pymdownx.caret +- pymdownx.tilde +- pymdownx.mark +- pymdownx.smartsymbols +- pymdownx.arithmatex +- pymdownx.progressbar +- pymdownx.escapeall + +- toc: + permalink: true diff --git a/ontology_platform/vendored/ontocast/ontocast/__init__.py b/ontology_platform/vendored/ontocast/ontocast/__init__.py new file mode 100644 index 0000000..02f3ac7 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/__init__.py @@ -0,0 +1,20 @@ +"""OntoCast: Agentic ontology-assisted framework for semantic triple extraction. + +OntoCast is a comprehensive framework for extracting semantic triples from +documents using ontology assistance. It provides a complete pipeline for +document processing, ontology management, and knowledge graph construction. + +The framework includes: +- Document conversion and chunking +- Ontology selection and management +- Fact extraction and validation +- Triple store integration (Neo4j, Fuseki, Filesystem) +- LLM-powered semantic analysis +- REST API server for document processing + +For more information, see the documentation at https://growgraph.github.io/ontocast/ +""" + +from ontocast.stategraph.atomic import facts_loop, ontology_loop + +__all__ = ["facts_loop", "ontology_loop"] diff --git a/ontology_platform/vendored/ontocast/ontocast/agent/__init__.py b/ontology_platform/vendored/ontocast/ontocast/agent/__init__.py new file mode 100644 index 0000000..422f0d2 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/agent/__init__.py @@ -0,0 +1,31 @@ +"""Agent module for OntoCast. + +This module provides a collection of agents that handle various aspects of ontology +processing, including document conversion, text chunking, fact aggregation, and +ontology management. Each agent is designed to perform a specific task in the +ontology processing pipeline. +""" + +from .chunk_text import chunk_text +from .convert_document import convert_document +from .criticise_facts import criticise_facts +from .criticise_ontology import criticise_ontology +from .render_facts import render_facts, render_facts_fresh +from .render_ontology import render_ontology, render_ontology_fresh +from .select_ontology import select_ontology +from .serialize import serialize +from .sublimate_ontology import sublimate_ontology + +__all__ = [ + "chunk_text", + "convert_document", + "criticise_facts", + "criticise_ontology", + "render_facts", + "render_ontology", + "select_ontology", + "serialize", + "sublimate_ontology", + "render_ontology_fresh", + "render_facts_fresh", +] diff --git a/ontology_platform/vendored/ontocast/ontocast/agent/chunk_text.py b/ontology_platform/vendored/ontocast/ontocast/agent/chunk_text.py new file mode 100644 index 0000000..4dfcb53 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/agent/chunk_text.py @@ -0,0 +1,61 @@ +"""Text chunking agent for OntoCast. + +This module provides functionality for splitting text into manageable chunks +that can be processed independently, ensuring optimal processing of large +documents. +""" + +import logging + +from ontocast.onto.content_unit import ContentUnit +from ontocast.onto.enum import Status +from ontocast.onto.state import AgentState +from ontocast.toolbox import ToolBox + +logger = logging.getLogger(__name__) + + +def chunk_text(state: AgentState, tools: ToolBox) -> AgentState: + """Split text into manageable chunks. + + This function takes the converted document text and splits it into smaller, + manageable chunks that can be processed independently. + + Args: + state: The current agent state containing the text to chunk. + tools: The toolbox instance providing utility functions. + + Returns: + AgentState: Updated state with text chunks. + """ + logger.info("Chunking the text") + if state.input_text is not None: + chunks_txt: list[str] = tools.chunker(state.input_text) + logger.info( + f"Created {len(chunks_txt)} chunks for processing: {[len(c) for c in chunks_txt]}" + ) + + if state.max_chunks is not None: + logger.info(f"Selecting {state.max_chunks} chunks") + + chunks_txt = chunks_txt[: state.max_chunks] + + for i, chunk_txt in enumerate(chunks_txt): + state.content_units.append( + ContentUnit( + text=chunk_txt, + index=i, + doc_iri=state.doc_iri, + ) + ) + + logger.info( + "Created " + f"{len(state.content_units)} content units for processing: " + f"{[len(c) for c in state.content_units]}" + ) + state.status = Status.SUCCESS + else: + state.status = Status.FAILED + + return state diff --git a/ontology_platform/vendored/ontocast/ontocast/agent/common.py b/ontology_platform/vendored/ontocast/ontocast/agent/common.py new file mode 100644 index 0000000..5fe7bef --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/agent/common.py @@ -0,0 +1,151 @@ +import logging +from typing import Any, TypeVar + +from langchain_core.output_parsers import BaseOutputParser +from langchain_core.prompts import BasePromptTemplate + +from ontocast.onto.enum import WorkflowNode +from ontocast.onto.model import Suggestions +from ontocast.prompt.common import ( + suggestion_concrete_template, + suggestion_general_template, +) +from ontocast.prompt.render_facts import ( + improvement_instruction_template as facts_template, +) +from ontocast.prompt.render_ontology import ( + improvement_instruction_template as ontology_template, +) +from ontocast.tool import LLMTool + +logger = logging.getLogger(__name__) + +T = TypeVar("T") + + +def render_suggestions_prompt(suggestions: Suggestions, stage: WorkflowNode) -> str: + """Generate prompt templates from the suggestions. + + Returns: + Combined string with general and concrete templates. + Returns empty string if both fields are empty. + """ + + # Generate general template if systemic_critique_summary is not empty + general_template = "" + if suggestions.systemic_critique_summary.strip(): + general_template = suggestion_general_template.format( + general_suggestion=suggestions.systemic_critique_summary + ) + + concrete_template = "" + if suggestions.actionable_fixes: + # Generate concrete template if actionable_fixes is not empty + concrete_template = suggestion_concrete_template.format( + suggestion_str=suggestions.to_markdown() + ) + + if stage == WorkflowNode.TEXT_TO_FACTS: + template = facts_template + elif stage == WorkflowNode.TEXT_TO_ONTOLOGY: + template = ontology_template + else: + raise ValueError(f"Stage {stage} not supported") + if general_template or concrete_template: + final_prompt = template.format( + suggestions_instruction=f"\n\n{general_template}\n\n{concrete_template}" + ) + else: + final_prompt = "" + return final_prompt + + +async def call_llm_with_retry( + llm_tool: LLMTool, + prompt: BasePromptTemplate, + parser: BaseOutputParser[T], + prompt_kwargs: dict[str, Any], + max_retries: int = 3, + retry_error_feedback: bool = True, +) -> T: + """Call LLM and parse response with automatic retry on parsing failures. + + This utility function implements a common pattern across agent functions: + 1. Call LLM with a prompt + 2. Parse the response + 3. Retry if parsing fails (up to max_retries times) + + On retry, if retry_error_feedback is True, the error message from the previous + attempt is included in the prompt to help the LLM correct its output format. + + Args: + llm_tool: The LLM tool instance to use for generation. + prompt: The prompt template to format and send to the LLM. + parser: The output parser to parse the LLM response. + prompt_kwargs: Keyword arguments to pass to prompt.format_prompt(). + max_retries: Maximum number of retry attempts (default: 3). + retry_error_feedback: Whether to include error feedback in retry prompts (default: True). + + Returns: + The parsed output of type T. + + Raises: + Exception: If parsing fails after all retry attempts, raises the last parsing error. + """ + last_error: Exception | None = None + last_sanitized_content: str | None = None + original_format_instructions = prompt_kwargs.get("format_instructions", "") + + for attempt in range(max_retries): + try: + # Create a copy of prompt_kwargs for this attempt + attempt_kwargs = prompt_kwargs.copy() + + # On retry, add error feedback to help LLM correct format + if attempt > 0 and retry_error_feedback and last_error is not None: + # Use sanitized content in error feedback for consistency + feedback_content = ( + last_sanitized_content if last_sanitized_content else "" + ) + error_feedback = ( + f"\n\nIMPORTANT: The previous attempt failed to parse the response. " + f"Error: {str(last_error)}\n" + f"Previous response (for reference):\n{feedback_content}\n\n" + f"Please ensure your response strictly follows the format instructions " + f"and does not contain any control characters or invalid syntax." + ) + # Add error feedback to format_instructions if present + if "format_instructions" in attempt_kwargs: + attempt_kwargs["format_instructions"] = ( + original_format_instructions + error_feedback + ) + else: + # If no format_instructions, add as a new field + attempt_kwargs["parsing_error_feedback"] = error_feedback + + # Call LLM + response = await llm_tool(prompt.format_prompt(**attempt_kwargs)) + content_to_parse = response.content + + parsed = parser.parse(content_to_parse) + logger.debug( + f"Successfully parsed LLM response on attempt {attempt + 1}/{max_retries}" + ) + return parsed + + except Exception as e: + last_error = e + logger.warning( + f"Failed to parse LLM response on attempt {attempt + 1}/{max_retries}: {str(e)}" + ) + + # If this was the last attempt, raise the error + if attempt == max_retries - 1: + logger.error( + f"Failed to parse LLM response after {max_retries} attempts. " + f"Last error: {str(e)}" + ) + raise + + # This should never be reached, but type checker needs it + raise RuntimeError("Unexpected error in call_llm_with_retry") diff --git a/ontology_platform/vendored/ontocast/ontocast/agent/convert_document.py b/ontology_platform/vendored/ontocast/ontocast/agent/convert_document.py new file mode 100644 index 0000000..e7e5ffe --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/agent/convert_document.py @@ -0,0 +1,132 @@ +# MODIFIED 2026-05-13: Multi-file corpus support. +# The original code looped over all files but overwrote `state.input_text` on +# every iteration, so only the last file's content survived. We now accumulate +# all files into a single corpus with explicit file-boundary separators while +# keeping the single-file behavior byte-identical (no separator overhead when +# files.items() has length 1). User instructions and source URL keep their +# first-found value (corpus-level metadata is treated as singleton). +# See: docs/통합설계서.md §5 Phase 0 and OntoCast 분석 §13.1, §21.1 (#4). +"""Document conversion agent for OntoCast. + +This module provides functionality for converting various document formats into +structured data that can be processed by the OntoCast system. +""" + +import json +import logging +import pathlib + +from ontocast.onto.enum import Status +from ontocast.onto.state import AgentState +from ontocast.toolbox import ToolBox + +logger = logging.getLogger(__name__) + + +def _file_boundary(filename: str) -> str: + """Return an explicit, parseable boundary marker for corpus concatenation.""" + return f"\n\n=== File: {filename} ===\n\n" + + +def convert_document(state: AgentState, tools: ToolBox) -> AgentState: + """Convert one or more documents into a single text corpus on `state`. + + Supports a mix of converter-recognized binary documents (PDF/DOCX/...), + JSON envelopes that may carry `text`, `url`, and user-instruction fields, + and plain `.txt` content. Each file's extracted text is appended to a + corpus separated by an explicit boundary marker so downstream chunking + can still observe file edges if needed. + + Behavior contract: + - Single file (one entry in `state.files`): the corpus equals the + file's text with no boundary marker (byte-identical to the legacy + single-file flow apart from earlier overwrite semantics). + - Multiple files: their texts are concatenated in iteration order with + `_file_boundary(filename)` between them. + - `ontology_user_instruction`, `facts_user_instruction`, `source_url` + come from the first JSON envelope that supplies them; later files + do not overwrite them. + - On the first unsupported extension the function fails fast (returns + with Status.FAILED), matching the original behavior. + + Args: + state: The current agent state. `state.files` is the input. + tools: The toolbox instance providing the converter tool. + + Returns: + AgentState: Updated state. `state.input_text` holds the corpus. + """ + state.status = Status.SUCCESS + files = state.files + + if not files: + logger.debug("convert_document: no files to process") + return state + + multi_file = len(files) > 1 + logger.debug( + f"convert_document: processing {len(files)} file(s)" + + (" as a single corpus with boundary markers" if multi_file else "") + ) + + texts: list[str] = [] + for filename, file_content in files.items(): + file_extension = pathlib.Path(filename).suffix.lower() + logger.debug(f"file ext: {file_extension}, {filename}") + + if file_extension in tools.converter.supported_extensions: + logger.debug("will apply converter") + result = tools.converter(file_content) + elif file_extension == ".json": + result = json.loads(file_content.decode("utf-8")) + + # Corpus-level metadata: first JSON wins so multiple JSON files + # in a single batch don't quietly fight over instructions. + ontology_user_instruction = result.get("ontology_user_instruction", "") + facts_user_instruction = result.get("facts_user_instruction", "") + source_url = result.get("url", None) + + if ontology_user_instruction and not state.ontology_user_instruction: + state.ontology_user_instruction = ontology_user_instruction + logger.debug( + f"Set ontology user instruction from {filename}: " + f"{ontology_user_instruction}" + ) + if facts_user_instruction and not state.facts_user_instruction: + state.facts_user_instruction = facts_user_instruction + logger.debug( + f"Set facts user instruction from {filename}: " + f"{facts_user_instruction}" + ) + if source_url and not state.source_url: + state.source_url = source_url + logger.debug(f"Extracted source URL from {filename}: {source_url}") + + elif file_extension == ".txt": + # Legacy: original code applied `json.loads` to the decoded text, + # which only works when the .txt content happens to be a JSON + # string literal. Preserve that contract for backward compatibility. + result = {"text": json.loads(file_content.decode("utf-8"))} + else: + logger.error(f"Unsupported file extension: {file_extension} ({filename})") + state.status = Status.FAILED + return state + + text = result.get("text", "") + if not isinstance(text, str): + logger.error( + f"Converter result for {filename} has non-string 'text' " + f"of type {type(text).__name__}; skipping" + ) + state.status = Status.FAILED + return state + + if multi_file: + texts.append(_file_boundary(filename)) + texts.append(text) + + # Stitch and commit. For a single file we end up with exactly its text; + # for multiple files boundary markers are interleaved. + corpus = "".join(texts).lstrip("\n") if multi_file else (texts[0] if texts else "") + state.set_text(corpus) + return state diff --git a/ontology_platform/vendored/ontocast/ontocast/agent/criticise_facts.py b/ontology_platform/vendored/ontocast/ontocast/agent/criticise_facts.py new file mode 100644 index 0000000..1d3e5b3 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/agent/criticise_facts.py @@ -0,0 +1,134 @@ +"""Enhanced fact criticism agent with memory and SPARQL operations. + +This module provides enhanced functionality for analyzing and validating facts +with SPARQL operation support. +""" + +import logging + +from langchain_core.output_parsers import PydanticOutputParser +from langchain_core.prompts import PromptTemplate + +from ontocast.agent.common import call_llm_with_retry +from ontocast.onto.enum import FailureStage, Status, WorkflowNode +from ontocast.onto.model import FactsCritiqueReport, Suggestions +from ontocast.onto.unit_states import UnitFactsState +from ontocast.prompt.common import ( + facts_template, + ontology_template, + text_template, + user_template, +) +from ontocast.prompt.criticise_facts import ( + evaluation_instruction, + preamble, + template_prompt, +) +from ontocast.tool.atomic import AtomicToolBox + +logger = logging.getLogger(__name__) + + +async def criticise_facts( + state: UnitFactsState, tools: AtomicToolBox +) -> UnitFactsState: + """Enhanced criticize facts with SPARQL operations. + + This function performs a critical analysis of the facts in the current content unit, + with SPARQL operation support. + + Args: + state: The current unit facts state containing the chunk to analyze. + tools: The toolbox instance providing utility functions. + + Returns: + UnitFactsState: Updated state with analysis results. + """ + if not state.content_unit: + logger.warning("No current content unit to analyze") + return state + + progress_info = state.get_content_unit_progress_string() + logger.info( + f"Facts critic for {progress_info}: visit {state.node_visits[WorkflowNode.CRITICISE_FACTS]}/{state.max_visits_per_node}" + ) + + llm_tool = await tools.get_llm_tool(state.budget_tracker) + parser = PydanticOutputParser(pydantic_object=FactsCritiqueReport) + + ontology_ttl = state.ontology_snapshot.graph.serialize(format="turtle") + + ontology_chapter = ontology_template.format( + ontology_ttl=ontology_ttl, + ) + + facts_ttl = state.content_unit.graph.serialize(format="turtle") + + facts_chapter = facts_template.format( + facts_ttl=facts_ttl, + ) + + text_chapter = text_template.format(text=state.content_unit.text) + + user_instruction = ( + user_template.format(user_instruction=state.facts_user_instruction) + if state.facts_user_instruction + else "" + ) + + prompt = PromptTemplate( + template=template_prompt, + input_variables=[ + "preamble", + "evaluation_instruction", + "user_instruction", + "ontology_chapter", + "facts_chapter", + "text_chapter", + "format_instructions", + ], + ) + + prompt_data = { + "preamble": preamble, + "evaluation_instruction": evaluation_instruction, + "user_instruction": user_instruction, + "ontology_chapter": ontology_chapter, + "facts_chapter": facts_chapter, + "text_chapter": text_chapter, + "format_instructions": parser.get_format_instructions(), + } + + try: + critique: FactsCritiqueReport = await call_llm_with_retry( + llm_tool=llm_tool, + prompt=prompt, + parser=parser, + prompt_kwargs=prompt_data, + ) + state.set_external_evidence_request( + WorkflowNode.CRITICISE_FACTS, critique.external_evidence_request + ) + logger.debug( + f"Parsed critique report - success: {critique.success}, " + f"score: {critique.score}" + ) + + if critique.success or critique.score > 90: + state.status = Status.SUCCESS + state.set_node_status(WorkflowNode.CRITICISE_FACTS, Status.SUCCESS) + logger.info("Facts critique passed") + else: + state.status = Status.FAILED + state.set_node_status(WorkflowNode.CRITICISE_FACTS, Status.FAILED) + state.failure_stage = FailureStage.FACTS_CRITIQUE + state.suggestions = Suggestions.from_critique_report(critique) + state.failure_reason = "Facts Critic suggests improvements" + + return state + + except Exception as e: + logger.error(f"Failed to criticize facts: {str(e)}") + state.set_failure(FailureStage.FACTS_CRITIQUE, str(e)) + state.set_node_status(WorkflowNode.CRITICISE_FACTS, Status.FAILED) + return state diff --git a/ontology_platform/vendored/ontocast/ontocast/agent/criticise_ontology.py b/ontology_platform/vendored/ontocast/ontocast/agent/criticise_ontology.py new file mode 100644 index 0000000..3c51824 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/agent/criticise_ontology.py @@ -0,0 +1,132 @@ +"""Enhanced ontology criticism agent with SPARQL operations. + +This module provides enhanced functionality for analyzing and validating ontologies of previous critiques and SPARQL operation support. +""" + +import logging + +from langchain_core.output_parsers import PydanticOutputParser +from langchain_core.prompts import PromptTemplate + +from ontocast.agent.common import call_llm_with_retry +from ontocast.onto.enum import FailureStage, Status, WorkflowNode +from ontocast.onto.model import OntologyCritiqueReport, Suggestions +from ontocast.onto.unit_states import UnitOntologyState +from ontocast.prompt.common import ontology_template, text_template +from ontocast.prompt.common import system_preamble_ontology as system_preamble +from ontocast.prompt.criticise_ontology import ( + intro_instruction, + ontology_criteria, + template_prompt, +) +from ontocast.tool import LLMTool +from ontocast.tool.atomic import AtomicToolBox + +logger = logging.getLogger(__name__) + + +async def criticise_ontology( + state: UnitOntologyState, tools: AtomicToolBox +) -> UnitOntologyState: + """Enhanced ontology criticism with SPARQL operations. + + This function performs a critical analysis of the ontology in the current + state, with SPARQL operation support. + + Args: + state: The current unit ontology state containing the ontology to analyze. + tools: The toolbox instance providing utility functions. + + Returns: + UnitOntologyState: Updated state with analysis results. + """ + + progress_info = state.get_content_unit_progress_string() + logger.info( + f"Ontology Critic for {progress_info}: visit {state.node_visits[WorkflowNode.CRITICISE_ONTOLOGY]}/{state.max_visits_per_node}" + ) + + if state.content_unit is None: + state.status = Status.FAILED + return state + + current = state.current_ontology or state.ontology_snapshot + if current.is_null(): + raise ValueError( + f"Null ontology cannot be criticised: {current.iri} is not a valid ontology" + ) + + parser = PydanticOutputParser(pydantic_object=OntologyCritiqueReport) + llm_tool: LLMTool = await tools.get_llm_tool(state.budget_tracker) + + ontology_ttl = current.graph.serialize(format="turtle") + + ontology_chapter = ontology_template.format( + ontology_ttl=ontology_ttl, + ) + + text_chapter = text_template.format(text=state.content_unit.text) + + user_instruction = state.ontology_user_instruction + external_evidence = state.external_evidence_text + if external_evidence: + state.mark_external_evidence_used(WorkflowNode.CRITICISE_ONTOLOGY) + + prompt = PromptTemplate( + template=template_prompt, + input_variables=[ + "preamble", + "intro_instruction", + "ontology_criteria", + "user_instruction", + "ontology_chapter", + "text_chapter", + "external_evidence", + "format_instructions", + ], + ) + + try: + critique: OntologyCritiqueReport = await call_llm_with_retry( + llm_tool=llm_tool, + prompt=prompt, + parser=parser, + prompt_kwargs={ + "preamble": system_preamble, + "intro_instruction": intro_instruction, + "ontology_criteria": ontology_criteria, + "text_chapter": text_chapter, + "user_instruction": user_instruction, + "ontology_chapter": ontology_chapter, + "external_evidence": external_evidence, + "format_instructions": parser.get_format_instructions(), + }, + ) + state.set_external_evidence_request( + WorkflowNode.CRITICISE_ONTOLOGY, critique.external_evidence_request + ) + logger.info( + f"Parsed critique report - success: {critique.success}, " + f"score: {critique.score}, n fixes: {len(critique.actionable_ontology_fixes)}." + ) + + if critique.success or critique.score > 90: + state.status = Status.SUCCESS + state.set_node_status(WorkflowNode.CRITICISE_ONTOLOGY, Status.SUCCESS) + logger.info("Ontology critique passed") + else: + state.status = Status.FAILED + state.failure_stage = FailureStage.ONTOLOGY_CRITIQUE + state.set_node_status(WorkflowNode.CRITICISE_ONTOLOGY, Status.FAILED) + state.suggestions = Suggestions.from_critique_report(critique) + state.failure_reason = "Ontology Critic suggests improvements" + logger.info( + f"Ontology critique failed: {critique.systemic_critique_summary}" + ) + return state + + except Exception as e: + logger.error(f"Failed to critique ontology: {str(e)}") + state.set_failure(FailureStage.ONTOLOGY_CRITIQUE, str(e)) + state.set_node_status(WorkflowNode.CRITICISE_ONTOLOGY, Status.FAILED) + return state diff --git a/ontology_platform/vendored/ontocast/ontocast/agent/external_evidence.py b/ontology_platform/vendored/ontocast/ontocast/agent/external_evidence.py new file mode 100644 index 0000000..24eeed9 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/agent/external_evidence.py @@ -0,0 +1,448 @@ +"""Helpers for optional web-grounded prompts with explicit plan/fetch steps.""" + +import logging +from typing import TypeVar +from urllib.parse import urlparse + +from langchain_core.output_parsers import PydanticOutputParser +from langchain_core.prompts import PromptTemplate + +from ontocast.agent.common import call_llm_with_retry +from ontocast.onto.enum import WorkflowNode +from ontocast.onto.model import ( + ExternalEvidenceCacheEntry, + ExternalEvidenceHit, + ExternalEvidencePlan, + ExternalEvidenceRequest, +) +from ontocast.onto.unit_states import UnitFactsState, UnitOntologyState +from ontocast.tool.atomic import AtomicToolBox, SearchHit + +logger = logging.getLogger(__name__) +UnitStateT = TypeVar("UnitStateT", UnitFactsState, UnitOntologyState) + +_planner_template = """ +You are planning optional web-search grounding for a knowledge-graph workflow. +Decide conservatively whether external web evidence is necessary. + +Target workflow node: +{target_node} + +Source text: +{content_text} + +User instruction: +{user_instruction} + +Node request rationale: +{search_rationale} + +Node query hints: +{query_hints} + +Rules: +1. Prefer NOT searching unless there is genuine ambiguity, domain-standard uncertainty, + or term disambiguation need. +2. If searching, propose short, focused queries, not broad summaries of the entire text. +3. Never propose more than {max_queries} queries. +4. If no search is needed, set should_search=false, intent=\"none\", and queries=[]. + +{format_instructions} +""" + + +def _get_int(tools: AtomicToolBox, key: str, default: int) -> int: + value = getattr(tools, key, default) + return int(value) if isinstance(value, int | float) else default + + +def _get_float(tools: AtomicToolBox, key: str, default: float) -> float: + value = getattr(tools, key, default) + return float(value) if isinstance(value, int | float) else default + + +def _get_bool(tools: AtomicToolBox, key: str, default: bool) -> bool: + value = getattr(tools, key, default) + return bool(value) if isinstance(value, bool) else default + + +def _get_set(tools: AtomicToolBox, key: str) -> set[str]: + value = getattr(tools, key, set()) + if isinstance(value, set): + return {str(entry).strip().lower() for entry in value if str(entry).strip()} + return set() + + +def _web_grounding_enabled_for_node( + tools: AtomicToolBox, target_node: WorkflowNode +) -> bool: + checker = getattr(tools, "web_grounding_enabled_for_node", None) + if checker is None or not callable(checker): + return False + return bool(checker(target_node)) + + +def build_evidence_query( + content_text: str, user_instruction: str, max_chars: int = 220 +) -> str: + """Backward-compatible fallback query from content and user guidance.""" + source = user_instruction.strip() if user_instruction.strip() else content_text + query = " ".join(source.split()) + return query[:max_chars].strip() + + +def _resolve_user_instruction(state: UnitFactsState | UnitOntologyState) -> str: + if isinstance(state, UnitOntologyState): + return state.ontology_user_instruction + return state.facts_user_instruction + + +def _resolve_content_text(state: UnitFactsState | UnitOntologyState) -> str: + return state.content_unit.text + + +def _resolve_search_request( + state: UnitFactsState | UnitOntologyState, target_node: WorkflowNode +) -> ExternalEvidenceRequest: + return state.get_external_evidence_request(target_node) + + +def _normalize_query(query: str) -> str: + return " ".join(query.split()).strip() + + +def _extract_domain(url: str) -> str: + parsed = urlparse(url) + domain = parsed.netloc.lower() + if domain.startswith("www."): + return domain[4:] + return domain + + +def _domain_matches(domain: str, patterns: set[str]) -> bool: + for pattern in patterns: + if domain == pattern or domain.endswith(f".{pattern}"): + return True + return False + + +def sanitize_external_evidence_plan( + plan: ExternalEvidencePlan, tools: AtomicToolBox +) -> ExternalEvidencePlan: + """Apply deterministic guardrails to planner output.""" + deduped_queries: list[str] = [] + seen_queries: set[str] = set() + min_chars = max(3, _get_int(tools, "web_search_planner_min_query_chars", 12)) + for raw_query in plan.queries: + query = _normalize_query(raw_query) + if len(query) < min_chars: + continue + alpha_chars = sum(1 for char in query if char.isalpha()) + if alpha_chars < max(4, min_chars // 2): + continue + lowered = query.lower() + if lowered in seen_queries: + continue + deduped_queries.append(query) + seen_queries.add(lowered) + + max_queries = max(1, _get_int(tools, "web_search_planner_max_queries", 3)) + min_confidence = _get_float(tools, "web_search_planner_min_confidence", 0.35) + deduped_queries = deduped_queries[:max_queries] + should_search = ( + plan.should_search + and plan.intent != "none" + and plan.confidence >= min_confidence + and len(deduped_queries) > 0 + ) + return ExternalEvidencePlan( + should_search=should_search, + rationale=plan.rationale, + intent=plan.intent if should_search else "none", + confidence=plan.confidence, + queries=deduped_queries if should_search else [], + ) + + +def normalize_search_hits( + hits: list[SearchHit], tools: AtomicToolBox +) -> list[ExternalEvidenceHit]: + """Filter and normalize search hits with deterministic quality checks.""" + normalized_hits: list[ExternalEvidenceHit] = [] + seen_urls: set[str] = set() + allowed_domains = _get_set(tools, "web_search_allowed_domains") + blocked_domains = _get_set(tools, "web_search_blocked_domains") + min_snippet_chars = max(0, _get_int(tools, "web_search_min_snippet_chars", 40)) + + for hit in hits: + url = hit.url.strip() + if not url or url in seen_urls: + continue + domain = _extract_domain(url) + if not domain: + continue + if blocked_domains and _domain_matches(domain, blocked_domains): + continue + if allowed_domains and not _domain_matches(domain, allowed_domains): + continue + + snippet = " ".join(hit.snippet.split()).strip() + if len(snippet) < min_snippet_chars: + continue + + seen_urls.add(url) + normalized_hits.append( + ExternalEvidenceHit( + title=hit.title.strip() or url, + url=url, + snippet=snippet, + domain=domain, + ) + ) + + return normalized_hits + + +async def plan_external_evidence_for_node( + state: UnitStateT, tools: AtomicToolBox, target_node: WorkflowNode +) -> UnitStateT: + """Plan evidence retrieval for a workflow node using LLM + guardrails.""" + state.node_visits[WorkflowNode.PLAN_EXTERNAL_EVIDENCE] += 1 + + if not _web_grounding_enabled_for_node(tools, target_node): + state.set_external_evidence_cache_entry( + target_node, ExternalEvidenceCacheEntry() + ) + state.external_evidence_hits = [] + state.external_evidence_text = "" + state.external_evidence_source_count = 0 + state.external_evidence_domains = [] + return state + + request = _resolve_search_request(state, target_node) + if not request.initiate_search: + state.set_external_evidence_cache_entry( + target_node, ExternalEvidenceCacheEntry() + ) + state.external_evidence_hits = [] + state.external_evidence_text = "" + state.external_evidence_source_count = 0 + state.external_evidence_domains = [] + state.external_evidence_planned_at_node = target_node + return state + + cached_entry = state.get_external_evidence_cache_entry(target_node) + if ( + _get_bool(tools, "web_search_reuse_evidence_across_attempt", True) + and cached_entry.text + and cached_entry.plan.should_search + ): + state.load_external_evidence_for_node(target_node) + return state + + user_instruction = _resolve_user_instruction(state) + content_text = _resolve_content_text(state) + + if not _get_bool(tools, "web_search_planner_enabled", True): + fallback_query = build_evidence_query( + content_text=content_text, user_instruction=user_instruction + ) + fallback_plan = ExternalEvidencePlan( + should_search=bool(fallback_query) or bool(request.query_hints), + rationale="Planner disabled; fallback query from content/instruction.", + intent="background", + confidence=1.0, + queries=[ + *([fallback_query] if fallback_query else []), + *request.query_hints, + ], + ) + sanitized_plan = sanitize_external_evidence_plan(fallback_plan, tools) + state.set_external_evidence_cache_entry( + target_node, + ExternalEvidenceCacheEntry( + plan=sanitized_plan, + hits=[], + text="", + source_count=0, + domains=[], + ), + ) + state.load_external_evidence_for_node(target_node) + return state + + parser = PydanticOutputParser(pydantic_object=ExternalEvidencePlan) + prompt = PromptTemplate( + template=_planner_template, + input_variables=[ + "target_node", + "content_text", + "user_instruction", + "max_queries", + "search_rationale", + "query_hints", + "format_instructions", + ], + ) + llm_tool = await tools.get_llm_tool(state.budget_tracker) + try: + planned: ExternalEvidencePlan = await call_llm_with_retry( + llm_tool=llm_tool, + prompt=prompt, + parser=parser, + prompt_kwargs={ + "target_node": target_node.value, + "content_text": content_text, + "user_instruction": user_instruction, + "max_queries": str( + max(1, _get_int(tools, "web_search_planner_max_queries", 3)) + ), + "search_rationale": request.rationale or "none", + "query_hints": ( + "\n".join(f"- {hint}" for hint in request.query_hints) + if request.query_hints + else "none" + ), + "format_instructions": parser.get_format_instructions(), + }, + ) + except Exception as error: + logger.warning( + "Evidence planner failed for %s; skipping external evidence (%s).", + target_node.value, + str(error), + ) + planned = ExternalEvidencePlan( + should_search=False, + rationale="Planner failure fallback: skip search.", + intent="none", + confidence=0.0, + queries=[], + ) + + merged_plan = ExternalEvidencePlan( + should_search=planned.should_search or bool(request.query_hints), + rationale=planned.rationale or request.rationale, + intent=planned.intent, + confidence=max(planned.confidence, request.confidence), + queries=[*planned.queries, *request.query_hints], + ) + sanitized_plan = sanitize_external_evidence_plan(merged_plan, tools) + state.set_external_evidence_cache_entry( + target_node, + ExternalEvidenceCacheEntry( + plan=sanitized_plan, + hits=[], + text="", + source_count=0, + domains=[], + ), + ) + state.load_external_evidence_for_node(target_node) + return state + + +async def fetch_external_evidence_for_node( + state: UnitStateT, tools: AtomicToolBox, target_node: WorkflowNode +) -> UnitStateT: + """Fetch and render evidence for a previously planned workflow node.""" + state.node_visits[WorkflowNode.FETCH_EXTERNAL_EVIDENCE] += 1 + + if not _web_grounding_enabled_for_node(tools, target_node): + state.external_evidence_hits = [] + state.external_evidence_text = "" + state.external_evidence_source_count = 0 + state.external_evidence_domains = [] + return state + + request = _resolve_search_request(state, target_node) + if not request.initiate_search: + state.set_external_evidence_cache_entry( + target_node, ExternalEvidenceCacheEntry() + ) + state.external_evidence_hits = [] + state.external_evidence_text = "" + state.external_evidence_source_count = 0 + state.external_evidence_domains = [] + state.external_evidence_planned_at_node = target_node + return state + + cache_entry = state.get_external_evidence_cache_entry(target_node) + plan = cache_entry.plan + if ( + _get_bool(tools, "web_search_reuse_evidence_across_attempt", True) + and cache_entry.text + and plan.should_search + ): + state.load_external_evidence_for_node(target_node) + return state + + if not plan.should_search or not plan.queries: + state.set_external_evidence_cache_entry( + target_node, ExternalEvidenceCacheEntry() + ) + state.load_external_evidence_for_node(target_node) + return state + + combined_hits: list[SearchHit] = [] + for query in plan.queries: + search_hits = await tools.search(query) + combined_hits.extend(search_hits) + + normalized_hits = normalize_search_hits(combined_hits, tools) + evidence_text = render_external_evidence( + hits=normalized_hits, + max_snippet_chars=max(40, _get_int(tools, "web_search_max_snippet_chars", 400)), + max_total_chars=max(200, _get_int(tools, "web_search_max_total_chars", 1800)), + ) + state.set_external_evidence_cache_entry( + target_node, + ExternalEvidenceCacheEntry( + plan=plan, + hits=normalized_hits, + text=evidence_text, + source_count=len(normalized_hits), + domains=sorted({hit.domain for hit in normalized_hits}), + ), + ) + state.load_external_evidence_for_node(target_node) + return state + + +def render_external_evidence( + hits: list[ExternalEvidenceHit], + max_snippet_chars: int, + max_total_chars: int, +) -> str: + """Render bounded external evidence as a prompt chapter.""" + if not hits: + return "" + + rendered_lines: list[str] = [] + remaining_chars = max_total_chars + for index, hit in enumerate(hits, start=1): + clean_snippet = " ".join(hit.snippet.split()) + if len(clean_snippet) > max_snippet_chars: + clean_snippet = f"{clean_snippet[: max_snippet_chars - 3]}..." + + line = f"{index}. {hit.title} | {hit.url}\n {clean_snippet}" + if len(line) > remaining_chars: + if remaining_chars < 80: + break + truncated = line[: remaining_chars - 3].rstrip() + line = f"{truncated}..." + rendered_lines.append(line) + break + + rendered_lines.append(line) + remaining_chars -= len(line) + + if not rendered_lines: + return "" + + return ( + "### EXTERNAL EVIDENCE (WEB SEARCH)\n" + "Use these sources to clarify uncertain terms or standards only.\n" + "When evidence conflicts, prioritize the source text and ontology context.\n\n" + f"{chr(10).join(rendered_lines)}" + ) diff --git a/ontology_platform/vendored/ontocast/ontocast/agent/normalize_ontology.py b/ontology_platform/vendored/ontocast/ontocast/agent/normalize_ontology.py new file mode 100644 index 0000000..5b8163e --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/agent/normalize_ontology.py @@ -0,0 +1,187 @@ +"""Reducers for parallel map/reduce workflow outputs.""" + +import logging + +from rdflib import OWL, RDF, BNode, Node, URIRef + +from ontocast.onto.constants import PROV, RDF_REIFIES, SCHEMA +from ontocast.onto.content_unit import ContentUnit +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.sparql_models import GraphUpdate, TripleOp +from ontocast.onto.state import AgentState +from ontocast.toolbox import ToolBox + +logger = logging.getLogger(__name__) + + +def split_ontology_and_provenance_graph( + graph: RDFGraph, +) -> tuple[RDFGraph, RDFGraph]: + """Split normalized ontology graph into clean ontology + provenance artifact. + + Provenance/reification and normalization-time alignment artifacts are moved + to a side graph so downstream consolidation works with a clean ontology graph. + """ + clean_graph = RDFGraph() + provenance_graph = RDFGraph() + + for prefix, namespace in graph.namespaces(): + if prefix: + clean_graph.bind(prefix, namespace) + provenance_graph.bind(prefix, namespace) + + reifier_nodes: set[BNode] = { + subject + for subject, _, _ in graph.triples((None, RDF_REIFIES, None)) + if isinstance(subject, BNode) + } + chunk_nodes: set[Node] = set() + + def is_schema_chunk_metadata(predicate: Node) -> bool: + predicate_str = str(predicate) + return predicate_str in { + str(SCHEMA.identifier), + str(SCHEMA.position), + "http://schema.org/identifier", + "http://schema.org/position", + } + + for subject, predicate, obj in graph: + if is_schema_chunk_metadata(predicate) or predicate == PROV.generatedAtTime: + chunk_nodes.add(subject) + if predicate == RDF.type and str(obj) in { + str(PROV.Entity), + str(SCHEMA.text), + "http://schema.org/text", + }: + chunk_nodes.add(subject) + + def is_provenance_or_alignment_triple( + subject: Node, predicate: Node, obj: Node + ) -> bool: + if predicate == RDF_REIFIES: + return True + if predicate == PROV.wasDerivedFrom: + # Keep ontology lineage hashes in the clean ontology graph. + if isinstance(obj, URIRef) and str(obj).startswith("urn:hash:"): + return False + return True + if predicate == PROV.generatedAtTime or is_schema_chunk_metadata(predicate): + return True + if predicate == OWL.sameAs: + return True + if subject in reifier_nodes or obj in reifier_nodes: + return True + if subject in chunk_nodes or obj in chunk_nodes: + return True + if predicate == RDF.type and str(obj) in { + str(PROV.Entity), + str(SCHEMA.text), + "http://schema.org/text", + }: + return True + return False + + for triple in graph: + if is_provenance_or_alignment_triple(*triple): + provenance_graph.add(triple) + else: + clean_graph.add(triple) + + return clean_graph, provenance_graph + + +def normalize_ontology_units( + units: list[ContentUnit], + tools: ToolBox, + base_ontology: Ontology | None = None, + require_base: bool = False, +) -> tuple[Ontology, list[GraphUpdate], RDFGraph]: + """Merge ontology unit deltas as TripleOps, then apply to base ontology. + + Units contain ontology delta graphs (insert triples only). To preserve the + exact unit output shape (and avoid ontology/facts aggregation rewrites), we + convert each unit graph into an ``insert`` TripleOp and apply them as one + GraphUpdate. + + Args: + units: ContentUnits with type=ONTOLOGIES and delta graph from each unit. + tools: ToolBox instance. + base_ontology: Optional ontology to use as base; merged delta is applied to it. + require_base: Whether map/reduce caller expects a base ontology. + + Returns: + Tuple of ( + ontology with cleaned graph, + list of applied GraphUpdates for versioning, + provenance artifact graph stripped from ontology output, + ). + """ + if not units: + if base_ontology is not None: + return base_ontology, [], RDFGraph() + return Ontology(graph=RDFGraph()), [], RDFGraph() + + for unit in units: + unit.sanitize() + _ = tools + + if require_base and (base_ontology is None or base_ontology.is_null()): + logger.warning( + "normalize_ontology_units expected a base ontology but none was available; " + "continuing with merged aggregated ontology output." + ) + + merged_update = GraphUpdate( + triple_operations=[ + TripleOp(type="insert", graph=unit.graph) + for unit in units + if len(unit.graph) > 0 + ] + ) + if not merged_update.triple_operations: + merged_update = None + + if base_ontology is not None and not base_ontology.is_null(): + base_graph = base_ontology.graph + if merged_update is not None: + updated_graph, _ = AgentState.render_updated_graph( + base_graph, [merged_update], max_triples=None + ) + graph_changed = set(updated_graph) != set(base_graph) + if graph_changed: + result = base_ontology.derive_updated_version(updated_graph) + else: + result = base_ontology.model_copy(deep=True) + result.graph = updated_graph + else: + result = base_ontology.model_copy(deep=True) + result.sync_properties_to_graph() + cleaned_graph, provenance_graph = split_ontology_and_provenance_graph( + result.graph + ) + result.graph = cleaned_graph + result.sync_properties_to_graph() + applied = [merged_update] if merged_update else [] + return result, applied, provenance_graph + + aggregated_delta = RDFGraph() + for unit in units: + for triple in unit.graph: + aggregated_delta.add(triple) + for prefix, namespace in unit.graph.namespaces(): + if prefix: + aggregated_delta.bind(prefix, namespace) + + cleaned_graph, provenance_graph = split_ontology_and_provenance_graph( + aggregated_delta + ) + result = Ontology( + graph=cleaned_graph, + ontology_id=base_ontology.ontology_id if base_ontology else None, + title=base_ontology.title if base_ontology else None, + description=base_ontology.description if base_ontology else None, + ) + applied = [merged_update] if merged_update else [] + return result, applied, provenance_graph diff --git a/ontology_platform/vendored/ontocast/ontocast/agent/render_facts.py b/ontology_platform/vendored/ontocast/ontocast/agent/render_facts.py new file mode 100644 index 0000000..02c303b --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/agent/render_facts.py @@ -0,0 +1,294 @@ +"""Fact rendering agent for OntoCast. + +This module provides functionality for rendering facts from RDF graphs into +human-readable formats, making the extracted knowledge more accessible and +understandable. +""" + +import logging + +from langchain_core.output_parsers import PydanticOutputParser +from langchain_core.prompts import PromptTemplate + +from ontocast.agent.common import call_llm_with_retry, render_suggestions_prompt +from ontocast.onto.constants import DEFAULT_IRI +from ontocast.onto.enum import FailureStage, Status, WorkflowNode +from ontocast.onto.model import FactsRenderReport, GraphUpdateRenderReport +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.unit_states import UnitFactsState +from ontocast.prompt.common import ( + facts_template, + ontology_template, + output_instruction_empty, + output_instruction_sparql, + text_template, + user_template, +) +from ontocast.prompt.render_facts import ( + facts_instruction_template, + preamble, + template_prompt, +) +from ontocast.tool.atomic import AtomicToolBox + +logger = logging.getLogger(__name__) + + +def _extract_known_prefixes(state: UnitFactsState) -> dict[str, str]: + """Extract ontology prefixes used to patch missing declarations in LLM TTL output.""" + known_prefixes: dict[str, str] = {} + + if state.ontology_snapshot and state.ontology_snapshot.graph: + for prefix, namespace_uri in state.ontology_snapshot.graph.namespaces(): + if prefix: # Skip empty prefixes + known_prefixes[prefix] = str(namespace_uri) + + # Also add the ontology prefix explicitly if available. + if state.ontology_snapshot.prefix and state.ontology_snapshot.namespace: + known_prefixes[state.ontology_snapshot.prefix] = ( + state.ontology_snapshot.namespace + ) + + return known_prefixes + + +async def render_facts(state: UnitFactsState, tools: AtomicToolBox) -> UnitFactsState: + """Structured hybrid facts renderer with Turtle/SPARQL decision logic. + + This function decides between generating bare Turtle for fresh facts + and SPARQL operations for updates based on whether facts exist. + + Args: + state: The current unit facts state + tools: The toolbox containing necessary tools + + Returns: + UnitFactsState: Updated state with rendered facts + """ + + is_fresh_facts_graph = len(state.content_unit.graph) == 0 + + progress_info = state.get_content_unit_progress_string() + logger.info(f"Render facts for {progress_info}") + + if is_fresh_facts_graph: + logger.info("Generating fresh facts as Turtle") + return await render_facts_fresh(state, tools) + else: + logger.info("Generating facts update") + return await render_facts_update(state, tools) + + +def _prepare_prompt_data(state: UnitFactsState) -> dict[str, str]: + """Prepare common prompt data for both fresh and update rendering. + + Args: + state: The current unit facts state + + Returns: + Dictionary containing formatted prompt components + """ + ontology_chapter = ontology_template.format( + ontology_ttl=state.ontology_snapshot.graph.serialize(format="turtle") + ) + + facts_instruction_str = facts_instruction_template.format( + ontology_namespace=state.ontology_snapshot.namespace, + ontology_prefix=state.ontology_snapshot.prefix, + facts_namespace=DEFAULT_IRI, + ) + + text_chapter = text_template.format(text=state.content_unit.text) + + fact_chapter = "" + + user_instruction = ( + user_template.format(user_instruction=state.facts_user_instruction) + if state.facts_user_instruction + else "" + ) + + return { + "ontology_chapter": ontology_chapter, + "user_instruction": user_instruction, + "facts_instruction": facts_instruction_str, + "text_chapter": text_chapter, + "fact_chapter": fact_chapter, + } + + +def _create_prompt_template() -> PromptTemplate: + """Create the common prompt template used by both rendering functions. + + Returns: + Configured PromptTemplate instance + """ + return PromptTemplate( + template=template_prompt, + input_variables=[ + "preamble", + "facts_instruction", + "user_instruction", + "ontology_chapter", + "text_chapter", + "improvement_instruction", + "output_instruction", + "format_instructions", + ], + ) + + +def _handle_rendering_error( + state: UnitFactsState, error: Exception, stage: FailureStage +) -> UnitFactsState: + """Handle rendering errors consistently. + + Args: + state: The current agent state + error: The exception that occurred + stage: The failure stage to set + + Returns: + Updated state with failure information + """ + logger.error(f"Failed to generate triples: {str(error)}") + state.set_failure(stage, str(error)) + state.set_node_status(WorkflowNode.TEXT_TO_FACTS, Status.FAILED) + return state + + +async def render_facts_fresh( + state: UnitFactsState, tools: AtomicToolBox +) -> UnitFactsState: + """Render fresh facts from the current chunk into Turtle format. + + Args: + state: The current unit facts state containing the chunk to render. + tools: The toolbox instance providing utility functions. + + Returns: + UnitFactsState: Updated state with rendered facts. + """ + logger.info("Rendering fresh facts") + llm_tool = await tools.get_llm_tool(state.budget_tracker) + parser = PydanticOutputParser(pydantic_object=FactsRenderReport) + + known_prefixes = _extract_known_prefixes(state) + + prompt_data = _prepare_prompt_data(state) + prompt_data_fresh = { + "preamble": preamble, + "improvement_instruction": "", + "output_instruction": output_instruction_empty, + } + prompt_data.update(prompt_data_fresh) + + prompt = _create_prompt_template() + + try: + # Set known prefixes in context before parsing + RDFGraph.set_known_prefixes(known_prefixes if known_prefixes else None) + + render_report: FactsRenderReport = await call_llm_with_retry( + llm_tool=llm_tool, + prompt=prompt, + parser=parser, + prompt_kwargs={ + "format_instructions": parser.get_format_instructions(), + **prompt_data, + }, + ) + state.set_external_evidence_request( + WorkflowNode.TEXT_TO_FACTS, render_report.external_evidence_request + ) + facts_report = render_report.facts_report + facts_report.semantic_graph.sanitize_prefixes_namespaces() + state.content_unit.graph = facts_report.semantic_graph + + # Track triples in budget tracker (fresh facts) + num_triples = len(facts_report.semantic_graph) + logger.info(f"Fresh facts generated with {num_triples} triple(s).") + state.budget_tracker.add_facts_update(num_operations=1, num_triples=num_triples) + + state.clear_failure() + state.set_node_status(WorkflowNode.TEXT_TO_FACTS, Status.SUCCESS) + return state + + except Exception as e: + return _handle_rendering_error(state, e, FailureStage.GENERATE_TTL_FOR_FACTS) + finally: + # Clear the context after parsing + RDFGraph.set_known_prefixes(None) + + +async def render_facts_update( + state: UnitFactsState, tools: AtomicToolBox +) -> UnitFactsState: + """Render facts updates using SPARQL operations. + + Args: + state: The current unit facts state containing the chunk to render. + tools: The toolbox instance providing utility functions. + + Returns: + UnitFactsState: Updated state with rendered facts. + """ + logger.info("Rendering updates for facts") + llm_tool = await tools.get_llm_tool(state.budget_tracker) + parser = PydanticOutputParser(pydantic_object=GraphUpdateRenderReport) + + prompt_data = _prepare_prompt_data(state) + prompt_data_update = { + "preamble": preamble, + "improvement_instruction": render_suggestions_prompt( + state.suggestions, WorkflowNode.TEXT_TO_FACTS + ), + "output_instruction": output_instruction_sparql, + "fact_chapter": facts_template.format( + facts_ttl=state.content_unit.graph.serialize(format="turtle") + ), + } + prompt_data.update(prompt_data_update) + prompt = _create_prompt_template() + known_prefixes = _extract_known_prefixes(state) + + try: + # Set known prefixes in context before parsing + RDFGraph.set_known_prefixes(known_prefixes if known_prefixes else None) + + render_report: GraphUpdateRenderReport = await call_llm_with_retry( + llm_tool=llm_tool, + prompt=prompt, + parser=parser, + prompt_kwargs={ + "format_instructions": parser.get_format_instructions(), + **prompt_data, + }, + ) + state.set_external_evidence_request( + WorkflowNode.TEXT_TO_FACTS, render_report.external_evidence_request + ) + graph_update = render_report.graph_update + state.facts_updates.append(graph_update) + state.update_facts() + + num_operations, num_triples = graph_update.count_total_triples() + logger.info( + f"Facts update has {num_operations} operation(s) " + f"with {num_triples} total triple(s)." + ) + + # Track triples in budget tracker + state.budget_tracker.add_facts_update(num_operations, num_triples) + + state.set_node_status(WorkflowNode.TEXT_TO_FACTS, Status.SUCCESS) + state.clear_failure() + return state + + except Exception as e: + return _handle_rendering_error( + state, e, FailureStage.GENERATE_SPARQL_UPDATE_FOR_FACTS + ) + finally: + # Clear the context after parsing + RDFGraph.set_known_prefixes(None) diff --git a/ontology_platform/vendored/ontocast/ontocast/agent/render_ontology.py b/ontology_platform/vendored/ontocast/ontocast/agent/render_ontology.py new file mode 100644 index 0000000..56c66ba --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/agent/render_ontology.py @@ -0,0 +1,289 @@ +"""Ontology triple rendering agent for OntoCast. + +This module provides functionality for rendering RDF triples from ontologies into +human-readable formats, making the ontological knowledge more accessible and +understandable. +The agent decides between generating bare Turtle for fresh ontologies and SPARQL operations for updates. + +""" + +import logging + +from langchain_core.output_parsers import PydanticOutputParser +from langchain_core.prompts import PromptTemplate + +from ontocast.agent.common import call_llm_with_retry, render_suggestions_prompt +from ontocast.onto.enum import FailureStage, Status, WorkflowNode +from ontocast.onto.model import GraphUpdateRenderReport, OntologyRenderReport +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.unit_states import UnitOntologyState +from ontocast.prompt.common import ( + ontology_template, + output_instruction_sparql, + output_instruction_ttl, + text_template, +) +from ontocast.prompt.common import system_preamble_ontology as system_preamble +from ontocast.prompt.render_ontology import ( + general_ontology_instruction, + intro_instruction_fresh, + intro_instruction_update, + prefix_instruction, + prefix_instruction_fresh, + template_prompt, +) +from ontocast.tool.atomic import AtomicToolBox + +logger = logging.getLogger(__name__) + + +def _extract_known_prefixes(state: UnitOntologyState) -> dict[str, str]: + """Extract ontology prefixes used to patch missing declarations in LLM TTL output.""" + current = state.current_ontology or state.ontology_snapshot + known_prefixes: dict[str, str] = {} + + if current and current.graph: + for prefix, namespace_uri in current.graph.namespaces(): + if prefix: # Skip empty prefixes + known_prefixes[prefix] = str(namespace_uri) + + if current.prefix and current.namespace: + known_prefixes[current.prefix] = current.namespace + + return known_prefixes + + +async def render_ontology( + state: UnitOntologyState, tools: AtomicToolBox +) -> UnitOntologyState: + """Structured hybrid ontology renderer with Turtle/SPARQL decision logic. + + This function decides between generating bare Turtle for fresh ontologies + and SPARQL operations for updates based on whether the ontology exists. + + Args: + state: The current unit ontology state + tools: The toolbox containing necessary tools + + Returns: + UnitOntologyState: Updated state with rendered ontology + """ + + progress_info = state.get_content_unit_progress_string() + logger.info( + f"Ontology Renderer for {progress_info}: visit {state.node_visits[WorkflowNode.TEXT_TO_ONTOLOGY]}/{state.max_visits_per_node}" + ) + current = state.current_ontology or state.ontology_snapshot + # Guardrail for map/reduce flow: if a non-null snapshot exists, stay in update mode. + has_seed_ontology = not state.ontology_snapshot.is_null() + has_no_seed_ontology = current.is_null() and not has_seed_ontology + + if has_no_seed_ontology: + return await render_ontology_fresh(state, tools) + else: + return await render_ontology_update(state, tools) + + +async def render_ontology_fresh( + state: UnitOntologyState, tools: AtomicToolBox +) -> UnitOntologyState: + """Render ontology triples into a human-readable format. + + This function takes the triples from the current ontology and renders them + into a more accessible format, making the ontological knowledge easier to + understand. + + Args: + state: The current agent state containing the ontology to render. + tools: The toolbox instance providing utility functions. + + Returns: + AgentState: Updated state with rendered triples. + """ + + parser = PydanticOutputParser(pydantic_object=OntologyRenderReport) + logger.info("Rendering fresh ontology") + intro_instruction = intro_instruction_fresh.format( + current_domain=state.current_domain + ) + output_instruction = output_instruction_ttl + ontology_ttl = "" + improvement_instruction_str = "" + general_ontology_instruction_str = general_ontology_instruction.format( + prefix_instruction=prefix_instruction_fresh + ) + + text_chapter = text_template.format(text=state.content_unit.text) + + external_evidence = state.external_evidence_text + if external_evidence: + state.mark_external_evidence_used(WorkflowNode.TEXT_TO_ONTOLOGY) + + prompt = PromptTemplate( + template=template_prompt, + input_variables=[ + "preamble", + "intro_instruction", + "ontology_instruction", + "output_instruction", + "user_instruction", + "improvement_instruction", + "ontology_ttl", + "text", + "external_evidence", + "format_instructions", + ], + ) + + try: + llm_tool = await tools.get_llm_tool(state.budget_tracker) + render_report: OntologyRenderReport = await call_llm_with_retry( + llm_tool=llm_tool, + prompt=prompt, + parser=parser, + prompt_kwargs={ + "preamble": system_preamble, + "intro_instruction": intro_instruction, + "ontology_instruction": general_ontology_instruction_str, + "output_instruction": output_instruction, + "ontology_ttl": ontology_ttl, + "user_instruction": state.ontology_user_instruction, + "improvement_instruction": improvement_instruction_str, + "text": text_chapter, + "external_evidence": external_evidence, + "format_instructions": parser.get_format_instructions(), + }, + ) + state.set_external_evidence_request( + WorkflowNode.TEXT_TO_ONTOLOGY, render_report.external_evidence_request + ) + state.current_ontology = render_report.ontology + state.current_ontology.graph.sanitize_prefixes_namespaces() + + num_triples = len(state.current_ontology.graph) + logger.info(f"New ontology created with {num_triples} triple(s).") + + # Track triples in budget tracker (fresh ontology) + state.budget_tracker.add_ontology_update( + num_operations=1, num_triples=num_triples + ) + + state.clear_failure() + state.set_node_status(WorkflowNode.TEXT_TO_ONTOLOGY, Status.SUCCESS) + return state + + except Exception as e: + logger.error(f"Failed to generate triples: {str(e)}") + state.set_node_status(WorkflowNode.TEXT_TO_ONTOLOGY, Status.FAILED) + state.set_failure(FailureStage.GENERATE_TTL_FOR_ONTOLOGY, str(e)) + return state + + +async def render_ontology_update( + state: UnitOntologyState, tools: AtomicToolBox +) -> UnitOntologyState: + """Render ontology triples into a human-readable format. + + This function takes the triples from the current ontology and renders them + into a more accessible format, making the ontological knowledge easier to + understand. + + Args: + state: The current unit ontology state containing the ontology to render. + tools: The toolbox instance providing utility functions. + + Returns: + UnitOntologyState: Updated state with rendered triples. + """ + + parser = PydanticOutputParser(pydantic_object=GraphUpdateRenderReport) + current = state.current_ontology or state.ontology_snapshot + ontology_iri = current.iri + ontology_desc = current.describe() + intro_instruction = intro_instruction_update.format( + ontology_iri=ontology_iri, ontology_desc=ontology_desc + ) + ontology_chapter = ontology_template.format( + ontology_ttl=current.graph.serialize(format="turtle") + ) + output_instruction = output_instruction_sparql + improvement_instruction_str = render_suggestions_prompt( + state.suggestions, WorkflowNode.TEXT_TO_ONTOLOGY + ) + + general_ontology_instruction_str = general_ontology_instruction.format( + prefix_instruction=prefix_instruction.format(ontology_prefix=current.prefix), + ontology_prefix=current.prefix, + ) + text_chapter = text_template.format(text=state.content_unit.text) + external_evidence = state.external_evidence_text + if external_evidence: + state.mark_external_evidence_used(WorkflowNode.TEXT_TO_ONTOLOGY) + + prompt = PromptTemplate( + template=template_prompt, + input_variables=[ + "preamble", + "intro_instruction", + "ontology_instruction", + "output_instruction", + "user_instruction", + "improvement_instruction", + "ontology_ttl", + "text", + "external_evidence", + "format_instructions", + ], + ) + known_prefixes = _extract_known_prefixes(state) + + try: + llm_tool = await tools.get_llm_tool(state.budget_tracker) + # Set known prefixes in context before parsing + RDFGraph.set_known_prefixes(known_prefixes if known_prefixes else None) + + render_report: GraphUpdateRenderReport = await call_llm_with_retry( + llm_tool=llm_tool, + prompt=prompt, + parser=parser, + prompt_kwargs={ + "preamble": system_preamble, + "intro_instruction": intro_instruction, + "ontology_instruction": general_ontology_instruction_str, + "output_instruction": output_instruction, + "improvement_instruction": improvement_instruction_str, + "ontology_ttl": ontology_chapter, + "user_instruction": state.ontology_user_instruction, + "text": text_chapter, + "external_evidence": external_evidence, + "format_instructions": parser.get_format_instructions(), + }, + ) + state.set_external_evidence_request( + WorkflowNode.TEXT_TO_ONTOLOGY, render_report.external_evidence_request + ) + graph_update = render_report.graph_update + state.ontology_updates.append(graph_update) + state.update_ontology() + + num_operations, num_triples = graph_update.count_total_triples() + logger.info( + f"Ontology update has {num_operations} operation(s) " + f"with {num_triples} total triple(s)." + ) + + # Track triples in budget tracker + state.budget_tracker.add_ontology_update(num_operations, num_triples) + + state.clear_failure() + state.set_node_status(WorkflowNode.TEXT_TO_ONTOLOGY, Status.SUCCESS) + return state + + except Exception as e: + logger.error(f"Failed to generate ontology update: {str(e)}") + state.set_node_status(WorkflowNode.TEXT_TO_ONTOLOGY, Status.FAILED) + state.set_failure(FailureStage.GENERATE_SPARQL_UPDATE_FOR_ONTOLOGY, str(e)) + return state + finally: + # Clear the context after parsing + RDFGraph.set_known_prefixes(None) diff --git a/ontology_platform/vendored/ontocast/ontocast/agent/select_ontology.py b/ontology_platform/vendored/ontocast/ontocast/agent/select_ontology.py new file mode 100644 index 0000000..5399327 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/agent/select_ontology.py @@ -0,0 +1,204 @@ +# MODIFIED 2026-05-13: Fixed None-selection index inconsistency. +# The dynamic Pydantic model (create_ontology_selector_report_model) constrains +# answer_index to [1, num_ontologies + 1], where num_ontologies + 1 means "None". +# The original code checked `answer_index == 0` for None, which was dead code +# (impossible under the Pydantic constraint) and caused every legitimate None +# selection to fall through to the warning fallback branch. +# See: docs/통합설계서.md §5 Phase 0 and OntoCast 분석 §13.1. +"""Ontology selection agent for OntoCast. + +This module provides functionality for selecting appropriate ontologies based on +the content of source text segments, ensuring that the chosen ontology best matches the +domain and requirements of the text. +""" + +import logging + +from langchain_core.output_parsers import PydanticOutputParser +from langchain_core.prompts import PromptTemplate + +from ontocast.agent.common import call_llm_with_retry +from ontocast.onto.enum import Status +from ontocast.onto.model import create_ontology_selector_report_model +from ontocast.onto.null import NULL_ONTOLOGY +from ontocast.onto.state import AgentState +from ontocast.prompt.select_ontology import template_prompt +from ontocast.tool import OntologyManager +from ontocast.toolbox import ToolBox + +logger = logging.getLogger(__name__) + + +def _create_document_excerpt(state: AgentState, max_length: int = 3000) -> str: + """Create a representative excerpt from the document for ontology selection. + + This function samples text from multiple content units to provide a better + representation of the document content than just the first unit. + + Args: + state: The current agent state. + max_length: Maximum total length of the excerpt. + + Returns: + str: A representative excerpt from the document. + """ + excerpt_parts = [] + total_length = 0 + chunk_length = max_length // 3 # Aim for ~3 source chunks, ~1000 chars each + + # Strategy: Sample from first, middle, and last units if available + if state.content_units: + num_chunks = len(state.content_units) + indices_to_sample = [] + + if num_chunks == 1: + indices_to_sample = [0] + elif num_chunks == 2: + indices_to_sample = [0, 1] + else: + # Sample first, middle, and last + indices_to_sample = [0, num_chunks // 2, num_chunks - 1] + + for idx in indices_to_sample: + if idx < num_chunks and total_length < max_length: + chunk_text = state.content_units[idx].text + # Take a portion of this source chunk + remaining = max_length - total_length + sample_length = min(chunk_length, remaining, len(chunk_text)) + + if sample_length > 0: + if sample_length < len(chunk_text): + excerpt_parts.append(chunk_text[:sample_length] + " ...") + else: + excerpt_parts.append(chunk_text) + total_length += sample_length + + if excerpt_parts: + return "\n\n[...]\n\n".join(excerpt_parts) + + # Fallback: Use input_text if available + if state.input_text: + if len(state.input_text) <= max_length: + return state.input_text + return state.input_text[:max_length] + " ..." + + # Last resort: Use current content unit + if state.current_content_unit and state.current_content_unit.text: + chunk_text = state.current_content_unit.text + if len(chunk_text) <= max_length: + return chunk_text + return chunk_text[:max_length] + " ..." + + return "" + + +async def select_ontology(state: AgentState, tools: ToolBox) -> AgentState: + """Select an appropriate ontology for the document. + + This function analyzes the document and selects the most appropriate + ontology based on its content and requirements using a numbered list selection. + If an ontology is already selected, it skips selection to ensure one ontology + per document. + + Args: + state: The current agent state containing the document to process. + tools: The toolbox instance providing utility functions. + + Returns: + AgentState: Updated state with selected ontology. + """ + # Skip if ontology already selected (for subsequent chunks in the loop) + if not state.current_ontology.is_null(): + logger.debug( + f"Ontology already selected: {state.current_ontology.ontology_id}, " + "skipping selection to maintain one ontology per document" + ) + state.status = Status.SUCCESS + return state + + progress_info = state.get_content_unit_progress_string() + logger.info(f"Selecting ontology for document ({progress_info})") + llm_tool = tools.llm + om_tool: OntologyManager = tools.ontology_manager + + if om_tool.has_ontologies: + ontologies = om_tool.ontologies + num_ontologies = len(ontologies) + + # Create numbered list of ontologies + ontologies_list_lines = [] + for i, ontology in enumerate(ontologies, start=1): + ontologies_list_lines.append(f"{i}. {ontology.describe()}") + + ontologies_list = "\n\n".join(ontologies_list_lines) + + logger.info(f"Presenting {num_ontologies} ontologies for selection") + + # Create a better document excerpt using multiple chunks + excerpt = _create_document_excerpt(state, max_length=3000) + + # Create dynamic model with correct constraint + ontology_selector_report_model = create_ontology_selector_report_model( + num_ontologies + ) + parser = PydanticOutputParser(pydantic_object=ontology_selector_report_model) + + prompt = PromptTemplate( + template=template_prompt, + input_variables=[ + "excerpt", + "ontologies_list", + "num_ontologies", + "format_instructions", + ], + ) + + selector = await call_llm_with_retry( + llm_tool=llm_tool, + prompt=prompt, + parser=parser, + prompt_kwargs={ + "excerpt": excerpt, + "ontologies_list": ontologies_list, + "num_ontologies": num_ontologies, + "format_instructions": parser.get_format_instructions(), + }, + ) + + # Map answer_index to ontology (constrained by Pydantic to [1, num_ontologies + 1]): + # - 1 to num_ontologies -> select ontology at (answer_index - 1) + # - num_ontologies + 1 -> select None (no suitable ontology) + state.status = Status.SUCCESS + none_index = num_ontologies + 1 + if selector.answer_index == none_index: + # Explicit None selection from LLM + logger.debug("LLM selected: None (no suitable ontology)") + state.current_ontology = NULL_ONTOLOGY + elif 1 <= selector.answer_index <= num_ontologies: + # Select ontology at index (answer_index - 1) since list is 0-based + selected_ontology = ontologies[selector.answer_index - 1] + logger.debug( + f"LLM selected ontology at index {selector.answer_index}: " + f"{selected_ontology.ontology_id} ({selected_ontology.iri})" + ) + state.current_ontology = selected_ontology + else: + # Defensive branch: should not happen due to Pydantic ge/le constraint + logger.warning( + f"Out-of-range answer_index {selector.answer_index} " + f"(expected 1..{none_index}); defaulting to NULL_ONTOLOGY" + ) + state.current_ontology = NULL_ONTOLOGY + else: + state.current_ontology = NULL_ONTOLOGY + + # Set the initial version if not already set (tracks original version when ontology was selected) + if state.current_ontology.initial_version is None: + state.current_ontology.initial_version = state.current_ontology.version + logger.debug( + f"Set initial version for ontology {state.current_ontology.ontology_id}: {state.current_ontology.initial_version}" + ) + + logger.debug(f"Current ontology set to: {state.current_ontology.ontology_id}") + + return state diff --git a/ontology_platform/vendored/ontocast/ontocast/agent/serialize.py b/ontology_platform/vendored/ontocast/ontocast/agent/serialize.py new file mode 100644 index 0000000..b605724 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/agent/serialize.py @@ -0,0 +1,78 @@ +"""Serialization agent for OntoCast. + +This module provides functionality for serializing the knowledge graph +(ontology and facts) to the triple store. +""" + +import logging + +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.state import AgentState +from ontocast.toolbox import ToolBox + +logger = logging.getLogger(__name__) + + +def serialize(state: AgentState, tools: ToolBox) -> AgentState: + """Serialize the knowledge graph to the triple store. + + This function: + - Handles version management for updated ontologies + - Tracks budget usage + - Serializes both ontology and facts to the triple store + + Args: + state: Current agent state with ontology and facts + tools: ToolBox containing serialization tools + + Returns: + Updated agent state after serialization + """ + # Initialize empty facts graph if not set (for ontology-only render mode) + if state.aggregated_facts is None: + state.aggregated_facts = RDFGraph() + logger.info("No facts to serialize (ontology-only render mode)") + + # Ontology versioning: reduce_ontology sets ontology_updates_applied with the + # merged GraphUpdate when aggregating parallel ontology units. + if state.ontology_updates_applied: + logger.info( + f"Ontology was updated during processing ({len(state.ontology_updates_applied)} update operations). " + f"Analyzing changes to determine version increment..." + ) + state.current_ontology.mark_as_updated(state.ontology_updates_applied) + state.current_ontology.sync_properties_to_graph() + elif state.ontology_units: + logger.debug("Ontology from EmbeddingBasedAggregator; skipping version bump") + else: + logger.debug( + f"Ontology unchanged during processing (version: {state.current_ontology.version})" + ) + + # Report LLM budget usage + if state.budget_tracker: + logger.info(state.budget_tracker.get_summary()) + + provenance_graph_uri = f"{str(state.graph_uri).rstrip('/')}/ontology-provenance" + if len(state.ontology_provenance_artifact) > 0: + logger.info( + "Persisting ontology provenance artifact (%d triples) to graph %s", + len(state.ontology_provenance_artifact), + provenance_graph_uri, + ) + if tools.filesystem_manager is not None: + tools.filesystem_manager.serialize( + state.ontology_provenance_artifact, + graph_uri=provenance_graph_uri, + ) + if ( + tools.triple_store_manager is not None + and tools.triple_store_manager != tools.filesystem_manager + ): + tools.triple_store_manager.serialize( + state.ontology_provenance_artifact, + graph_uri=provenance_graph_uri, + ) + + tools.serialize(state) + return state diff --git a/ontology_platform/vendored/ontocast/ontocast/agent/sublimate_ontology.py b/ontology_platform/vendored/ontocast/ontocast/agent/sublimate_ontology.py new file mode 100644 index 0000000..da2e6fd --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/agent/sublimate_ontology.py @@ -0,0 +1,153 @@ +"""Ontology sublimation agent for OntoCast. + +This module provides functionality for refining and enhancing ontologies through +a process of sublimation, which involves improving the structure, consistency, +and expressiveness of the ontological knowledge. +""" + +import logging +from typing import Iterable, cast + +from rdflib.term import Node + +from ontocast.onto.constants import DEFAULT_IRI +from ontocast.onto.enum import FailureStage +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.state import AgentState +from ontocast.toolbox import ToolBox + +logger = logging.getLogger(__name__) + + +def _sublimate_ontology(state: AgentState) -> tuple[RDFGraph, RDFGraph]: + graph_onto_addendum = RDFGraph() + graph_facts_pure = RDFGraph() + + # Copy all prefixes from the original graph to both new graphs + for prefix, namespace in state.current_content_unit.graph.namespaces(): + graph_onto_addendum.bind(prefix, namespace) + graph_facts_pure.bind(prefix, namespace) + + query_ontology = f""" + PREFIX cd: <{DEFAULT_IRI}> + + SELECT ?s ?p ?o + WHERE {{ + ?s ?p ?o . + FILTER ( + !( + STRSTARTS(STR(?s), STR(cd:)) || + STRSTARTS(STR(?p), STR(cd:)) || + (isIRI(?o) && STRSTARTS(STR(?o), STR(cd:))) + ) + ) + }} + """ + results = cast( + Iterable[tuple[Node, Node, Node]], + state.current_content_unit.graph.query(query_ontology), + ) + + # Add filtered triples to the new graph + for s, p, o in results: + graph_onto_addendum.add((s, p, o)) + + query_facts = f""" + PREFIX cd: <{DEFAULT_IRI}> + + SELECT ?s ?p ?o + WHERE {{ + ?s ?p ?o . + FILTER ( + STRSTARTS(STR(?s), STR(cd:)) || + STRSTARTS(STR(?p), STR(cd:)) || + (isIRI(?o) && STRSTARTS(STR(?o), STR(cd:))) + ) + }} + """ + + results = cast( + Iterable[tuple[Node, Node, Node]], + state.current_content_unit.graph.query(query_facts), + ) + + # Add filtered triples to the new graph + for s, p, o in results: + graph_facts_pure.add((s, p, o)) + + logger.info( + f"Found triples: facts {len(graph_facts_pure)}; ontology {len(graph_onto_addendum)}" + ) + return graph_onto_addendum, graph_facts_pure + + +def sublimate_ontology(state: AgentState, tools: ToolBox): + logger.debug("Starting ontology sublimation") + + if state.current_ontology is None: + return state + try: + state.update_facts() + graph_onto_addendum, graph_facts = _sublimate_ontology(state=state) + + # Ensure ontology is not null and ontology_id is set before updating + if len(graph_onto_addendum) > 0: + logger.info("ontology seeped into facts:") + logger.info(f"graph: {graph_onto_addendum.serialize()}") + if state.current_ontology.is_null(): + logger.warning( + "Cannot update ontology: null ontology cannot be updated" + ) + elif state.current_ontology.ontology_id: + # Check if adding triples would exceed max_triples limit + max_triples = state.ontology_max_triples + if max_triples is not None: + current_size = len(state.current_ontology.graph) + addendum_size = len(graph_onto_addendum) + if current_size + addendum_size > max_triples: + logger.warning( + f"Ontology sublimation skipped: would exceed limit " + f"({current_size + addendum_size} > {max_triples} triples). " + f"Current size: {current_size} triples." + ) + else: + # Only update state.current_ontology, not OntologyManager + # OntologyManager will be updated in serialize() during final serialization + state.current_ontology.graph += graph_onto_addendum + logger.debug( + f"Updated state.current_ontology with {len(graph_onto_addendum)} triples from sublimation" + ) + else: + # No limit set, proceed with update + state.current_ontology.graph += graph_onto_addendum + logger.debug( + f"Updated state.current_ontology with {len(graph_onto_addendum)} triples from sublimation" + ) + else: + logger.warning("Cannot update ontology: ontology_id is None") + + # Ensure graph_facts is an RDFGraph instance + if not isinstance(graph_facts, RDFGraph): + logger.warning("received an rdflib.Graph rather than RDFGraph") + new_graph = RDFGraph() + graph_facts_rdflib = cast(Iterable[tuple[Node, Node, Node]], graph_facts) + for triple in graph_facts_rdflib: + new_graph.add(triple) + graph_facts_namespaces = cast( + Iterable[tuple[str, str]], graph_facts.namespaces() + ) + for prefix, namespace in graph_facts_namespaces: + new_graph.bind(prefix, namespace) + graph_facts = new_graph + + state.current_content_unit.graph = graph_facts + + state.clear_failure() + except Exception as e: + logger.error(f"Error in sublimate_ontology: {str(e)}") + state.set_failure( + FailureStage.SUBLIMATE_ONTOLOGY, + str(e), + ) + + return state diff --git a/ontology_platform/vendored/ontocast/ontocast/cli/__init__.py b/ontology_platform/vendored/ontocast/ontocast/cli/__init__.py new file mode 100644 index 0000000..2cd3df4 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/cli/__init__.py @@ -0,0 +1,16 @@ +"""Command-line interface tools for OntoCast. + +This package provides command-line tools for interacting with the OntoCast +framework, including document processing, server management, and utility +functions. + +Available commands: +- serve: Start the OntoCast API server +- test_api: Test the API server with sample requests +- batch_process: Batch process multiple files asynchronously +- plot_graph: Visualize workflow graphs +- merge_ontologies: Merge terminal ontologies from Fuseki +- split_chunks: Split documents into chunks +- cmp_states: Compare agent states +- pdfs_to_markdown: Convert PDFs to Markdown format +""" diff --git a/ontology_platform/vendored/ontocast/ontocast/cli/batch_process.py b/ontology_platform/vendored/ontocast/ontocast/cli/batch_process.py new file mode 100644 index 0000000..94ce898 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/cli/batch_process.py @@ -0,0 +1,304 @@ +"""Batch processing client for OntoCast. + +This module provides a command-line client for batch processing multiple files +through the OntoCast API server. It supports async processing with configurable +concurrency limits. + +The client supports: +- Recursive directory scanning +- File pattern matching (e.g., by extension) +- Async processing with concurrency control +- Progress tracking and error reporting +- JSON and PDF file types + +Example: + # Process all JSON files in a directory (max 3 concurrent) + python batch_process.py --url http://localhost:8999 --path ./data --pattern "*.json" --max-concurrent 3 + + # Process all PDF files recursively + python batch_process.py --url http://localhost:8999 --path ./documents --pattern "*.pdf" --recursive +""" + +import asyncio +import json +import pathlib +from typing import Optional + +import click +import httpx + + +async def process_file( + client: httpx.AsyncClient, + url: str, + file_path: pathlib.Path, + semaphore: asyncio.Semaphore, + results: dict, + dataset: Optional[str] = None, +) -> None: + """Process a single file by sending it to the OntoCast API. + + Args: + client: httpx async client + url: API endpoint URL + file_path: Path to the file to process + semaphore: Semaphore to limit concurrent requests + results: Dictionary to store results (success/error counts) + dataset: Optional dataset name for triple store storage + """ + async with semaphore: + try: + file_ext = file_path.suffix.lower() + mime_type = "application/pdf" if file_ext == ".pdf" else "application/json" + + with open(file_path, "rb") as f: + file_content = f.read() + + files = {"file": (file_path.name, file_content, mime_type)} + + # Add dataset as query parameter if provided + params = {} + if dataset: + params["dataset"] = dataset + + response = await client.post(url, files=files, params=params) + status = response.status_code + + if status == 200: + results["success"] += 1 + click.echo(f"✓ {file_path.name} - Success") + else: + error_text = ( + response.text[:200] if response.text else "No error message" + ) + results["errors"] += 1 + results["error_details"][file_path.name] = { + "status": status, + "error": error_text, + } + click.echo(f"✗ {file_path.name} - Error {status}") + + except Exception as e: + results["errors"] += 1 + results["error_details"][file_path.name] = { + "status": None, + "error": str(e)[:200], + } + click.echo(f"✗ {file_path.name} - Exception: {str(e)[:100]}") + + +async def process_files_async( + url: str, + file_paths: list[pathlib.Path], + max_concurrent: int, + dataset: Optional[str] = None, +) -> dict: + """Process multiple files asynchronously with concurrency control. + + Args: + url: API endpoint URL + file_paths: List of file paths to process + max_concurrent: Maximum number of concurrent requests + dataset: Optional dataset name for triple store storage + + Returns: + Dictionary with processing results (success count, error count, details) + """ + results = { + "success": 0, + "errors": 0, + "error_details": {}, + "total": len(file_paths), + } + + if not file_paths: + click.echo("No files found to process.") + return results + + semaphore = asyncio.Semaphore(max_concurrent) + click.echo( + f"Processing {len(file_paths)} file(s) with max {max_concurrent} concurrent requests..." + ) + if dataset: + click.echo(f"Using dataset: {dataset}") + + async with httpx.AsyncClient(timeout=300.0) as client: + tasks = [ + process_file(client, url, file_path, semaphore, results, dataset) + for file_path in file_paths + ] + await asyncio.gather(*tasks) + + return results + + +def find_files( + path: pathlib.Path, pattern: Optional[str], recursive: bool +) -> list[pathlib.Path]: + """Find files matching the given pattern. + + Args: + path: Base path to search + pattern: Glob pattern (e.g., "*.json", "*.pdf") or None for all files + recursive: Whether to search recursively + + Returns: + List of matching file paths + """ + if not path.exists(): + raise click.BadParameter(f"Path does not exist: {path}", param_hint="--path") + + if path.is_file(): + return [path] + + if pattern: + if recursive: + files = list(path.rglob(pattern)) + else: + files = list(path.glob(pattern)) + else: + if recursive: + files = [f for f in path.rglob("*") if f.is_file()] + else: + files = [f for f in path.glob("*") if f.is_file()] + + # Filter to only JSON and PDF files + supported_extensions = {".json", ".pdf"} + files = [f for f in files if f.suffix.lower() in supported_extensions] + + return sorted(files) + + +@click.command() +@click.option( + "--url", + required=True, + help="Base URL for the server (e.g. http://localhost:8999)", +) +@click.option( + "--path", + type=click.Path(path_type=pathlib.Path, exists=True), + required=True, + help="Path to file or directory to process", +) +@click.option( + "--pattern", + type=str, + default=None, + help="Glob pattern to match files (e.g., '*.json', '*.pdf'). If not provided, processes all supported files.", +) +@click.option( + "--recursive", + is_flag=True, + default=True, + help="Search for files recursively in subdirectories", +) +@click.option( + "--max-concurrent", + type=int, + default=3, + help="Maximum number of concurrent requests (default: 3)", +) +@click.option( + "--output", + type=click.Path(path_type=pathlib.Path), + default=None, + help="Optional path to save results summary as JSON", +) +@click.option( + "--dataset", + type=str, + default=None, + help="Dataset name for triple store storage (Fuseki only). If provided, all files will be processed into this dataset.", +) +def main( + url: str, + path: pathlib.Path, + pattern: Optional[str], + recursive: bool, + max_concurrent: int, + output: Optional[pathlib.Path], + dataset: Optional[str], +): + """Batch process files through the OntoCast API server. + + This command finds files matching the given pattern (or all supported files) + and sends them to the OntoCast API server for processing. Files are processed + asynchronously with a configurable concurrency limit. + + Supported file types: .json, .pdf + + Examples: + # Process all JSON files in a directory + batch_process.py --url http://localhost:8999 --path ./data --pattern "*.json" + + # Process all PDFs recursively with 5 concurrent requests + batch_process.py --url http://localhost:8999 --path ./documents --pattern "*.pdf" --recursive --max-concurrent 5 + + # Process files into a specific dataset + batch_process.py --url http://localhost:8999 --path ./data --pattern "*.json" --dataset my_dataset + + # Process a single file + batch_process.py --url http://localhost:8999 --path ./document.pdf + """ + if not url.endswith("/process"): + url = f"{url.rstrip('/')}/process" + + if max_concurrent < 1: + raise click.BadParameter( + "max-concurrent must be at least 1", param_hint="--max-concurrent" + ) + + # Expand user path + path = path.expanduser() + + # Find files + try: + file_paths = find_files(path, pattern, recursive) + except Exception as e: + raise click.ClickException(f"Error finding files: {e}") + + if not file_paths: + click.echo(f"No files found matching pattern '{pattern or '*.*'}' in {path}") + return + + click.echo(f"Found {len(file_paths)} file(s) to process") + if pattern: + click.echo(f"Pattern: {pattern}") + if recursive: + click.echo("Recursive search: enabled") + if dataset: + click.echo(f"Dataset: {dataset}") + click.echo(f"Max concurrent requests: {max_concurrent}") + click.echo("") + + # Process files + results = asyncio.run(process_files_async(url, file_paths, max_concurrent, dataset)) + + # Print summary + click.echo("") + click.echo("=" * 60) + click.echo("Processing Summary") + click.echo("=" * 60) + click.echo(f"Total files: {results['total']}") + click.echo(f"Successful: {results['success']}") + click.echo(f"Errors: {results['errors']}") + + if results["error_details"]: + click.echo("") + click.echo("Error Details:") + for filename, details in results["error_details"].items(): + click.echo(f" {filename}: {details['error']}") + + # Save results if output path provided + if output: + output = output.expanduser() + output.parent.mkdir(parents=True, exist_ok=True) + with open(output, "w") as f: + json.dump(results, f, indent=2) + click.echo("") + click.echo(f"Results saved to: {output}") + + +if __name__ == "__main__": + main() diff --git a/ontology_platform/vendored/ontocast/ontocast/cli/cmp_states.py b/ontology_platform/vendored/ontocast/ontocast/cli/cmp_states.py new file mode 100644 index 0000000..de784f3 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/cli/cmp_states.py @@ -0,0 +1,86 @@ +import pathlib + +import click +from rich.console import Console +from rich.table import Table + +from ontocast.onto.state import AgentState + +console = Console() + + +def get_state_files( + directory: pathlib.Path, pattern: str = "agent_state.onto.update*.json" +) -> list[pathlib.Path]: + """Get all AgentState files matching the pattern in the directory.""" + return sorted(directory.glob(pattern)) + + +def compare_states(states: list[tuple[pathlib.Path, AgentState]]) -> None: + """Compare states and print a table with graph lengths.""" + table = Table(title="AgentState Comparison") + table.add_column("File", style="orange1") + table.add_column("Graph Facts", justify="right") + table.add_column("Current Ontology", justify="right") + table.add_column("Ontology Addendum", justify="right") + table.add_column("Success Score", justify="right") + + # Sort rows by the last number in the filename + sorted_rows = sorted( + [(fp, state) for fp, state in states], + key=lambda x: int(x[0].stem.split(".")[-1]) + if x[0].stem.split(".")[-1].isdigit() + else 0, + ) + + for fp, state in sorted_rows: + table.add_row( + str(fp.stem), + str(len(state.current_content_unit.graph)), + str(len(state.current_ontology.graph)) + if state.current_ontology is not None + else "", + str(len(state.ontology_addendum.graph)), + str(state.success_score), + ) + console.print(table) + + +@click.command() +@click.argument( + "directory", + type=click.Path( + exists=True, file_okay=False, dir_okay=True, path_type=pathlib.Path + ), +) +@click.option( + "--pattern", + default="agent_state.onto.update*.json", + help="Pattern to match state files", +) +def main(directory: pathlib.Path, pattern: str): + """Compare AgentState files in a directory.""" + state_files = get_state_files(directory, pattern) + if not state_files: + console.print( + f"[red]No state files found matching " + f"pattern '{pattern}' in {directory}[/red]" + ) + return + + states = [] + for file_path in sorted(state_files): + try: + state = AgentState.load(file_path) + states.append((file_path, state)) + except Exception as e: + console.print(f"[red]Error loading {file_path}: {str(e)}[/red]") + + if states: + compare_states(states) + else: + console.print("[red]No valid state files found[/red]") + + +if __name__ == "__main__": + main() diff --git a/ontology_platform/vendored/ontocast/ontocast/cli/merge_ontologies.py b/ontology_platform/vendored/ontocast/ontocast/cli/merge_ontologies.py new file mode 100644 index 0000000..d58946e --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/cli/merge_ontologies.py @@ -0,0 +1,87 @@ +"""CLI script to merge terminal ontologies from Fuseki. + +This script: +1. Fetches all terminal ontologies for a given IRI from Fuseki +2. Merges them pair-wise starting from the oldest pair +3. Continues until only one terminal ontology per IRI remains +4. Plots the ontology graph using pygraphviz +""" + +import asyncio +import logging +from pathlib import Path + +import click +from dotenv import load_dotenv + +from ontocast.config import Config +from ontocast.onto.ontology_operations import ( + merge_terminal_ontologies, + plot_ontology_graph, +) +from ontocast.tool.ontology_manager import OntologyManager +from ontocast.tool.triple_manager.fuseki import FusekiTripleStoreManager + +logger = logging.getLogger(__name__) + + +@click.command() +@click.option( + "--env-file", + type=click.Path(path_type=Path), + required=True, + default=".env", + help="Path to .env file containing configuration", +) +@click.option( + "--iri", + type=str, + required=True, + help="IRI of the ontology to merge", +) +@click.option( + "--output", + type=click.Path(path_type=Path), + default="ontology_graph.png", + help="Output path for the graph visualization", +) +def main(env_file: Path, iri: str, output: Path): + """Merge terminal ontologies from Fuseki and plot the result.""" + # Load configuration + load_dotenv(dotenv_path=env_file.expanduser()) + config = Config() + + # Validate configuration + config.validate_llm_config() + + # Create Fuseki manager + tool_config = config.get_tool_config() + if not (tool_config.fuseki.uri and tool_config.fuseki.auth): + raise ValueError("Fuseki configuration required (FUSEKI_URI and FUSEKI_AUTH)") + + fuseki_manager = FusekiTripleStoreManager( + uri=tool_config.fuseki.uri, + auth=tool_config.fuseki.auth, + dataset=tool_config.fuseki.dataset, + ontologies_dataset=tool_config.fuseki.ontologies_dataset, + ) + + # Create ontology manager + ontology_manager = OntologyManager() + + # Merge ontologies + async def run_merge(): + result = await merge_terminal_ontologies(fuseki_manager, ontology_manager, iri) + if result: + logger.info("Merge completed successfully") + # Plot the graph + plot_ontology_graph(ontology_manager, output, iri) + else: + logger.error("Merge failed - no result") + + asyncio.run(run_merge()) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() diff --git a/ontology_platform/vendored/ontocast/ontocast/cli/pdfs_to_markdown.py b/ontology_platform/vendored/ontocast/ontocast/cli/pdfs_to_markdown.py new file mode 100644 index 0000000..6b9f88b --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/cli/pdfs_to_markdown.py @@ -0,0 +1,39 @@ +import json +import logging +import pathlib +import sys + +import click + +from ontocast.cli.util import crawl_directories, pdf2markdown + +logger = logging.getLogger(__name__) + + +def process(output_path, f: pathlib.Path): + fn_json = (output_path / f.name).with_suffix(".json") + jdata = pdf2markdown(f) + with open(fn_json, "w", encoding="utf-8") as fpnt: + json.dump(jdata, fpnt, ensure_ascii=False, indent=4) + + +@click.command() +@click.option("--input-path", type=click.Path(path_type=pathlib.Path), required=True) +@click.option("--output-path", type=click.Path(path_type=pathlib.Path), required=True) +@click.option("--prefix", type=click.STRING, default=None) +def main(input_path, output_path, prefix): + input_path = input_path.expanduser() + output_path = output_path.expanduser() + + files = sorted( + crawl_directories(input_path.expanduser(), suffixes=(".pdf",), prefix=prefix) + ) + + for f in files: + logger.debug(f"processing {f}") + process(output_path, f) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) + main() diff --git a/ontology_platform/vendored/ontocast/ontocast/cli/plot_graph.py b/ontology_platform/vendored/ontocast/ontocast/cli/plot_graph.py new file mode 100644 index 0000000..ce17483 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/cli/plot_graph.py @@ -0,0 +1,148 @@ +import importlib +import logging +import re +from pathlib import Path + +from ontocast.config import ( + Config, + LLMConfig, + LLMProvider, + OllamaModel, + PathConfig, + ToolConfig, +) +from ontocast.stategraph import create_agent_graph +from ontocast.toolbox import ToolBox + +logger = logging.getLogger(__name__) + + +def update_mermaid_graph_in_markdown(file_path: str, new_graph: str): + md_path = Path(file_path) + content = md_path.read_text() + + # Regex pattern to find "### Agent graph" followed by a mermaid block + pattern = r"(### Agent graph\s+```mermaid\n)(.*?)(\n```)" + replacement = r"\1" + new_graph + r"\3" + + if re.search(pattern, content, flags=re.DOTALL): + new_content = re.sub(pattern, replacement, content, flags=re.DOTALL) + print("✅ Replaced existing Mermaid block.") + else: + # Append new section at the end + new_section = f"\n\n### Agent graph\n\n```mermaid\n{new_graph}\n```" + new_content = content + new_section + print("➕ Appended new Mermaid block at the end.") + + md_path.write_text(new_content) + print(f"📄 Updated {file_path}") + + +frontmatter_config = { + "config": { + "theme": "base", + "look": "handDrawn", + "themeVariables": { + "primaryColor": "#FFF3E0", + "primaryBorderColor": "#143642", + "primaryTextColor": "#372237", + "lineColor": "#FFAB91", + "fontFamily": "'Architects Daughter', cursive", + "fontSize": "20px", + }, + "flowchart": {"curve": "basis", "htmlLabels": True, "useMaxWidth": True}, + } +} + + +def main(): + # Create a minimal config for plotting (no API keys needed) + config = Config( + tool_config=ToolConfig( + path_config=PathConfig( + ontology_directory=None, working_directory=Path("/tmp") + ), + llm_config=LLMConfig( + provider=LLMProvider.OLLAMA, + model_name=OllamaModel.LLAMA3_1, + base_url="http://localhost:11434", + ), + ) + ) + toolbox = ToolBox(config) + + # Get the graph and save it as PNG + app = create_agent_graph(toolbox) + graph = app.get_graph() + mmd_data = graph.draw_mermaid(frontmatter_config=frontmatter_config) + + # Save the PNG data to a file + with open("graph.mmd", "w") as f: + f.write(mmd_data) + mmd_data = mmd_data.replace("__start__", "START").replace("__end__", "END") + # update_mermaid_graph_in_markdown("README.md", mmd_data) + + labels = { + "nodes": {"__end__": "END", "__start__": "START"}, + } + + def tweak_draw(fname, extensions: tuple[str, ...]): + fontname = "'Architects Daughter'" + + subtle_green = "#a9cca9" + subtle_orange = "#ffdb99" + viz = pgv.AGraph(directed=True, nodesep=0.7, ranksep=0.5) + for node in graph.nodes: + viz.add_node( + node, + label=labels.get("nodes", {}).get(node, node), + style="filled", + fillcolor=subtle_green, + fontsize=12, + fontname=fontname, + ) + for start, end, data, conditional in graph.edges: + label = str(data) if data is not None else "" + label = labels.get("edges", {}).get(label, label) + viz.add_edge( + start, + end, + label=label, + fontsize=10, + fontname=fontname, + style="dashed" if conditional else "solid", + ) + if first := graph.first_node(): + viz.get_node(first.id).attr.update(fillcolor=subtle_orange) + if last := graph.last_node(): + viz.get_node(last.id).attr.update(fillcolor=subtle_orange) + for ext in extensions: + if ext == "svg": + viz.draw(fname + ".svg", format="svg:cairo", prog="dot") + elif ext == "png": + viz.draw(fname + ".png", format="png", prog="dot", args="-Gdpi=300") + + try: + pgv = importlib.import_module("pygraphviz") + + tweak_draw("docs/assets/graph", extensions=("svg", "png")) + except ImportError as e: + logger.info(f"Could not import graphviz: {e}") + + try: + from langchain_core.runnables.graph import MermaidDrawMethod + + png_data = graph.draw_mermaid_png( + draw_method=MermaidDrawMethod.API, + frontmatter_config=frontmatter_config, + padding=20, + ) + + with open("docs/assets/graph.mmd", "wb") as f: + f.write(png_data) + except ImportError as e: + logger.info(f"Could not import MermaidDrawMethod: {e}") + + +if __name__ == "__main__": + main() diff --git a/ontology_platform/vendored/ontocast/ontocast/cli/serve.py b/ontology_platform/vendored/ontocast/ontocast/cli/serve.py new file mode 100644 index 0000000..591a367 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/cli/serve.py @@ -0,0 +1,562 @@ +"""OntoCast API server implementation. + +This module provides a web server implementation for the OntoCast framework +using Robyn. It exposes REST API endpoints for processing documents and +extracting semantic triples with ontology assistance. + +The server supports: +- Health check endpoint (/health) +- Service information endpoint (/info) +- Document processing endpoint (/process) +- Triple store flush endpoint (/flush) +- Multiple input formats (JSON, multipart/form-data) +- Streaming workflow execution +- Comprehensive error handling and logging + +The server integrates with the OntoCast workflow graph to process documents +through the complete pipeline: chunking, ontology selection, fact extraction, +and aggregation. + +Example: + # With Fuseki backend (auto-detected from FUSEKI_URI and FUSEKI_AUTH) + ontocast --env-path .env + + # Process specific file + ontocast --env-path .env --input-path ./document.pdf + + # Process with chunk limit + ontocast --env-path .env --head-chunks 5 +""" + +import asyncio +import logging +import logging.config +import pathlib + +import click +from dotenv import load_dotenv +from langchain_core.runnables import RunnableConfig +from langgraph.graph.state import CompiledStateGraph + +from ontocast.cli.util import crawl_directories +from ontocast.config import Config, ServerConfig +from ontocast.onto.enum import RenderMode +from ontocast.onto.state import AgentState +from ontocast.stategraph import create_agent_graph +from ontocast.toolbox import ToolBox + +logger = logging.getLogger(__name__) + + +def calculate_recursion_limit( + head_chunks: int | None, + server_config: ServerConfig, +) -> int: + """Calculate the recursion limit based on max visits and head chunks. + + Args: + head_chunks: Optional maximum number of chunks to process + + Returns: + int: Calculated recursion limit + """ + if head_chunks is not None: + # If we know the number of chunks, calculate exact limit + return max( + server_config.base_recursion_limit, + server_config.max_visits_per_node * head_chunks * 10, + ) + else: + # If we don't know chunks, use a conservative estimate + return max( + server_config.base_recursion_limit, + server_config.max_visits_per_node * server_config.estimated_chunks * 10, + ) + + +def create_app( + tools: ToolBox, + server_config: ServerConfig, + head_chunks: int | None = None, +): + from robyn import Headers, Request, Response, Robyn, jsonify + + app = Robyn(__file__) + workflow: CompiledStateGraph = create_agent_graph(tools) + recursion_limit = calculate_recursion_limit( + head_chunks, + server_config, + ) + + @app.get("/health") + async def health_check(): + """MCP health check endpoint.""" + try: + # Check if LLM is available + if tools.llm is None: + return Response( + status_code=503, + headers=Headers({"Content-Type": "application/json"}), + description=jsonify( + {"status": "unhealthy", "error": "LLM not initialized"} + ), + ) + + return Response( + status_code=200, + headers=Headers({"Content-Type": "application/json"}), + description=jsonify( + { + "status": "healthy", + "version": "0.1.1", + "llm_provider": tools.llm_provider, + } + ), + ) + except Exception as e: + logger.error(f"Health check failed: {str(e)}") + return Response( + status_code=503, + headers=Headers({"Content-Type": "application/json"}), + description=jsonify({"status": "unhealthy", "error": str(e)}), + ) + + @app.get("/info") + async def info(): + """MCP info endpoint.""" + return Response( + status_code=200, + headers=Headers({"Content-Type": "application/json"}), + description=jsonify( + { + "name": "ontocast", + "version": "0.1.1", + "description": "Agentic ontology assisted framework " + "for semantic triple extraction", + "capabilities": ["text-to-triples", "ontology-extraction"], + "input_types": ["text", "json", "pdf", "markdown"], + "output_types": ["turtle", "json"], + } + ), + ) + + @app.post("/flush") + async def flush(request: Request): + """Flush/clean data from the triple store. + + This endpoint deletes data from the configured triple store. + For Fuseki, you can specify a dataset query parameter to clean a specific dataset, + or omit it to clean all datasets. For Neo4j, this deletes all nodes (dataset parameter is ignored). + + Query Parameters: + dataset (optional): For Fuseki only - name of the dataset to clean. + If omitted, cleans all datasets (main and ontologies). + + Warning: This operation is irreversible and will delete all data. + + Returns: + JSON response with status and message. + + Example: + # Clean all datasets (Fuseki) or entire database (Neo4j) + POST /flush + + # Clean specific Fuseki dataset + POST /flush?dataset=my_dataset + """ + try: + if tools.triple_store_manager is None: + return Response( + status_code=400, + headers=Headers({"Content-Type": "application/json"}), + description=jsonify( + { + "status": "error", + "error": "No triple store manager configured", + } + ), + ) + + # Extract dataset parameter (used by Fuseki, ignored by others) + dataset = request.query_params.get("dataset", None) + + # All implementations accept the dataset parameter + # Fuseki uses it, Neo4j and Filesystem ignore it with a warning + await tools.triple_store_manager.clean(dataset=dataset) + + # Generate appropriate success message + from ontocast.tool.triple_manager.fuseki import FusekiTripleStoreManager + + if isinstance(tools.triple_store_manager, FusekiTripleStoreManager): + if dataset: + message = f"Fuseki dataset '{dataset}' flushed successfully" + else: + message = "Fuseki triple store flushed successfully (all datasets)" + else: + message = "Triple store flushed successfully" + + return Response( + status_code=200, + headers=Headers({"Content-Type": "application/json"}), + description=jsonify( + { + "status": "success", + "message": message, + } + ), + ) + except Exception as e: + logger.error(f"Error flushing triple store: {str(e)}") + return Response( + status_code=500, + headers=Headers({"Content-Type": "application/json"}), + description=jsonify( + { + "status": "error", + "error": str(e), + "error_type": type(e).__name__, + } + ), + ) + + @app.post("/process") + async def process(request: Request): + """MCP process endpoint.""" + workflow_state: dict | None = None + try: + content_type = request.headers.get("content-type") + logger.debug(f"Content-Type: {content_type}") + logger.debug(f"Request headers: {request.headers}") + logger.debug(f"Request body: {request.body}") + + # Extract parameters from query parameters + dataset = request.query_params.get("dataset", None) + if dataset: + logger.debug(f"Using dataset: {dataset}") + + # Preferred rendering mode parameter + render_mode = request.query_params.get("render_mode", None) + if render_mode: + logger.debug(f"Using render_mode: {render_mode}") + + # Extract user instructions from query parameters (available for both JSON and multipart) + ontology_user_instruction = request.query_params.get( + "ontology_user_instruction", "" + ) + facts_user_instruction = request.query_params.get( + "facts_user_instruction", "" + ) + if ontology_user_instruction: + logger.debug( + f"Query param - ontology_user_instruction: {ontology_user_instruction}" + ) + if facts_user_instruction: + logger.debug( + f"Query param - facts_user_instruction: {facts_user_instruction}" + ) + + if content_type and content_type.startswith("application/json"): + data = request.body + # Convert string to bytes if needed + if isinstance(data, str): + bytes_data = data.encode("utf-8") + else: + bytes_data = data + logger.debug( + f"Parsed JSON data: {data}, bytes length: {len(bytes_data)}" + ) + files = {"input.json": bytes_data} + # User instructions already extracted from query params above + # They can also be overridden by convert_document.py for JSON files + elif content_type and content_type.startswith("multipart/form-data"): + files = request.files + logger.debug(f"Files: {files.keys()}") + logger.debug(f"Files-types: {[(k, type(v)) for k, v in files.items()]}") + + # Check if form data contains user instructions (overrides query params) + if hasattr(request, "form_data") and request.form_data: + form_ontology_instruction = request.form_data.get( + "ontology_user_instruction", "" + ) + form_facts_instruction = request.form_data.get( + "facts_user_instruction", "" + ) + if form_ontology_instruction: + ontology_user_instruction = form_ontology_instruction + logger.debug( + f"Form data - ontology_user_instruction: " + f"{ontology_user_instruction}" + ) + if form_facts_instruction: + facts_user_instruction = form_facts_instruction + logger.debug( + f"Form data - facts_user_instruction: {facts_user_instruction}" + ) + if not files: + return Response( + status_code=400, + headers=Headers({"Content-Type": "application/json"}), + description=jsonify( + { + "status": "error", + "error": "No file provided", + "error_type": "ValidationError", + } + ), + ) + else: + logger.debug(f"Unsupported content type: {content_type}") + return Response( + status_code=400, + headers=Headers({"Content-Type": "application/json"}), + description=jsonify( + { + "status": "error", + "error": f"Unsupported content type: {content_type}", + "error_type": "ValidationError", + } + ), + ) + + # Update dataset if provided (efficient - no model reloading) + if dataset: + await tools.update_dataset(dataset) + + def parse_render_mode_param(value, default: RenderMode) -> RenderMode: + """Parse render mode from query string or use default.""" + if value is None: + return default + if isinstance(value, RenderMode): + return value + if isinstance(value, str): + normalized = value.lower().strip() + try: + return RenderMode(normalized) + except ValueError: + logger.warning( + f"Invalid render_mode '{value}', using default '{default.value}'" + ) + return default + + render_mode_value: RenderMode = parse_render_mode_param( + render_mode, + server_config.render_mode, + ) + + initial_state = AgentState( + files=files, + max_visits=server_config.max_visits_per_node, + max_chunks=head_chunks, + render_mode=render_mode_value, + ontology_max_triples=server_config.ontology_max_triples, + dataset=dataset, + ontology_user_instruction=ontology_user_instruction, + facts_user_instruction=facts_user_instruction, + ) + + async for chunk in workflow.astream( + initial_state, + stream_mode="values", + config=RunnableConfig(recursion_limit=recursion_limit), + ): + workflow_state = chunk + + if workflow_state is None: + raise ValueError("Workflow did not return a valid state") + + # Extract budget tracker data if available + budget_tracker_data = {} + if workflow_state.get("budget_tracker"): + budget_tracker = workflow_state["budget_tracker"] + # Convert Pydantic model to dict using model_dump() + budget_tracker_data = budget_tracker.model_dump() + + total_content_units = len( + workflow_state.get("content_units", workflow_state.get("chunks", [])) + ) + render_mode = workflow_state.get("render_mode") + render_facts_enabled = render_mode in ( + RenderMode.FACTS, + RenderMode.ONTOLOGY_AND_FACTS, + RenderMode.FACTS.value, + RenderMode.ONTOLOGY_AND_FACTS.value, + ) + if render_facts_enabled: + processed_content_units = len( + workflow_state.get("parallel_facts_units", []) + ) + else: + processed_content_units = total_content_units + chunks_remaining = max(total_content_units - processed_content_units, 0) + + result = { + "status": "success", + "data": { + "facts": workflow_state["aggregated_facts"].serialize( + format="turtle" + ) + if workflow_state.get("aggregated_facts") + else "", + "ontology": workflow_state["current_ontology"].graph.serialize( + format="turtle" + ) + if workflow_state.get("current_ontology") + else "", + }, + "metadata": { + "status": workflow_state["status"], + "chunks_processed": processed_content_units, + "chunks_remaining": chunks_remaining, + "budget": budget_tracker_data, + }, + } + + return Response( + status_code=200, + headers=Headers({"Content-Type": "application/json"}), + description=jsonify(result), + ) + + except Exception as e: + logger.error(f"Error processing document: {str(e)}") + logger.error(f"Error type: {type(e)}") + logger.error("Error traceback:", exc_info=True) + + # Try to get error details from workflow_state if available + error_details = None + if workflow_state: + error_details = { + "stage": workflow_state.get("failure_stage", "unknown"), + "reason": workflow_state.get("failure_reason", "unknown"), + } + + return Response( + status_code=500, + headers=Headers({"Content-Type": "application/json"}), + description=jsonify( + { + "status": "error", + "error": str(e), + "error_type": type(e).__name__, + "error_details": error_details, + } + ), + ) + + return app + + +@click.command() +@click.option( + "--env-file", + type=click.Path(path_type=pathlib.Path), + required=True, + default=".env", + help="Path to .env file containing backend and configuration settings", +) +@click.option("--input-path", type=click.Path(path_type=pathlib.Path), default=None) +@click.option("--head-chunks", type=int, default=None) +def run( + env_file: pathlib.Path, + input_path: pathlib.Path | None, + head_chunks: int | None, +): + """ + Main entry point for the OntoCast server/CLI. + + Backend selection is automatically inferred from available configuration: + - Fuseki: If FUSEKI_URI and FUSEKI_AUTH are provided (preferred) + - Neo4j: If NEO4J_URI and NEO4J_AUTH are provided (fallback) + - Filesystem Triple Store: If ONTOCAST_WORKING_DIRECTORY and ONTOCAST_ONTOLOGY_DIRECTORY are provided + - Filesystem Manager: If ONTOCAST_WORKING_DIRECTORY is provided (can be combined with other backends) + + No explicit backend configuration flags are needed - backends are automatically detected. + + """ + + _ = load_dotenv(dotenv_path=env_file.expanduser()) + # Global configuration instance + config = Config() + + # Validate LLM configuration + config.validate_llm_config() + + if config.logging_level is not None: + try: + logger_conf = f"logging.{config.logging_level}.conf" + logging.config.fileConfig(logger_conf, disable_existing_loggers=False) + logger.debug("debug is on") + except Exception as e: + logger.error(f"could set logging level correctly {e}") + + if config.tool_config.path_config.working_directory is not None: + config.tool_config.path_config.working_directory = pathlib.Path( + config.tool_config.path_config.working_directory + ).expanduser() + config.tool_config.path_config.working_directory.mkdir( + parents=True, exist_ok=True + ) + else: + raise ValueError( + "Working directory must be provided via CLI argument or WORKING_DIRECTORY config" + ) + + if config.tool_config.path_config.ontology_directory is not None: + config.tool_config.path_config.ontology_directory = pathlib.Path( + config.tool_config.path_config.ontology_directory + ).expanduser() + + # Create ToolBox with config + tools: ToolBox = ToolBox(config) + asyncio.run(tools.initialize()) + + workflow: CompiledStateGraph = create_agent_graph(tools) + + if input_path: + input_path = input_path.expanduser() + + files = sorted( + crawl_directories( + input_path, + suffixes=tuple([".json"] + list(tools.converter.supported_extensions)), + ) + ) + + recursion_limit = calculate_recursion_limit( + head_chunks, + config.server, + ) + + async def process_files(): + for file_path in files: + try: + state = AgentState( + files={file_path.as_posix(): file_path.read_bytes()}, + max_visits=config.server.max_visits_per_node, + max_chunks=head_chunks, + render_mode=config.server.render_mode, + dataset=config.tool_config.fuseki.dataset, + ) + async for _ in workflow.astream( + state, + stream_mode="values", + config=RunnableConfig(recursion_limit=recursion_limit), + ): + pass + + except Exception as e: + logger.error(f"Error processing {file_path}: {str(e)}") + + asyncio.run(process_files()) + else: + app = create_app( + tools=tools, + server_config=config.server, + head_chunks=head_chunks, + ) + logger.info(f"Starting Ontocast server on port {config.server.port}") + app.start(port=config.server.port) + + +if __name__ == "__main__": + run() diff --git a/ontology_platform/vendored/ontocast/ontocast/cli/split_chunks.py b/ontology_platform/vendored/ontocast/ontocast/cli/split_chunks.py new file mode 100644 index 0000000..0ed0d1f --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/cli/split_chunks.py @@ -0,0 +1,172 @@ +import json +import logging +import pathlib +import sys + +import click +from suthing import FileHandle + +from ontocast.cli.util import crawl_directories +from ontocast.tool.chunk.chunker import ChunkerTool + +logger = logging.getLogger(__name__) + + +def json_to_md(data: dict | list, title: str = "JSON Data", depth: int = 1) -> str: + """ + Convert nested JSON data to Markdown format. + + Args: + data: The JSON data (dict or list) + title: Title for the top-level markdown document + depth: Current heading depth (internal recursion parameter) + + Returns: + Markdown formatted string + """ + if not data: + return "" + + md = [] + + # Add title only at the top level + if depth == 1 and title: + md.append(f"# {title}\n\n") + + if isinstance(data, dict): + # First, handle simple key-value pairs + simple_pairs = [] + complex_items = [] + + for key, value in data.items(): + if isinstance(value, (str, int, float, bool, type(None))): + simple_pairs.append((key, value)) + else: + complex_items.append((key, value)) + + # Add simple pairs first + for key, value in simple_pairs: + md.append(f"**{key}**: {_format_value(value)}\n") + + if simple_pairs: + md.append("\n") + + # Then handle complex items with headers + for key, value in complex_items: + header_level = depth + 1 + + # Clean spacing - add extra newline before major sections + if header_level < 3 and md and not md[-1].endswith("\n\n"): + md.append("\n") + + md.append(f"{'#' * header_level} {key}\n") + + if isinstance(value, dict): + if value: # Skip empty dicts + md.append(json_to_md(value, title=f"doc: {key}", depth=depth + 1)) + elif isinstance(value, list): + md.extend(_handle_list(value, header_level, depth)) + + elif isinstance(data, list): + md.extend(_handle_list(data, depth, depth)) + + return "".join(md) + + +def _handle_list(items: list, header_level: int, depth: int) -> list[str]: + """Handle list items and return markdown lines.""" + md = [] + + if not items: + md.append("*Empty list*\n\n") + return md + + if all(isinstance(item, (str, int, float, bool, type(None))) for item in items): + # Simple list items + for item in items: + md.append(f"- {_format_value(item)}\n") + md.append("\n") + else: + # Complex list items + for i, item in enumerate(items): + if isinstance(item, dict) and len(items) > 1: + md.append(f"{'#' * (header_level + 1)} Item {i + 1}\n\n") + md.append(json_to_md(item, title=f"# doc {i}", depth=depth + 1)) + + return md + + +def _format_value(value) -> str: + """Format a simple value for markdown output.""" + if value is None: + return "*null*" + elif isinstance(value, bool): + return str(value).lower() + elif isinstance(value, str): + # Escape markdown special characters in content + return value.replace("*", r"-") + else: + return str(value) + + +def process(fn_json: pathlib.Path, output_path: pathlib.Path, chunker: ChunkerTool): + logger.debug(f"Processing fn_json: {fn_json}") + + jdata = FileHandle.load(fn_json) + + text = jdata.get("text", None) + if text is None: + if len(jdata.keys()) > 2: + md_lines = json_to_md(jdata) + text = "".join(md_lines) + else: + raise ValueError(f"Not sure about the json format {fn_json}") + + docs_txt = chunker(text) + + sizes = [len(x) for x in docs_txt] + logger.debug(f"Chunk size: {sizes}") + + chunked = {"chunks": docs_txt} + + logger.debug(f"Saving to {output_path / fn_json.name}") + + with open(output_path / fn_json.name, "w", encoding="utf-8") as f: + json.dump(chunked, f, ensure_ascii=False, indent=4) + + return docs_txt + + +@click.command() +@click.option("--input-path", type=click.Path(path_type=pathlib.Path), required=True) +@click.option("--output-path", type=click.Path(path_type=pathlib.Path), required=True) +@click.option("--prefix", type=click.STRING, default=None) +def main(input_path, output_path, prefix): + input_path = input_path.expanduser() + output_path = output_path.expanduser() + + from ontocast.config import ChunkConfig + + chunk_config = ChunkConfig( + breakpoint_threshold_amount=95.0, + breakpoint_threshold_type="percentile", + max_size=4000, # Match the reported issue parameters + min_size=2000, # Match the reported issue parameters + ) + + chunker = ChunkerTool( + chunk_config=chunk_config, + model="sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + ) + + files = sorted( + crawl_directories(input_path.expanduser(), suffixes=(".json",), prefix=prefix) + ) + + for f in files: + process(f, output_path, chunker) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.DEBUG, stream=sys.stdout) + main() diff --git a/ontology_platform/vendored/ontocast/ontocast/cli/test_api.py b/ontology_platform/vendored/ontocast/ontocast/cli/test_api.py new file mode 100644 index 0000000..9cdc8ce --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/cli/test_api.py @@ -0,0 +1,144 @@ +"""Test API client for OntoCast. + +This module provides a simple command-line client for testing the OntoCast API. +It can send requests to the API server with either a default payload, a JSON file, +or a PDF file. + +The client supports: +- Custom server URLs +- JSON or PDF file uploads +- Default test payload for Apple 10-Q document +- Response formatting and display + +Example: + # Send default test payload + python test_api.py --url http://localhost:8999 + + # Send a JSON file (as multipart/form-data) + python test_api.py --url http://localhost:8999 --file sample.json + + # Send a PDF file + python test_api.py --url http://localhost:8999 --file document.pdf +""" + +import io +import json +import pathlib + +import click +import requests + + +@click.command() +@click.option( + "--url", + required=True, + help="Base URL for the server (e.g. http://localhost:8999)", +) +@click.option( + "--file", + type=str, + default=None, + help="Path to JSON or PDF file to upload (supports ~ expansion)", +) +def main(url, file): + """Send a test request to the OntoCast API server. + + This function sends a POST request to the /process endpoint with either: + - A file upload (JSON or PDF) as multipart/form-data + - A JSON text payload as application/json + - A default test payload if no file or json-text is provided + + Args: + url: The base URL of the API server (e.g. http://localhost:8999). + file: Optional path to a JSON or PDF file to upload. + + Example: + >>> main("http://localhost:8999", None, None) + # Sends default Apple 10-Q payload + + >>> main("http://localhost:8999", pathlib.Path("document.pdf"), None) + # Sends PDF file as multipart/form-data + + >>> main("http://localhost:8999", None, '{"text": "Hello"}') + # Sends JSON text payload + """ + if not url.endswith("/process"): + url = f"{url.rstrip('/')}/process" + + if file: + # Expand ~ and convert to Path + file_path = pathlib.Path(file).expanduser() + if not file_path.exists(): + raise click.BadParameter( + f"File does not exist: {file_path}", + param_hint="--file", + ) + + file_ext = file_path.suffix.lower() + if file_ext not in [".json", ".pdf"]: + raise click.BadParameter( + f"File must be .json or .pdf, got {file_ext}", + param_hint="--file", + ) + + print(f"POSTing file '{file_path.name}' to: {url}") + with open(file_path, "rb") as f: + file_content = f.read() + + # Determine MIME type + mime_type = "application/pdf" if file_ext == ".pdf" else "application/json" + # Use BytesIO to create a file-like object for requests + file_obj = io.BytesIO(file_content) + files = {"file": (file_path.name, file_obj, mime_type)} + r = requests.post(url, files=files) + else: + # Default test payload + payload = { + "text": ( + "## UNITED STATES SECURITIES AND EXCHANGE COMMISSION\n\n" + "Washington, D.C. 20549 ## FORM 10-Q\n\n" + "\n\n" + "(Mark One)\n\n" + "☒ QUARTERLY REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES EXCHANGE ACT OF 1934\n\n" + "For the quarterly period ended April 1, 2023\n\n" + "or\n\n" + "☐ TRANSITION REPORT PURSUANT TO SECTION 13 OR 15(d) OF THE SECURITIES EXCHANGE ACT OF 1934\n\n" + "For the transition period from to .\n\n" + "Commission File Number: 001-36743 ## Apple Inc.\n\n" + "(Exact name of Registrant as specified in its charter)\n\n" + "California\n\n" + "94-2404110\n\n" + "(State or other jurisdiction of incorporation or organization)\n\n" + "(I.R.S. Employer Identification No.)\n\n" + "One Apple Park Way Cupertino, California\n\n" + "95014\n\n" + "(Address of principal executive offices)\n\n" + "(Zip Code) ## (408) 996-1010\n\n" + "(Registrant's telephone number, including area code)\n\n" + "Securities registered pursuant to Section 12(b) of the Act:\n\n" + "Title of each class\n\n" + "Trading symbol(s)\n\n" + "Name of each exchange on which registered\n\n" + "Common Stock, $0.00001 par value per share\n\n" + "AAPL\n\n" + "The Nasdaq Stock Market LLC\n\n" + "15,728,702,000 shares of common stock were issued and outstanding as of April 21, 2023. " + "## Apple Inc. ## Form 10-Q ## For the Fiscal Quarter Ended April 1, 2023" + ), + } + print(f"POSTing default test payload to: {url}") + r = requests.post( + url, json=payload, headers={"Content-Type": "application/json"} + ) + + print(f"Status: {r.status_code}") + print("Response:") + try: + print(json.dumps(r.json(), indent=2)) + except Exception: + print(r.text) + + +if __name__ == "__main__": + main() diff --git a/ontology_platform/vendored/ontocast/ontocast/cli/util.py b/ontology_platform/vendored/ontocast/ontocast/cli/util.py new file mode 100644 index 0000000..d7907b9 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/cli/util.py @@ -0,0 +1,35 @@ +import logging.config +import pathlib + +from ontocast.tool.converter import ConverterTool + +logger = logging.getLogger(__name__) + + +def crawl_directories( + input_path: pathlib.Path, suffixes=(".pdf", ".json"), prefix=None +) -> list[pathlib.Path]: + file_paths: list[pathlib.Path] = [] + + if not input_path.is_dir(): + print(f"The path {input_path} is not a valid directory.") + return file_paths + + for file in input_path.rglob("*"): + if ( + file.is_file() + and file.suffix in suffixes + and (file.stem.startswith(prefix) if prefix is not None else True) + ): + file_paths.append(file) + return file_paths + + +def pdf2markdown(file_path: pathlib.Path, converter: ConverterTool | None = None): + if file_path.suffix == ".pdf": + if converter is None: + converter = ConverterTool() + result = converter(file_path) + return result + else: + raise ValueError(f"Unsupported extension {str(file_path.suffix)}") diff --git a/ontology_platform/vendored/ontocast/ontocast/config.py b/ontology_platform/vendored/ontocast/ontocast/config.py new file mode 100644 index 0000000..3912511 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/config.py @@ -0,0 +1,410 @@ +"""Configuration management for OntoCast. + +This module provides hierarchical configuration classes that map to the +environment variables and usage patterns in the OntoCast system. +""" + +from enum import StrEnum +from pathlib import Path +from typing import Literal + +from pydantic import AliasChoices, Field, field_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + +from ontocast.onto.constants import DEFAULT_DATASET, DEFAULT_ONTOLOGIES_DATASET +from ontocast.onto.enum import RenderMode + + +class LLMProvider(StrEnum): + """Supported LLM providers.""" + + OPENAI = "openai" + OLLAMA = "ollama" + + +class LLMModelNameAbstract(StrEnum): + """Abstract base class for all model names.""" + + +class OpenAIModel(LLMModelNameAbstract): + """OpenAI model names.""" + + GPT4_O = "gpt-4o" + GPT4_O_MINI = "gpt-4o-mini" + GPT4_1 = "gpt-41" + GPT4_1_MINI = "gpt-41-mini" + GPT5 = "gpt-5" + GPT5_MINI = "gpt-5-mini" + GPT5_NANO = "gpt-5-nano" + + +class OllamaModel(LLMModelNameAbstract): + """Ollama model names.""" + + QWEN2_5 = "qwen2.5" + QWEN2_5_72B = "qwen2.5:72b" + LLAMA3_1 = "llama3.1" + LLAMA3_1_70B = "llama3.1:70b" + GRANITE3_3_2B = "granite3.3:2b" + GRANITE3_3_8B = "granite3.3:8b" + + +LLMModelName = OpenAIModel | OllamaModel + + +class WebSearchProvider(StrEnum): + """Supported web-search providers.""" + + DUCKDUCKGO = "duckduckgo" + + +class LLMConfig(BaseSettings): + """LLM configuration settings.""" + + provider: LLMProvider = Field( + default=LLMProvider.OPENAI, description="LLM provider" + ) + model_name: LLMModelName = Field( + default=OpenAIModel.GPT4_O_MINI, description="LLM model name" + ) + temperature: float = Field(default=0.0, description="LLM temperature setting") + base_url: str | None = Field( + default=None, description="LLM base URL (for ollama, etc.)" + ) + api_key: str | None = Field(default=None, description="API key for LLM provider") + + model_config = SettingsConfigDict( + env_prefix="LLM_", + case_sensitive=False, + ) + + @field_validator("model_name") + @classmethod + def validate_model_name(cls, v: LLMModelName, info) -> LLMModelName: + """Validate that model_name is compatible with the provider.""" + if "provider" not in info.data: + return v + + provider = info.data["provider"] + + if provider == LLMProvider.OPENAI and not isinstance(v, OpenAIModel): + raise ValueError( + f"Model {v} is not compatible with OpenAI provider. Use OpenAIModel values." + ) + + if provider == LLMProvider.OLLAMA and not isinstance(v, OllamaModel): + raise ValueError( + f"Model {v} is not compatible with Ollama provider. Use OllamaModel values." + ) + + return v + + +class ChunkConfig(BaseSettings): + """Chunking configuration settings.""" + + breakpoint_threshold_type: Literal[ + "percentile", "standard_deviation", "interquartile", "gradient" + ] = Field( + default="percentile", description="Type of threshold calculation for chunking" + ) + breakpoint_threshold_amount: float = Field( + default=95.0, description="Threshold amount for breakpoint detection" + ) + min_size: int = Field(default=3000, description="Minimum chunk size in characters") + max_size: int = Field(default=12000, description="Maximum chunk size in characters") + + model_config = SettingsConfigDict( + env_prefix="CHUNK_", + case_sensitive=False, + ) + + +class ServerConfig(BaseSettings): + """Server configuration settings.""" + + port: int = Field(default=8999, description="Server port") + base_recursion_limit: int = Field( + default=1000, description="Recursion limit for workflow" + ) + estimated_chunks: int = Field(default=30, description="Estimated number of chunks") + max_visits_per_node: int = Field( + default=1, + ge=1, + description="Maximum number of visits allowed per node", + validation_alias=AliasChoices("max_visits_per_node", "max_visits"), + ) + render_mode: RenderMode = Field( + default=RenderMode.ONTOLOGY_AND_FACTS, + description="Rendering mode: ontology, facts, or ontology_and_facts.", + ) + ontology_max_triples: int | None = Field( + default=50000, + description="Maximum number of triples allowed in ontology graph. " + "Updates that would exceed this limit are skipped with a warning. " + "Set to None for unlimited.", + ) + parallel_workers: int = Field( + default=4, + description="Maximum number of concurrent unit workers in parallel pipeline", + ) + parallel_facts_retries: int = Field( + default=3, + description="Retry budget for unit facts loop", + ) + parallel_ontology_retries: int = Field( + default=3, + description="Retry budget for unit ontology loop", + ) + enable_ontology_consolidation: bool = Field( + default=False, + description="Run optional ontology consolidation pass after normalization", + ) + + model_config = SettingsConfigDict( + case_sensitive=False, + ) + + +class Neo4jConfig(BaseSettings): + """Neo4j triple store configuration.""" + + uri: str | None = Field(default=None, description="Neo4j URI") + auth: str | None = Field(default=None, description="Neo4j authentication") + port: int = Field(default=7476, description="Neo4j HTTP port") + bolt_port: int = Field(default=7689, description="Neo4j Bolt port") + + model_config = SettingsConfigDict( + env_prefix="NEO4J_", + case_sensitive=False, + ) + + +class FusekiConfig(BaseSettings): + """Fuseki triple store configuration.""" + + uri: str | None = Field(default=None, description="Fuseki URI") + auth: str | None = Field(default=None, description="Fuseki authentication") + dataset: str = Field(default=DEFAULT_DATASET, description="Fuseki dataset name") + ontologies_dataset: str = Field( + default=DEFAULT_ONTOLOGIES_DATASET, + description="Fuseki dataset name for ontologies", + ) + + model_config = SettingsConfigDict( + env_prefix="FUSEKI_", + case_sensitive=False, + ) + + +class DomainConfig(BaseSettings): + """Domain and URI configuration.""" + + current_domain: str = Field( + default="https://example.com", description="Current domain for URI generation" + ) + + model_config = SettingsConfigDict( + case_sensitive=False, + ) + + +class PathConfig(BaseSettings): + """Path and directory configuration.""" + + working_directory: Path | None = Field( + default=None, + description="Working directory for OntoCast (required if filesystem_manager is enabled)", + ) + ontology_directory: Path | None = Field( + default=None, description="Directory containing ontology files" + ) + cache_dir: Path | None = Field( + default=None, description="Cache directory for LLM responses and tool outputs" + ) + + model_config = SettingsConfigDict( + env_prefix="ONTOCAST_", + case_sensitive=False, + ) + + +class WebSearchConfig(BaseSettings): + """Optional web-search settings for ontology grounding.""" + + enabled: bool = Field( + default=False, + description=( + "Enable optional web grounding. Node execution still starts without " + "search and only searches when node output requests it." + ), + ) + provider: WebSearchProvider = Field( + default=WebSearchProvider.DUCKDUCKGO, description="Web-search provider" + ) + top_k: int = Field(default=3, ge=1, le=10, description="Number of results to fetch") + timeout_seconds: float = Field( + default=8.0, ge=1.0, le=60.0, description="Search request timeout" + ) + max_snippet_chars: int = Field( + default=400, ge=80, le=2000, description="Snippet truncation limit per hit" + ) + max_total_chars: int = Field( + default=1800, ge=200, le=10000, description="Total evidence text budget" + ) + ontology_render_enabled: bool = Field( + default=True, + description=( + "Allow search-eligible retries for ontology render prompts " + "(first pass remains no-search)." + ), + ) + ontology_critic_enabled: bool = Field( + default=True, + description=( + "Allow search-eligible retries for ontology critic prompts " + "(first pass remains no-search)." + ), + ) + facts_render_enabled: bool = Field( + default=False, + description=( + "Allow search-eligible retries for facts render prompts " + "(first pass remains no-search)." + ), + ) + facts_critic_enabled: bool = Field( + default=False, + description=( + "Allow search-eligible retries for facts critic prompts " + "(first pass remains no-search)." + ), + ) + planner_enabled: bool = Field( + default=True, description="Enable LLM planner for web-search decisions" + ) + planner_max_queries: int = Field( + default=3, ge=1, le=8, description="Maximum focused search queries per node" + ) + planner_min_query_chars: int = Field( + default=12, + ge=3, + le=100, + description="Minimum query length accepted by guardrails", + ) + planner_min_confidence: float = Field( + default=0.35, + ge=0.0, + le=1.0, + description="Minimum planner confidence to run search", + ) + reuse_evidence_across_attempt: bool = Field( + default=True, + description=("Reuse node-scoped evidence between retries for the same unit."), + ) + min_snippet_chars: int = Field( + default=40, + ge=0, + le=1000, + description="Minimum snippet length to keep a search hit", + ) + allowed_domains: list[str] = Field( + default_factory=list, + description="Optional allowlist of source domains for evidence", + ) + blocked_domains: list[str] = Field( + default_factory=list, + description="Optional blocklist of source domains for evidence", + ) + region: str = Field(default="wt-wt", description="DuckDuckGo region code") + safesearch: str = Field( + default="moderate", description="DuckDuckGo safesearch mode" + ) + + @field_validator("allowed_domains", "blocked_domains", mode="before") + @classmethod + def parse_domains(cls, value: str | list[str]) -> list[str]: + if isinstance(value, list): + return [entry.strip().lower() for entry in value if entry.strip()] + if isinstance(value, str): + raw_values = [entry.strip().lower() for entry in value.split(",")] + return [entry for entry in raw_values if entry] + return [] + + model_config = SettingsConfigDict( + env_prefix="WEB_SEARCH_", + case_sensitive=False, + ) + + +class AggregationConfig(BaseSettings): + """Aggregation settings for entity clustering/disambiguation.""" + + embedding_model: str = Field( + default="paraphrase-multilingual-MiniLM-L12-v2", + description="Sentence-transformers model name used for entity embeddings.", + ) + similarity_threshold: float = Field( + default=0.80, + ge=0.0, + le=1.0, + description="Cosine similarity threshold used by DBSCAN clustering.", + ) + + model_config = SettingsConfigDict( + env_prefix="AGG_", + case_sensitive=False, + ) + + +class ToolConfig(BaseSettings): + """Configuration for tools (LLM, triple stores, paths, chunking).""" + + llm_config: LLMConfig = Field(default_factory=LLMConfig) + chunk_config: ChunkConfig = Field(default_factory=ChunkConfig) + path_config: PathConfig = Field(default_factory=PathConfig) + neo4j: Neo4jConfig = Field(default_factory=Neo4jConfig) + fuseki: FusekiConfig = Field(default_factory=FusekiConfig) + domain: DomainConfig = Field(default_factory=DomainConfig) + web_search: WebSearchConfig = Field(default_factory=WebSearchConfig) + aggregation: AggregationConfig = Field(default_factory=AggregationConfig) + + +class Config(BaseSettings): + """Main OntoCast configuration. + + This class aggregates all configuration sections and provides + a unified interface for accessing configuration values. + """ + + # Tool configuration (for ToolBox) + tool_config: ToolConfig = Field(default_factory=ToolConfig) + + # Server configuration (for serve.py) + server: ServerConfig = Field(default_factory=ServerConfig) + + # Additional settings + logging_level: str | None = Field(default=None, description="Logging level") + + model_config = SettingsConfigDict( + case_sensitive=False, + extra="ignore", + ) + + def get_tool_config(self) -> ToolConfig: + """Get tool configuration. + + Returns: + ToolConfig: Configuration for tools + """ + return self.tool_config + + def validate_llm_config(self) -> None: + """Validate LLM configuration and raise errors for missing required settings.""" + if ( + self.tool_config.llm_config.provider == LLMProvider.OPENAI + and not self.tool_config.llm_config.api_key + ): + raise ValueError( + "LLM_API_KEY environment variable is required for OpenAI provider" + ) diff --git a/참고/playwright-main/tests/assets/empty.html b/ontology_platform/vendored/ontocast/ontocast/onto/__init__.py similarity index 100% rename from 참고/playwright-main/tests/assets/empty.html rename to ontology_platform/vendored/ontocast/ontocast/onto/__init__.py diff --git a/ontology_platform/vendored/ontocast/ontocast/onto/constants.py b/ontology_platform/vendored/ontocast/ontocast/onto/constants.py new file mode 100644 index 0000000..2daf513 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/onto/constants.py @@ -0,0 +1,28 @@ +from rdflib import Namespace, URIRef + +DEFAULT_DOMAIN = "https://growgraph.dev" +ONTOLOGY_NULL_ID = "__null__" +ONTOLOGY_NULL_IRI = f"{DEFAULT_DOMAIN}/{ONTOLOGY_NULL_ID}" +DEFAULT_IRI = f"{DEFAULT_DOMAIN}/facts" +CHUNK_NULL_IRI = f"{DEFAULT_DOMAIN}/__null__" +DEFAULT_DATASET = "dataset0" +DEFAULT_ONTOLOGIES_DATASET = "ontologies" +COMMON_PREFIXES = { + "xsd": "", + "rdf": "", + "rdfs": "", + "owl": "", + "dc": "", + "dcterms": "", + "skos": "", + "foaf": "", + "schema": "", + "prov": "", + "ex": "", +} +PROV = Namespace("http://www.w3.org/ns/prov#") +SCHEMA = Namespace("https://schema.org/") + +# RDF 1.2 term for linking a reification node to its quoted triple. +# Not yet in rdflib's RDF namespace, so we define it manually. +RDF_REIFIES = URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#reifies") diff --git a/ontology_platform/vendored/ontocast/ontocast/onto/content_unit.py b/ontology_platform/vendored/ontocast/ontocast/onto/content_unit.py new file mode 100644 index 0000000..c350722 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/onto/content_unit.py @@ -0,0 +1,128 @@ +from datetime import datetime, timezone +from enum import StrEnum + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + PrivateAttr, + computed_field, + field_validator, +) +from rdflib import URIRef + +from ontocast.onto.constants import DEFAULT_IRI +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.util import iri2namespace, render_text_hash + + +class OutputType(StrEnum): + FACTS = "facts" + ONTOLOGIES = "ontologies" + + +class SourceUnit(BaseModel): + """Immutable source unit identity and input text. + + Attributes: + text: Source text content for this unit. + index: Position of this unit in the source document. + hid: A stable hash id derived from text. + doc_iri: IRI of parent document. + type: Type of content unit (facts or ontology). + """ + + model_config = ConfigDict(arbitrary_types_allowed=True) + + text: str = Field(description="Source text content for this unit") + index: int = Field(description="Position of this unit in the source document") + doc_iri: URIRef = Field(description="IRI of parent doc") + type: OutputType = Field( + default=OutputType.FACTS, description="Type of content unit" + ) + _hid: str = PrivateAttr(default="") + + @field_validator("doc_iri", mode="before") + @classmethod + def _coerce_doc_iri(cls, value: URIRef | str) -> URIRef: + if isinstance(value, URIRef): + return value + return URIRef(value) + + @computed_field(return_type=str) + @property + def hid(self) -> str: + """Stable hash id generated from source text.""" + rendered_hid = render_text_hash(self.text) + if self._hid != rendered_hid: + self._hid = rendered_hid + return self._hid + + @property + def iri(self): + """Get the base IRI for this unit. + + Returns: + str: The base unit IRI. + """ + return DEFAULT_IRI + + @property + def iri_absolute(self): + """Get the absolute IRI for this unit. + + Returns: + str: The unit IRI. + """ + return f"{self.doc_iri}/{self.hid}" + + @property + def namespace(self): + """Get the namespace for this unit. + + Returns: + str: The unit namespace. + """ + return iri2namespace(self.iri, ontology=False) + + def __len__(self): + return len(self.text) + + +class ContentUnit(SourceUnit): + """A processing unit that extends source data with mutable output fields.""" + + graph: RDFGraph = Field( + description="RDF triples representing facts rendered from this source unit in turtle format " + "as a string in compact form: use prefixes for namespaces, do NOT add comments", + default_factory=RDFGraph, + ) + + _graph_absolute: RDFGraph | None = PrivateAttr(default=None) + + processed: bool = Field(default=False, description="Was this unit processed?") + generated_at: datetime | None = Field( + default=None, description="generated timestamp" + ) + + @property + def graph_absolute(self): + if self._graph_absolute is None: + self._graph_absolute = self.graph.copy() + self._graph_absolute.remap_namespaces(self.iri, self.iri_absolute) + return self._graph_absolute + + @property + def generated_at_iso(self): + """Get generated timestamp in ISO format. + + Returns: + str: Timestamp in ISO format. + """ + if self.generated_at is None: + self.generated_at = datetime.now(timezone.utc) + return self.generated_at.isoformat() + + def sanitize(self): + self.graph = self.graph.unbind_chunk_namespaces() + self.graph.sanitize_prefixes_namespaces() diff --git a/ontology_platform/vendored/ontocast/ontocast/onto/context.py b/ontology_platform/vendored/ontocast/ontocast/onto/context.py new file mode 100644 index 0000000..a1a5f57 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/onto/context.py @@ -0,0 +1,404 @@ +"""Context passing system for agent-based workflow. + +This module provides functionality for passing context between agents, +enabling memory and incremental processing. +""" + +import logging +from datetime import datetime +from enum import StrEnum +from typing import Any + +from pydantic import BaseModel, Field + +from ontocast.onto.sparql_models import SPARQLOperationModel +from ontocast.tool.graph_version_manager import GraphVersion + +logger = logging.getLogger(__name__) + +summary_template = """ +ONTOLOGY CONTEXT: +{ontology_context} + +FACTS CONTEXT: +{facts_context} + +CONTEXT METADATA: +- Agent type: `{agent_type}` +- Timestamp: {context_timestamp} +- Additional metadata: {context_metadata} +""" + + +class AgentType(StrEnum): + """Enumeration of agent types for type safety.""" + + RENDERER_FACTS = "renderer_facts" + RENDERER_ONTOLOGY = "renderer_ontology" + CRITIC_FACTS = "critic_facts" + CRITIC_ONTOLOGY = "critic_ontology" + AGGREGATOR = "aggregator" + CONVERTER = "converter" + CHUNKER = "chunker" + + +class Role(StrEnum): + """Enumeration of conversation roles for type safety.""" + + SYSTEM = "system" + USER = "user" + ASSISTANT = "assistant" + + +class AgentContext(BaseModel): + """Context information passed between agents. + + This class encapsulates all the context information that agents + need to build upon previous work rather than starting fresh. + """ + + # Agent identification + agent_type: AgentType = Field(description="Type of agent for type safety") + + # Previous work context + previous_ontology_version: GraphVersion | None = Field( + default=None, description="Previous ontology version if available" + ) + previous_facts_version: GraphVersion | None = Field( + default=None, description="Previous facts version if available" + ) + + # Previous operations (append-only for performance) + previous_ontology_operations: list[SPARQLOperationModel] = Field( + default_factory=list, description="Previous ontology SPARQL operations" + ) + previous_facts_operations: list[SPARQLOperationModel] = Field( + default_factory=list, description="Previous facts SPARQL operations" + ) + + # Previous critiques (append-only for consistency) + previous_ontology_critique: dict[str, Any] | None = Field( + default=None, description="Previous ontology critique if available" + ) + previous_facts_critique: dict[str, Any] | None = Field( + default=None, description="Previous facts critique if available" + ) + + # Context metadata (append-only strategy) + context_timestamp: datetime = Field( + default_factory=datetime.now, description="When this context was created" + ) + context_metadata: dict[str, Any] = Field( + default_factory=dict, description="Additional context metadata" + ) + + # Conversation memory for LLM calls + conversation_memory: list[dict[str, Any]] = Field( + default_factory=list, description="Conversation history for LLM context" + ) + + # Dynamic context construction + dynamic_context: dict[str, Any] = Field( + default_factory=dict, + description="Dynamically constructed context for current interaction", + ) + + def get_ontology_context_summary(self) -> str: + """Get a summary of ontology context for prompts.""" + if not self.previous_ontology_version: + return "No previous ontology context available." + + summary = f"Previous ontology version: {self.previous_ontology_version.id}\n" + summary += f"Previous ontology size: {self.previous_ontology_version.get_size()} triples\n" + summary += f"Previous ontology operations: {len(self.previous_ontology_operations)} SPARQL operations\n" + + if self.previous_ontology_critique: + summary += f"Previous ontology critique score: {self.previous_ontology_critique.get('score', 'N/A')}\n" + summary += f"Previous ontology critique issues: {self.previous_ontology_critique.get('issues', 'None')}\n" + + return summary + + def get_facts_context_summary(self) -> str: + """Get a summary of facts context for prompts.""" + if not self.previous_facts_version: + return "No previous facts context available." + + summary = f"Previous facts version: {self.previous_facts_version.id}\n" + summary += ( + f"Previous facts size: {self.previous_facts_version.get_size()} triples\n" + ) + summary += f"Previous facts operations: {len(self.previous_facts_operations)} SPARQL operations\n" + + if self.previous_facts_critique: + summary += f"Previous facts critique score: {self.previous_facts_critique.get('score', 'N/A')}\n" + summary += f"Previous facts critique issues: {self.previous_facts_critique.get('issues', 'None')}\n" + + return summary + + def get_full_context_summary(self) -> str: + """Get a complete context summary for prompts.""" + ontology_context = self.get_ontology_context_summary() + facts_context = self.get_facts_context_summary() + + summary = summary_template.format( + facts_context=facts_context, + ontology_context=ontology_context, + agent_type=self.agent_type.value, + context_timestamp=self.context_timestamp.isoformat(), + context_metadata=self.context_metadata, + ) + + return summary + + def add_conversation_memory( + self, role: Role, content: str, metadata: dict[str, Any] | None = None + ) -> None: + """Add a conversation entry to memory (append-only strategy). + + Args: + role: Role of the speaker (Role.SYSTEM, Role.USER, Role.ASSISTANT) + content: Content of the message + metadata: Optional metadata for the conversation entry + """ + entry = { + "role": role.value, + "content": content, + "timestamp": datetime.now().isoformat(), + "metadata": metadata or {}, + } + self.conversation_memory.append(entry) + logger.debug(f"Added conversation memory for {self.agent_type}: {role.value}") + + def get_conversation_context(self, max_entries: int = 10) -> str: + """Get conversation context for LLM calls. + + Args: + max_entries: Maximum number of conversation entries to include + + Returns: + str: Formatted conversation context + """ + if not self.conversation_memory: + return "No conversation history available." + + # Get the most recent entries (append-only strategy preserves order) + recent_entries = self.conversation_memory[-max_entries:] + + context = "CONVERSATION HISTORY:\n" + for entry in recent_entries: + context += f"{entry['role'].upper()}: {entry['content']}\n" + if entry.get("metadata"): + context += f" Metadata: {entry['metadata']}\n" + context += "\n" + + return context + + def build_dynamic_context(self, interaction_type: str, **kwargs) -> dict[str, Any]: + """Build dynamic context for current interaction. + + Args: + interaction_type: Type of interaction (render, critique, etc.) + **kwargs: Additional context parameters + + Returns: + dict[str, Any]: Dynamic context for the interaction + """ + dynamic_context = { + "interaction_type": interaction_type, + "timestamp": datetime.now().isoformat(), + "agent_type": self.agent_type, + "context_summary": self.get_full_context_summary(), + "conversation_context": self.get_conversation_context(), + **kwargs, + } + + # Update the dynamic context + self.dynamic_context.update(dynamic_context) + + return dynamic_context + + def get_llm_context(self) -> str: + """Get complete context for LLM calls including conversation memory. + + Returns: + str: Complete context for LLM calls + """ + + return ( + f"{self.get_full_context_summary()}\n\n" + f"{self.get_conversation_context()}\n\n" + f"DYNAMIC CONTEXT:\n{self.dynamic_context}" + ) + + +class ContextManager(BaseModel): + """Manages context passing between agents. + + This class handles the creation, storage, and retrieval of context + information for agent-based workflows. + """ + + context_history: list[AgentContext] = Field( + default_factory=list, description="History of agent contexts" + ) + current_context: AgentContext | None = Field( + default=None, description="Current active context" + ) + + def __init__(self, **kwargs): + """Initialize the context manager.""" + super().__init__(**kwargs) + + def create_context( + self, + agent_type: AgentType, + previous_ontology_version: GraphVersion | None = None, + previous_facts_version: GraphVersion | None = None, + previous_ontology_operations: list[SPARQLOperationModel] | None = None, + previous_facts_operations: list[SPARQLOperationModel] | None = None, + previous_ontology_critique: dict[str, Any] | None = None, + previous_facts_critique: dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, + ) -> AgentContext: + """Create a new context for an agent. + + Args: + agent_type: Name of the agent creating the context. + agent_type: Type of agent (renderer, critic, etc.). + previous_ontology_version: Previous ontology version if available. + previous_facts_version: Previous facts version if available. + previous_ontology_operations: Previous ontology operations if available. + previous_facts_operations: Previous facts operations if available. + previous_ontology_critique: Previous ontology critique if available. + previous_facts_critique: Previous facts critique if available. + metadata: Additional metadata for the context. + + Returns: + AgentContext: The created context. + """ + context = AgentContext( + agent_type=agent_type, + previous_ontology_version=previous_ontology_version, + previous_facts_version=previous_facts_version, + previous_ontology_operations=previous_ontology_operations or [], + previous_facts_operations=previous_facts_operations or [], + previous_ontology_critique=previous_ontology_critique, + previous_facts_critique=previous_facts_critique, + context_metadata=metadata or {}, + ) + + self.context_history.append(context) + self.current_context = context + + logger.info(f"Created context for {agent_type} ({agent_type})") + return context + + def get_current_context(self) -> AgentContext | None: + """Get the current context. + + Returns: + AgentContext | None: The current context, or None if not set. + """ + return self.current_context + + def get_context_history(self) -> list[AgentContext]: + """Get the full context history. + + Returns: + list[AgentContext]: The complete context history. + """ + return self.context_history + + def get_context_by_agent(self, agent_type: AgentType) -> list[AgentContext]: + """Get context history for a specific agent. + + Args: + agent_type: Name of the agent to get context for. + + Returns: + list[AgentContext]: Context history for the specified agent. + """ + return [ctx for ctx in self.context_history if ctx.agent_type == agent_type] + + def get_latest_context_by_agent(self, agent_type: AgentType) -> AgentContext | None: + """Get the latest context for a specific agent. + + Args: + agent_type: Name of the agent to get latest context for. + + Returns: + AgentContext | None: The latest context for the specified agent, or None. + """ + agent_contexts = self.get_context_by_agent(agent_type) + return agent_contexts[-1] if agent_contexts else None + + def update_context( + self, + agent_type: AgentType, + ontology_version: GraphVersion | None = None, + facts_version: GraphVersion | None = None, + ontology_operations: list[SPARQLOperationModel] | None = None, + facts_operations: list[SPARQLOperationModel] | None = None, + ontology_critique: dict[str, Any] | None = None, + facts_critique: dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, + ) -> AgentContext: + """Update the current context with new information. + + Args: + agent_type: Name of the agent updating the context. + ontology_version: New ontology version if available. + facts_version: New facts version if available. + ontology_operations: New ontology operations if available. + facts_operations: New facts operations if available. + ontology_critique: New ontology critique if available. + facts_critique: New facts critique if available. + metadata: Additional metadata for the context. + + Returns: + AgentContext: The updated context. + """ + if not self.current_context: + # Create new context if none exists + return self.create_context( + agent_type=agent_type, + previous_ontology_version=ontology_version, + previous_facts_version=facts_version, + previous_ontology_operations=ontology_operations, + previous_facts_operations=facts_operations, + previous_ontology_critique=ontology_critique, + previous_facts_critique=facts_critique, + metadata=metadata, + ) + + # Update existing context + if ontology_version: + self.current_context.previous_ontology_version = ontology_version + if facts_version: + self.current_context.previous_facts_version = facts_version + if ontology_operations: + self.current_context.previous_ontology_operations = ontology_operations + if facts_operations: + self.current_context.previous_facts_operations = facts_operations + if ontology_critique: + self.current_context.previous_ontology_critique = ontology_critique + if facts_critique: + self.current_context.previous_facts_critique = facts_critique + if metadata: + self.current_context.context_metadata.update(metadata) + + self.current_context.context_timestamp = datetime.now() + + logger.info(f"Updated context for {agent_type}") + return self.current_context + + def clear_context(self): + """Clear the current context.""" + self.current_context = None + logger.info("Cleared current context") + + def clear_history(self): + """Clear the entire context history.""" + self.context_history = [] + self.current_context = None + logger.info("Cleared context history") diff --git a/ontology_platform/vendored/ontocast/ontocast/onto/enum.py b/ontology_platform/vendored/ontocast/ontocast/onto/enum.py new file mode 100644 index 0000000..814ab07 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/onto/enum.py @@ -0,0 +1,88 @@ +from enum import StrEnum + + +class Status(StrEnum): + """Enumeration of possible workflow status values.""" + + NOT_VISITED = "not visited" + SUCCESS = "success" + FAILED = "failed" + COUNTS_EXCEEDED = "counts exceeded" + + +class OntologyDecision(StrEnum): + """Enumeration of Ontology Decisions used in the workflow.""" + + SKIP_TO_FACTS = "ontology found; skip to facts" + FAILURE_NO_ONTOLOGY = "ontology not found; ffwd to END" + IMPROVE_CREATE_ONTOLOGY = "improve/create ontology" + + +class FactsDecision(StrEnum): + """Enumeration of Ontology Decisions used in the workflow.""" + + TEXT_TO_FACTS = "adequate ontology; render facts" + TEXT_TO_ONTOLOGY = "inadequate ontology; retry render onto" + SERIALIZE = "skip to serialize" + + +class RenderMode(StrEnum): + """Enumeration of supported rendering modes.""" + + ONTOLOGY = "ontology" + FACTS = "facts" + ONTOLOGY_AND_FACTS = "ontology_and_facts" + + +class FailureStage(StrEnum): + """Enumeration of possible failure stages in the workflow.""" + + NO_CHUNKS_TO_PROCESS = "No chunks to process" + ONTOLOGY_CRITIQUE = "The produced ontology did not pass the critique stage." + FACTS_CRITIQUE = "The produced graph of facts did not pass the critique stage." + GENERATE_TTL_FOR_ONTOLOGY = ( + "Failed to generate semantic triples (turtle) for ontology" + ) + GENERATE_SPARQL_UPDATE_FOR_ONTOLOGY = ( + "Failed to generate SPARQL update for ontology" + ) + GENERATE_TTL_FOR_FACTS = "Failed to generate semantic triples (turtle) for facts" + GENERATE_SPARQL_UPDATE_FOR_FACTS = "Failed to generate SPARQL update for ontology" + SUBLIMATE_ONTOLOGY = ( + "The produced semantic could not be validated " + "or separated into ontology and facts (technical issue)." + ) + + +class WorkflowNode(StrEnum): + """Enumeration of workflow nodes in the processing pipeline.""" + + CONVERT_TO_MD = "Convert to Markdown" + CHUNK = "Chunk Text" + TEXT_TO_ONTOLOGY = "Text to Ontology" + TEXT_TO_FACTS = "Text to Facts" + CRITICISE_ONTOLOGY = "Criticise Ontology" + CRITICISE_FACTS = "Criticise Facts" + AGGREGATE_FACTS = "Aggregate Facts" + SERIALIZE = "Serialize" + PARALLEL_MAP_UNITS = "Parallel Map Units" + SELECT_ONTOLOGY = "Select Ontology" + BOOTSTRAP_ONTOLOGY = "Bootstrap Ontology" + RENDER_ONTOLOGY_UPDATE = "Update Ontology" + RENDER_FACTS = "Render Facts" + NORMALIZE_ONTOLOGY_UPDATES = "Normalize Ontology Updates" + CONSOLIDATE_ONTOLOGY = "Consolidate Ontology" + MERGE_FACTS = "Merge Facts" + PLAN_EXTERNAL_EVIDENCE = "Plan External Evidence" + FETCH_EXTERNAL_EVIDENCE = "Fetch External Evidence" + + +class SPARQLOperationType(StrEnum): + """Enumeration of SPARQL operation types. + + This enum is used across the system for type-safe SPARQL operations. + """ + + INSERT = "INSERT" + UPDATE = "UPDATE" + DELETE = "DELETE" diff --git a/ontology_platform/vendored/ontocast/ontocast/onto/model.py b/ontology_platform/vendored/ontocast/ontocast/onto/model.py new file mode 100644 index 0000000..d8531a8 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/onto/model.py @@ -0,0 +1,513 @@ +import pathlib +from typing import Literal + +from pydantic import BaseModel, Field, field_validator + +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.sparql_models import GraphUpdate + + +class BasePydanticModel(BaseModel): + """Base class for Pydantic models with serialization capabilities.""" + + def __init__(self, **kwargs): + """Initialize the model with given keyword arguments.""" + super().__init__(**kwargs) + + def serialize(self, file_path: str | pathlib.Path) -> None: + """Serialize the state to a JSON file. + + Args: + file_path: Path to save the JSON file. + """ + state_json = self.model_dump_json(indent=4) + if isinstance(file_path, str): + file_path = pathlib.Path(file_path) + file_path.write_text(state_json) + + @classmethod + def load(cls, file_path: str | pathlib.Path): + """Load state from a JSON file. + + Args: + file_path: Path to the JSON file. + + Returns: + The loaded model instance. + """ + if isinstance(file_path, str): + file_path = pathlib.Path(file_path) + state_json = file_path.read_text() + return cls.model_validate_json(state_json) + + +def create_ontology_selector_report_model( + num_ontologies: int, +) -> type[BasePydanticModel]: + """Create a dynamic OntologySelectorReport model with answer_index constraint. + + The answer_index field is constrained to be between 1 and num_ontologies + 1, + where: + - 1 to num_ontologies: corresponds to the ontology at that index (1-based) + - num_ontologies + 1: represents "None" (no suitable ontology) + + Args: + num_ontologies: The number of ontologies in the selection list. + + Returns: + A dynamically created Pydantic model class with the appropriate constraint. + """ + max_index = num_ontologies + 1 + + class OntologySelectorReport(BasePydanticModel): + """Report from ontology selection process. + + Attributes: + answer_index: Index of the selected option (1-based). + 1 to num_ontologies: select the ontology at that position. + num_ontologies + 1: select None (no suitable ontology). + """ + + answer_index: int = Field( + ge=1, + le=max_index, + description=( + f"Index of the selected ontology from the numbered list (1-{num_ontologies}) " + f"or {max_index} for 'None' (no suitable ontology). " + f"Use the number corresponding to your choice from the list." + ), + ) + + # Set the class name for better error messages + OntologySelectorReport.__name__ = f"OntologySelectorReport_{num_ontologies}" + return OntologySelectorReport + + +# Keep a base class for backward compatibility and type hints +class OntologySelectorReport(BasePydanticModel): + """Base class for ontology selection report. + + Note: Use create_ontology_selector_report_model() to create + a model with the correct answer_index constraint. + """ + + answer_index: int = Field( + description="Index of the selected ontology from the numbered list (1-based). " + "The maximum value depends on the number of ontologies available." + ) + + +class SemanticTriplesFactsReport(BaseModel): + """Report containing semantic triples and evaluation scores. + + Attributes: + semantic_graph: Semantic triples (facts) representing the document + in turtle (ttl) format. + ontology_relevance_score: Score 0-100 for how relevant the ontology + is to the document. 0 is the worst, 100 is the best. + triples_generation_score: Score 0-100 for how well the facts extraction / + triples generation was performed. 0 is the worst, 100 is the best. + """ + + semantic_graph: RDFGraph = Field( + default_factory=RDFGraph, + description="Semantic triples (facts) representing the document " + "in turtle format: use prefixes for namespaces, do NOT add comments", + ) + ontology_relevance_score: float | None = Field( + ge=0, + le=100, + description=( + "Score between 0 and 100 of how well " + "the ontology represents the domain of the document." + ), + ) + triples_generation_score: float | None = Field( + ge=0, + le=100, + description=( + "Score 0-100 for how well the semantic triples " + "represent the document. 0 is the worst, 100 is the best." + ), + ) + + +class ExternalEvidenceRequest(BaseModel): + """Node-level request for optional web search. + + Nodes use this to explicitly signal whether downstream evidence planning/fetching + should run for another pass. + """ + + initiate_search: bool = Field( + default=False, + description="Whether this node requests external evidence before retrying.", + ) + rationale: str = Field( + default="", + description="Short reason explaining why search is needed (or not needed).", + ) + query_hints: list[str] = Field( + default_factory=list, + description="Optional focused query hints for planner targeting.", + ) + confidence: float = Field( + default=0.0, + ge=0.0, + le=1.0, + description="Confidence in this search decision.", + ) + + @field_validator("query_hints", mode="before") + @classmethod + def normalize_query_hints(cls, value: object) -> list[str]: + if value is None: + return [] + if not isinstance(value, list): + return [] + normalized: list[str] = [] + for item in value: + if not isinstance(item, str): + continue + hint = " ".join(item.split()).strip() + if hint: + normalized.append(hint) + return normalized + + +class FactsRenderReport(BaseModel): + """Facts rendering output with optional search decision.""" + + facts_report: SemanticTriplesFactsReport = Field( + description="Rendered facts payload." + ) + external_evidence_request: ExternalEvidenceRequest = Field( + default_factory=ExternalEvidenceRequest, + description="Optional request to run web search before retrying.", + ) + + +class GraphUpdateRenderReport(BaseModel): + """Graph update rendering output with optional search decision.""" + + graph_update: GraphUpdate = Field(description="SPARQL graph update payload.") + external_evidence_request: ExternalEvidenceRequest = Field( + default_factory=ExternalEvidenceRequest, + description="Optional request to run web search before retrying.", + ) + + +class TripleFix(BaseModel): + text_fragment: str = Field( + description="Exact quote from source text justifying this change" + ) + + action: Literal["ADD", "REMOVE", "REPLACE"] = Field( + description=( + "Type of fix:\n" + "- ADD: Add new triple, prefix declaration, or missing information\n" + "- REMOVE: Delete incorrect or redundant triple\n" + "- REPLACE: Substitute one entity, property, or literal for another" + ) + ) + + severity: Literal["critical", "important", "minor"] = Field( + description=( + "Severity level: " + "'critical' (breaks semantic graph), " + "'important' (significant gap), or " + "'minor' (polish). " + "Note: 'major' will be automatically converted to 'important'." + ) + ) + + @field_validator("severity", mode="before") + @classmethod + def normalize_severity(cls, v: str) -> str: + """Normalize severity values to accepted literals. + + Maps 'major' to 'important' for backward compatibility with prompts + that use 'major' terminology. This allows the LLM to use either term. + """ + if isinstance(v, str): + v_lower = v.lower().strip() + if v_lower == "major": + return "important" + # Return as-is if already valid (will be validated by Literal) + return v + return v + + target: str | None = Field( + default=None, + description=( + "What is being fixed. Examples:\n" + "- 'triple' (for triple-level changes)\n" + "- 'entity' (replacing cd: with ontology entity)\n" + "- 'property' (using correct property)\n" + "- 'datatype' (fixing literal type)\n" + "- 'prefix' (adding namespace declaration)\n" + "- 'language_tag' (adding/fixing @lang)" + ), + ) + + incorrect_value: str | None = Field( + default=None, + description="Current incorrect triple/entity/value (for REMOVE and REPLACE). Use Turtle syntax.", + ) + + correct_value: str | None = Field( + default=None, + description="Proposed correct triple/entity/value (for ADD and REPLACE). Use Turtle syntax.", + ) + + explanation: str = Field( + description=( + "Why this fix is needed. Examples:\n" + "- 'Missing xsd:date datatype for temporal literal'\n" + "- 'Namespace prefix fca: not declared'\n" + "- 'Property onto:decidedBy is canonical, not cd:judgedBy'" + ) + ) + + def to_markdown(self) -> str: + """Convert this TripleFix to markdown format. + + Returns: + Markdown formatted string representing this fix. + """ + lines = [] + + # Add the action and target + action_text = f"**{self.action}**" + if self.target: + action_text += f" ({self.target})" + lines.append(f"- {action_text}") + + # Add text fragment if available + if self.text_fragment: + lines.append(f' - **Source text:** "{self.text_fragment}"') + + # Add incorrect value for REMOVE and REPLACE actions + if self.action in ["REMOVE", "REPLACE"] and self.incorrect_value: + lines.append(f" - **Current (incorrect):** `{self.incorrect_value}`") + + # Add correct value for ADD and REPLACE actions + if self.action in ["ADD", "REPLACE"] and self.correct_value: + lines.append(f" - **Proposed (correct):** `{self.correct_value}`") + + # Add explanation + if self.explanation: + lines.append(f" - **Reason:** {self.explanation}") + + return "\n".join(lines) + + +class OntologyCritiqueReport(BaseModel): + """Report from ontology update critique process.""" + + success: bool = Field( + description="True if the presented ontology is appropriate, complete, consistent and represents well the domain of the provided text, False otherwise." + ) + score: float = Field( + ge=0, + le=100, + description="Score 0-100 for how well the presented ontology serves as the ontology for the document. 0 is the worst, 100 is the best.", + ) + + actionable_ontology_fixes: list[TripleFix] = Field( + default_factory=list, + description="List of specific fixes to correct the facts graph. " + "For each fix, provide the text evidence, the action type, and the relevant triples.", + ) + + systemic_critique_summary: str = Field( + default="", + description="A high-level summary of systemic deficiencies in the ontology (e.g., poor hierarchy structure, redundant concepts, lack of appropriate granularity, or general failures in Domain Coverage). This addresses strategic issues beyond individual term fixes.", + ) + external_evidence_request: ExternalEvidenceRequest = Field( + default_factory=ExternalEvidenceRequest, + description="Optional request to run web search before retrying.", + ) + + +class FactsCritiqueReport(BaseModel): + success: bool = Field( + description="True if the facts triples fully represent the document, False otherwise." + ) + + score: float = Field( + ge=0, + le=100, + description=( + "Score 0-100 for how well the triples of facts represent the original document. " + "0 is the worst, 100 is the best." + ), + ) + + actionable_triple_fixes: list[TripleFix] = Field( + default_factory=list, + description=( + "List of specific fixes to correct the facts graph. " + "For each fix, provide the text evidence, the action type, and the relevant triples." + ), + ) + + systemic_critique_summary: str = Field( + default="", + description=( + "A high-level, non-itemized summary of systemic or pattern-based issues identified across the facts graph.\n" + "Focus on strategic problems rather than individual triple fixes, such as:\n" + "- Consistent failure to extract certain data types (e.g., dates, currencies)\n" + "- Structural patterns like creating entities instead of reusing existing ontology entities\n" + "- Repeated misinterpretation of specific ontology properties or classes\n" + "- Missing coverage of entire categories of information\n\n" + "This guides strategic improvements to the fact-extraction process." + ), + ) + external_evidence_request: ExternalEvidenceRequest = Field( + default_factory=ExternalEvidenceRequest, + description="Optional request to run web search before retrying.", + ) + + +class ExternalEvidenceHit(BaseModel): + """Normalized external evidence hit metadata.""" + + title: str = Field(default="") + url: str = Field(default="") + snippet: str = Field(default="") + domain: str = Field(default="") + + +class ExternalEvidencePlan(BaseModel): + """Structured plan for optional external evidence retrieval.""" + + should_search: bool = Field( + default=False, + description="Whether external evidence retrieval should run for this node.", + ) + rationale: str = Field( + default="", + description="Short explanation of why search is or is not needed.", + ) + intent: Literal[ + "none", + "definition", + "disambiguation", + "standard", + "verification", + "background", + ] = Field( + default="none", + description="Primary reason for searching external evidence.", + ) + confidence: float = Field( + default=0.0, ge=0.0, le=1.0, description="Confidence in the decision." + ) + queries: list[str] = Field( + default_factory=list, description="Targeted search queries." + ) + + @field_validator("queries", mode="before") + @classmethod + def normalize_queries(cls, value: object) -> list[str]: + if value is None: + return [] + if not isinstance(value, list): + return [] + normalized: list[str] = [] + for item in value: + if not isinstance(item, str): + continue + query = " ".join(item.split()).strip() + if query: + normalized.append(query) + return normalized + + +class ExternalEvidenceCacheEntry(BaseModel): + """Node-scoped external evidence planning/fetch outputs.""" + + plan: ExternalEvidencePlan = Field(default_factory=ExternalEvidencePlan) + hits: list[ExternalEvidenceHit] = Field(default_factory=list) + text: str = Field(default="") + source_count: int = Field(default=0, ge=0) + domains: list[str] = Field(default_factory=list) + + +class OntologyRenderReport(BaseModel): + """Ontology rendering output with optional search decision.""" + + ontology: Ontology = Field(description="Rendered ontology payload.") + external_evidence_request: ExternalEvidenceRequest = Field( + default_factory=ExternalEvidenceRequest, + description="Optional request to run web search before retrying.", + ) + + +class Suggestions(BaseModel): + """Report from knowledge graph critique process. + + Attributes: + systemic_critique_summary: A compilation of general improvement suggestions. + actionable_fixes: An itemized list of concrete suggestions for improvement. + """ + + actionable_fixes: list[TripleFix] = Field( + default_factory=list, + description="An itemized list of concrete suggestions for improvement.", + ) + + systemic_critique_summary: str = Field( + default="", description="A general improvement suggestion." + ) + + @classmethod + def from_critique_report( + cls, critique: OntologyCritiqueReport | FactsCritiqueReport + ) -> "Suggestions": + """Create Suggestions from any critique report. + + Args: + critique: Either an OntologyCritiqueReport or FactsCritiqueReport to convert. + + Returns: + Suggestions object with actionable fixes and systemic critique summary. + """ + # Extract actionable fixes based on the type of critique report + if isinstance(critique, OntologyCritiqueReport): + actionable_fixes = critique.actionable_ontology_fixes + elif isinstance(critique, FactsCritiqueReport): + actionable_fixes = critique.actionable_triple_fixes + else: + raise ValueError(f"Unsupported critique report type: {type(critique)}") + + return cls( + actionable_fixes=actionable_fixes, + systemic_critique_summary=critique.systemic_critique_summary, + ) + + def to_markdown(self) -> str: + """Convert actionable fixes and systemic critique summary to a unified markdown block. + + Returns: + Markdown formatted string with both actionable fixes and systemic critique summary. + """ + result = "" + + # Add systemic critique summary if available + if self.systemic_critique_summary: + result += "## Systemic Critique Summary\n\n" + result += self.systemic_critique_summary + "\n\n" + + # Add actionable fixes if available + if self.actionable_fixes: + result += "## Actionable Fixes\n\n" + + for i, fix in enumerate(self.actionable_fixes, 1): + result += f"{i}. {fix.to_markdown()}" + if i < len(self.actionable_fixes): + result += "\n\n" + + return result diff --git a/ontology_platform/vendored/ontocast/ontocast/onto/null.py b/ontology_platform/vendored/ontocast/ontocast/onto/null.py new file mode 100644 index 0000000..12e4a3f --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/onto/null.py @@ -0,0 +1,17 @@ +from rdflib import URIRef +from rdflib.namespace import OWL, RDF + +from ontocast.onto.constants import ONTOLOGY_NULL_IRI +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph + +NULL_ONTOLOGY = Ontology( + ontology_id=None, + title=None, + description=None, + graph=RDFGraph(), + iri=ONTOLOGY_NULL_IRI, +) +null_iri_ref = URIRef(ONTOLOGY_NULL_IRI) +# Add a marker in the graph to denote this is a null ontology +NULL_ONTOLOGY.graph.add((null_iri_ref, RDF.type, OWL.Ontology)) diff --git a/ontology_platform/vendored/ontocast/ontocast/onto/ontology.py b/ontology_platform/vendored/ontocast/ontocast/onto/ontology.py new file mode 100644 index 0000000..bc43051 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/onto/ontology.py @@ -0,0 +1,1281 @@ +import logging +import pathlib +import re +from collections import defaultdict +from datetime import datetime +from typing import Annotated, Union + +from pydantic import BaseModel, ConfigDict, Field +from rdflib import DCTERMS, OWL, RDF, RDFS, XSD, Literal, URIRef + +from ontocast.onto.constants import DEFAULT_DOMAIN, ONTOLOGY_NULL_IRI, PROV +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.sparql_models import GraphUpdate, TripleOp +from ontocast.onto.util import derive_ontology_id +from ontocast.util import iri2namespace + +logger = logging.getLogger(__name__) + +# Semantic version pattern: MAJOR.MINOR.PATCH (e.g., 1.2.3) +SemanticVersion = Annotated[ + str, + Field( + pattern=r"^\d+\.\d+\.\d+$", + description="Semantic version in MAJOR.MINOR.PATCH format (e.g., 1.2.3)", + ), +] + + +class OntologyProperties(BaseModel): + """Properties of an ontology. + + Attributes: + ontology_id: Ontology identifier. + title: Ontology title. + description: A concise description of the ontology. + version: Version of the ontology. + iri: Ontology IRI (Internationalized Resource Identifier). + """ + + ontology_id: str | None = Field( + default=None, + description="Ontology identifier, an human readable lower case abbreviation.", + ) + title: str | None = Field(default=None, description="Ontology title.") + description: str | None = Field( + default=None, + description="A concise description (3-4 sentences) of the ontology " + "(domain, purpose, applicability, etc.)", + ) + version: SemanticVersion | None = Field( + default=None, + description="Version of the ontology (use semantic versioning)", + ) + iri: str = Field( + default=ONTOLOGY_NULL_IRI, + description="Ontology IRI (Internationalized Resource Identifier)", + ) + initial_version: SemanticVersion | None = Field( + default=None, + description=( + "The initial version of the ontology when it was first loaded " + "in this session" + ), + ) + + @property + def namespace(self): + """Get the namespace for this ontology. + + Returns: + str: The namespace string. + """ + return iri2namespace(self.iri, ontology=True) + + +class OntologyPropertiesWithLineage(OntologyProperties): + """Properties of an ontology with versioning lineage information. + + This class extends OntologyProperties with hash-based versioning support, + similar to git-style versioning. Each ontology has a hash of its graph + and optionally multiple parent hashes to support parallel branches and merges. + + Attributes: + hash: Hash of the ontology graph (computed from canonicalized graph). + parent_hashes: List of hashes of parent ontologies. Supports multiple + parents for parallel branches and merges. Can be empty, indicating + this is a root ontology with no parents. + created_at: Timestamp when the ontology version was created (UTC). + This is set deterministically when a version is created, not by LLM. + """ + + hash: str | None = Field( + default=None, + description="Hash of the ontology graph (SHA256 of canonicalized graph)", + ) + parent_hashes: list[str] = Field( + default_factory=list, + description=( + "List of hashes of parent ontologies. Supports multiple parents " + "for parallel branches and merges. Can be empty, indicating " + "this is a root ontology with no parents." + ), + ) + created_at: datetime | None = Field( + default=None, + description="Timestamp when the ontology version was created (UTC). " + "Set deterministically when a version is created, not by LLM.", + ) + + @property + def versioned_iri(self) -> str: + """Get the versioned URI for this ontology (for storage purposes). + + This creates a versioned URI using hash-based fragments for git-style + versioning. Format: #. Falls back to semantic version + fragment (#v1.2.3) or base IRI if hash is not available. + + This allows multiple versions of the same ontology to coexist in storage + (e.g., Fuseki named graphs). The semantic ontology IRI in the graph + remains unchanged; this is only used for storage organization. + + Returns: + str: The versioned URI with hash fragment, or semantic version fragment, + or base IRI if neither is available. + + Examples: + >>> ont = Ontology(iri="https://growgraph.dev/fcaont", hash="abc123...") + >>> ont.versioned_iri + 'https://growgraph.dev/fcaont#abc123...' + >>> ont2 = Ontology(iri="http://example.org/ontology", version="1.0.0") + >>> ont2.versioned_iri + 'http://example.org/ontology#v1.0.0' + """ + if self.hash: + # Use hash-based fragment for git-style versioning + return f"{self.iri}#{self.hash}" + elif self.version: + # Fall back to semantic version fragment for backward compatibility + return f"{self.iri}#v{self.version}" + return self.iri + + +class Ontology(OntologyPropertiesWithLineage): + """A Pydantic model representing an ontology with its RDF graph and description. + + Attributes: + graph: The RDF graph containing the ontology data. + current_domain: The domain used to construct the ontology IRI + if ontology_id is set. + """ + + graph: RDFGraph = Field( + default_factory=RDFGraph, + description="RDF triples that define an ontology " + "in turtle format: use prefixes for namespaces, do NOT add comments.", + ) + + current_domain: str = Field( + default=DEFAULT_DOMAIN, description="Domain for ontology IRI construction." + ) + + model_config = ConfigDict(arbitrary_types_allowed=True) + + def __init__(self, **kwargs): + # Pop current_domain if provided, else use DEFAULT_DOMAIN + current_domain = kwargs.pop("current_domain", DEFAULT_DOMAIN) + super().__init__(**kwargs) + self.current_domain = current_domain + + # Check if this is explicitly a null ontology (only if both IRI is null AND no graph provided) + # Don't return early if graph is provided - graph might contain ontology information + is_explicitly_null = ( + self.iri == ONTOLOGY_NULL_IRI + and self.ontology_id is None + and (not self.graph or len(self.graph) == 0) + ) + if is_explicitly_null: + # This is explicitly a null ontology - don't derive ontology_id, don't compute hash, etc. + return + + # Parse IRI fragment for hash-based or version-based identifiers + if self.iri and "#" in self.iri: + base_iri, fragment = self.iri.rsplit("#", 1) + # Check if fragment is a hash (long hex string) or version (v1.2.3) + if len(fragment) > 20 and all(c in "0123456789abcdef" for c in fragment): + # Looks like a hash - extract it + if self.hash is None: + self.hash = fragment + self.iri = base_iri # Remove fragment from IRI + logger.debug(f"Extracted hash from IRI fragment: {fragment}") + elif fragment.startswith("v") and re.match(r"^v\d+\.\d+\.\d+$", fragment): + # Semantic version fragment - extract version + version_str = fragment[1:] # Remove 'v' prefix + if self.version is None: + self.version = version_str + self.iri = base_iri # Remove fragment from IRI + logger.debug(f"Extracted version from IRI fragment: {version_str}") + + # Try to sync from graph first (this is the primary source of truth) + graph_had_ontology = False + iri_from_graph = None + if self.graph: + # Try to extract from graph + self.sync_properties_from_graph() + # Check if graph provided valid ontology information + # IRI should be set and not null, ontology_id should be set + if self.iri and self.iri != ONTOLOGY_NULL_IRI: + iri_from_graph = self.iri # Remember that IRI came from graph + if self.ontology_id: + graph_had_ontology = True + else: + # IRI is set but ontology_id is missing - try to derive it + self.ontology_id = ( + self._extract_ontology_id_from_prefixes() + or derive_ontology_id(self.iri) + ) + if self.ontology_id: + graph_had_ontology = True + + # Only apply fallback if graph did not provide a valid pair + if not graph_had_ontology: + # Try to extract ontology_id from prefixes if IRI is available + if self.iri and self.iri != ONTOLOGY_NULL_IRI and not self.ontology_id: + # Prefer derivation from IRI over prefix + derived_id = derive_ontology_id(self.iri) + prefix_id = self._extract_ontology_id_from_prefixes() + + if derived_id: + self.ontology_id = derived_id + # If prefix exists but doesn't match ontology_id, rebind it + if prefix_id and prefix_id != derived_id: + self._rebind_prefix_to_ontology_id(prefix_id, derived_id) + elif prefix_id: + # Fallback to prefix if IRI derivation fails + self.ontology_id = prefix_id + + # Fallback logic: construct IRI from ontology_id or vice versa + # BUT: Never override IRI that came from graph + if self.ontology_id and (not self.iri or self.iri == ONTOLOGY_NULL_IRI): + self.iri = f"{self.current_domain}/{self.ontology_id}" + elif self.ontology_id and self.iri and self.iri != ONTOLOGY_NULL_IRI: + # IRI is set - check if it came from graph + if iri_from_graph and self.iri == iri_from_graph: + # IRI came from graph - don't override, just log if pattern doesn't match + expected_iri = f"{self.current_domain}/{self.ontology_id}" + if ( + not self.iri.endswith(f"/{self.ontology_id}") + and self.iri != expected_iri + ): + logger.debug( + f"Ontology IRI '{self.iri}' from graph does not match expected pattern " + f"'{expected_iri}', but keeping IRI from graph (authoritative)" + ) + else: + # IRI didn't come from graph - check if it matches expected pattern + expected_iri = f"{self.current_domain}/{self.ontology_id}" + if ( + not self.iri.endswith(f"/{self.ontology_id}") + and self.iri != expected_iri + ): + logger.warning( + f"Ontology IRI '{self.iri}' does not match expected " + f"'{expected_iri}', correcting IRI" + ) + self.iri = expected_iri + elif not self.ontology_id and self.iri and self.iri != ONTOLOGY_NULL_IRI: + # Extract ontology_id: prefer IRI derivation, rebind prefix if needed + derived_id = derive_ontology_id(self.iri) + prefix_id = self._extract_ontology_id_from_prefixes() + + if derived_id: + self.ontology_id = derived_id + # If prefix exists but doesn't match ontology_id, rebind it + if prefix_id and prefix_id != derived_id: + self._rebind_prefix_to_ontology_id(prefix_id, derived_id) + elif prefix_id: + # Fallback to prefix if IRI derivation fails + self.ontology_id = prefix_id + # Set default values for fields that are still None + if self.version is None: + self.version = "1.0.0" + + self._compute_and_set_hash() + + # Always ensure graph is up to date with properties (including hash/parent_hashes) + self.sync_properties_to_graph() + + # Set initial_version if not already set + if self.initial_version is None and self.version: + # Normalize version to ensure semantic versioning + self.initial_version = self._normalize_version(self.version) + + @property + def prefix(self) -> str | None: + """Get the namespace prefix for this ontology. + + Returns: + str | None: The namespace prefix if found, None otherwise. + """ + prefixes = [ + prefix + for prefix, iri in self.graph.namespaces() + if iri == URIRef(self.namespace) + ] + if len(prefixes) == 0: + return None + else: + return prefixes[0] + + def is_null(self) -> bool: + """Check if this ontology is the null ontology. + + Returns: + bool: True if this is NULL_ONTOLOGY or has null characteristics. + """ + from ontocast.onto.null import NULL_ONTOLOGY + + # Check identity first (fastest) + if self is NULL_ONTOLOGY: + return True + # Check characteristics + return self.iri == ONTOLOGY_NULL_IRI and self.ontology_id is None + + def set_properties(self, **kwargs): + """Set ontology properties from keyword arguments and sync to graph. + Only update properties if they are missing (None or empty). + Also enforces ontology_id/iri consistency as in __init__, but only + if graph does not provide a valid pair. + """ + for k, v in kwargs.items(): + if hasattr(self, k): + current = getattr(self, k) + if not current and v: + setattr(self, k, v) + # Try to sync from graph first + graph_had_ontology = False + if self.graph: + self.sync_properties_from_graph() + if self.iri and not self.is_null() and self.ontology_id: + graph_had_ontology = True + if not graph_had_ontology: + if self.ontology_id and (not self.iri or self.is_null()): + self.iri = f"{self.current_domain}/{self.ontology_id}" + elif self.ontology_id and self.iri: + expected_iri = f"{self.current_domain}/{self.ontology_id}" + if not self.iri.endswith(f"/{self.ontology_id}"): + logger.warning( + f"Ontology IRI '{self.iri}' does not match expected " + f"'{expected_iri}'" + ) + elif not self.ontology_id and self.iri and not self.is_null(): + self.ontology_id = derive_ontology_id(self.iri) + self.sync_properties_to_graph() + + def sync_properties_to_graph(self): + """ + Update the RDF graph with the Ontology's properties. + Only sync properties for the entity that is explicitly typed as owl:Ontology. + Only add property triples if they do not already exist in the graph. + Optimized to avoid multiple loops over triples. + """ + + # Early return for NULL_ONTOLOGY - don't sync anything + if self.is_null(): + return + + if self.ontology_id is not None: + if not self.iri or self.is_null(): + self.iri = f"{self.current_domain}/{self.ontology_id}" + elif self.iri: + expected_iri = f"{self.current_domain}/{self.ontology_id}" + # Only fix IRI if it doesn't match expected pattern AND it's not from an external source + # Don't override IRIs that came from the graph or were explicitly provided + if ( + not self.iri.endswith(f"/{self.ontology_id}") + and self.iri != expected_iri + ): + # Check if IRI looks like it came from an external source (not our default domain) + if self.current_domain not in self.iri: + # IRI is from external source (e.g., graph) - don't override + logger.debug( + f"Ontology IRI '{self.iri}' does not match expected pattern " + f"'{expected_iri}', but keeping IRI (likely from graph or external source)" + ) + else: + # IRI is from our domain but doesn't match - fix it + logger.warning( + f"Ontology IRI '{self.iri}' does not match expected " + f"'{expected_iri}', fixing" + ) + self.iri = expected_iri + elif self.iri and not self.is_null(): + # Only derive ontology_id if this is not a null ontology + self.ontology_id = derive_ontology_id(self.iri) + + onto_iri = URIRef(self.iri) + g = self.graph + + onto_triple = [ + subj + for subj, _, o in g.triples((None, RDF.type, None)) + if o == OWL.Ontology + ] + if not onto_triple: + if onto_iri is not None: + # iri set as a property, but not in ontology + g.add((onto_iri, RDF.type, OWL.Ontology)) + else: + onto_iri_graph = onto_triple[0] + onto_iri = onto_iri_graph + + # Collect all predicates for this subject in one pass + existing_preds = set(p for _, p, _ in g.triples((onto_iri, None, None))) + + def add_if_missing(p, v): + if p not in existing_preds: + g.add((onto_iri, p, Literal(v))) + + # Add label/title + if self.title: + add_if_missing(RDFS.label, self.title) + if self.ontology_id: + add_if_missing(DCTERMS.title, self.ontology_id) + # Add description + if self.description: + add_if_missing(DCTERMS.description, self.description) + add_if_missing(RDFS.comment, self.description) + # Add version (update if exists) + if self.version: + # Remove existing version triples to update them + for _, _, obj in g.triples((onto_iri, OWL.versionInfo, None)): + g.remove((onto_iri, OWL.versionInfo, obj)) + # Add new version + g.add((onto_iri, OWL.versionInfo, Literal(self.version))) + # Add created_at if set (only if not already present in graph) + if self.created_at: + # Check if created_at already exists in graph - don't overwrite if present + existing_created = [ + str(obj) for _, _, obj in g.triples((onto_iri, DCTERMS.created, None)) + ] + if not existing_created: + # Add new created_at with datetime type + g.add( + ( + onto_iri, + DCTERMS.created, + Literal(self.created_at.isoformat(), datatype=XSD.dateTime), + ) + ) + # Add hash (only if not already present in graph) + # Use dcterms:identifier for hash (with "hash:" prefix to distinguish from other identifiers) + if self.hash: + # Check if hash already exists in graph + existing_hash = [ + str(obj) + for _, _, obj in g.triples((onto_iri, DCTERMS.identifier, None)) + if str(obj).startswith("hash:") + ] + if not existing_hash: + g.add((onto_iri, DCTERMS.identifier, Literal(f"hash:{self.hash}"))) + + # Add parent_hashes (multiple parents supported) + # Use prov:wasDerivedFrom for each parent hash (standard PROV predicate) + if self.parent_hashes: + # Get existing parent hashes to avoid duplicates + existing_parent_uris = { + str(obj) + for _, _, obj in g.triples((onto_iri, PROV.wasDerivedFrom, None)) + } + + # Add each parent hash as a URIRef if not already present + for parent_hash in self.parent_hashes: + parent_hash_uri = URIRef(f"urn:hash:{parent_hash}") + if str(parent_hash_uri) not in existing_parent_uris: + g.add((onto_iri, PROV.wasDerivedFrom, parent_hash_uri)) + + def _resolve_ontology_subject(self) -> URIRef | None: + """Resolve ontology subject used for metadata triples.""" + onto_subjects = [ + subj for subj, _, obj in self.graph.triples((None, RDF.type, OWL.Ontology)) + ] + if onto_subjects: + first_subject = onto_subjects[0] + if isinstance(first_subject, URIRef): + return first_subject + if self.iri and self.iri != ONTOLOGY_NULL_IRI: + return URIRef(self.iri) + return None + + def _clear_lineage_metadata_triples(self) -> None: + """Remove lineage metadata triples before writing a new version.""" + onto_iri = self._resolve_ontology_subject() + if onto_iri is None: + return + + for _, _, obj in list(self.graph.triples((onto_iri, DCTERMS.created, None))): + self.graph.remove((onto_iri, DCTERMS.created, obj)) + + for _, _, obj in list( + self.graph.triples((onto_iri, PROV.wasDerivedFrom, None)) + ): + self.graph.remove((onto_iri, PROV.wasDerivedFrom, obj)) + + for _, _, obj in list(self.graph.triples((onto_iri, DCTERMS.identifier, None))): + if isinstance(obj, Literal) and str(obj).startswith("hash:"): + self.graph.remove((onto_iri, DCTERMS.identifier, obj)) + + def derive_updated_version(self, updated_graph: RDFGraph) -> "Ontology": + """Create a new ontology version from an updated graph snapshot.""" + from copy import deepcopy + from datetime import datetime, timezone + + updated_ontology = deepcopy(self) + updated_ontology.graph = updated_graph + updated_ontology.parent_hashes = [self.hash] if self.hash else [] + updated_ontology.created_at = datetime.now(timezone.utc) + updated_ontology.hash = None + updated_ontology._clear_lineage_metadata_triples() + updated_ontology._compute_and_set_hash() + if not updated_ontology.hash and updated_ontology.parent_hashes: + updated_ontology.hash = updated_ontology.parent_hashes[0] + updated_ontology.sync_properties_to_graph() + return updated_ontology + + def _compute_and_set_hash(self) -> None: + """Compute the hash of the ontology graph and set it. + + The hash is computed from the canonicalized graph using SHA256. + The hash is computed from the graph WITHOUT hash/parent_hash triples, + as these are metadata about the graph, not part of the graph content. + This method should only be called if hash is not already set. + """ + if self.graph and len(self.graph) > 0: + try: + # Find the ontology IRI from the graph if not set + onto_iri = None + if self.iri and not self.is_null(): + onto_iri = URIRef(self.iri) + else: + # Try to find ontology IRI from graph + onto_triples = [ + subj + for subj, _, o in self.graph.triples((None, RDF.type, None)) + if o == OWL.Ontology + ] + if onto_triples: + onto_iri = onto_triples[0] + + # Create a temporary graph without hash/parent_hash triples for hashing + temp_graph = RDFGraph() + + # Copy all triples except metadata triples - these are metadata, not content + # Metadata to exclude: hash, parent_hash, created_at, version, title, description + for s, p, o in self.graph: + # Skip metadata triples for the ontology IRI + if onto_iri and s == onto_iri: + if ( + p == DCTERMS.identifier + and isinstance(o, Literal) + and str(o).startswith("hash:") + ): + continue # Skip hash identifier + if p == PROV.wasDerivedFrom: + continue # Skip parent hash + if p == DCTERMS.created: + continue # Skip created_at + if p == OWL.versionInfo: + continue # Skip version + if p == RDFS.label: + continue # Skip title/label + if p == DCTERMS.title: + continue # Skip title + if p == DCTERMS.description: + continue # Skip description + if p == RDFS.comment: + continue # Skip description (comment) + temp_graph.add((s, p, o)) + + # Copy namespace bindings + for prefix, uri in self.graph.namespaces(): + temp_graph.bind(prefix, uri) + + # Use RDFGraph.hash() directly + self.hash = temp_graph.hash() + logger.debug( + f"Computed hash for ontology {self.ontology_id}: {self.hash}" + ) + except Exception as e: + logger.warning( + f"Failed to compute hash for ontology {self.ontology_id}: {e}" + ) + # Set a placeholder hash if computation fails + self.hash = None + + def _normalize_version(self, version: str) -> str: + """Normalize version string to semantic versioning format. + + Handles various version formats and converts them to MAJOR.MINOR.PATCH: + - "3.5.1" -> "3.5.1" (already valid) + - "3.5" -> "3.5.0" (adds missing PATCH) + - "3" -> "3.0.0" (adds missing MINOR and PATCH) + - Invalid formats -> "1.0.0" + + Args: + version: The version string to normalize + + Returns: + A valid semantic version string (MAJOR.MINOR.PATCH) + """ + # Already valid semantic version + match = re.match(r"^(\d+)\.(\d+)\.(\d+)$", version) + if match: + return version + + # Try to parse as MAJOR.MINOR (missing PATCH) + match = re.match(r"^(\d+)\.(\d+)$", version) + if match: + major, minor = match.groups() + normalized = f"{major}.{minor}.0" + logger.info( + f"Version '{version}' missing PATCH component, normalized to '{normalized}'" + ) + return normalized + + # Try to parse as just MAJOR (missing MINOR and PATCH) + match = re.match(r"^(\d+)$", version) + if match: + major = match.group(1) + normalized = f"{major}.0.0" + logger.info( + f"Version '{version}' missing MINOR and PATCH components, normalized to '{normalized}'" + ) + return normalized + + # Invalid format, use default + logger.warning( + f"Version '{version}' does not match any recognized format, " + f"normalizing to '1.0.0'" + ) + return "1.0.0" + + def _analyze_version_increment_type( + self, updates: list[GraphUpdate] + ) -> tuple[str, str]: + """Analyze the updates to determine the appropriate version increment type. + + Args: + updates: List of GraphUpdate objects that were applied to the ontology + + Returns: + Tuple of (increment_type, reason) where increment_type is + 'major', 'minor', or 'patch' and reason explains the decision + """ + if not updates: + return ("patch", "No updates to analyze") + + # Count operations by type + total_deletes = 0 + total_inserts = 0 + + # Track specific types of changes + class_changes = 0 + property_changes = 0 + instance_changes = 0 + + for update in updates: + for op in update.triple_operations: + if isinstance(op, TripleOp): + if op.type == "delete": + total_deletes += len(op.graph) + # Check if deleting core ontology constructs + for subject, predicate, object_ in op.graph: + predicate_str = str(predicate) + object_str = str(object_) + if "rdf:type" in predicate_str: + if any( + cls in object_str.lower() + for cls in ["class", "property", "ontology"] + ): + if ( + "owl:class" in object_str + or "rdfs:class" in object_str + ): + class_changes += 1 + elif "owl:ontology" in object_str: + class_changes += 1 + else: # insert + total_inserts += len(op.graph) + # Check if adding core ontology constructs + for subject, predicate, object_ in op.graph: + predicate_str = str(predicate) + object_str = str(object_) + if "rdf:type" in predicate_str: + if ( + "owl:class" in object_str + or "rdfs:class" in object_str + ): + class_changes += 1 + elif "owl:ontology" in object_str: + class_changes += 1 + elif ( + "owl:objectproperty" in object_str + or "owl:datatypeproperty" in object_str + or "rdf:property" in object_str + ): + property_changes += 1 + else: + instance_changes += 1 + + # Decision logic - conservative approach, favor PATCH + + # Check for substantial breaking changes first (MAJOR) + if total_deletes > 5 and (class_changes > 2 or property_changes > 3): + reason = ( + f"MAJOR: Deleted {total_deletes} triples including " + f"{class_changes} classes and {property_changes} properties " + "(significant breaking change)" + ) + return ("major", reason) + + # Any deletions trigger MINOR (even small ones indicate changes) + if total_deletes > 0: + reason = ( + f"MINOR: Deleted {total_deletes} triples " + f"({class_changes} classes, {property_changes} properties removed)" + ) + return ("minor", reason) + + # Only increment MINOR for substantial new features (>=5 classes or properties) + if class_changes >= 5 or property_changes >= 5: + reason = ( + f"MINOR: Added {total_inserts} triples including " + f"{class_changes} classes and {property_changes} properties " + "(substantial new features)" + ) + return ("minor", reason) + + # Default to PATCH for most additions + # This includes: instances, descriptions, small numbers of classes/properties + reason = f"PATCH: Added {total_inserts} triples" + if class_changes > 0 or property_changes > 0: + reason += f" ({class_changes} classes, {property_changes} properties)" + reason += " (updates to existing structures)" + return ("patch", reason) + + def _increment_version(self, increment_type: str = "patch") -> None: + """Increment the ontology version using semantic versioning. + + Args: + increment_type: Type of increment - 'major', 'minor', or 'patch' + """ + # If version is None, set to default + if self.version is None: + self.version = "1.0.0" + return + + # Normalize to ensure semantic versioning + normalized_version = self._normalize_version(self.version) + if normalized_version != self.version: + logger.warning( + f"Version '{self.version}' normalized to '{normalized_version}' " + "before incrementing" + ) + self.version = normalized_version + + # Parse and increment version string based on increment_type + match = re.match(r"^(\d+)\.(\d+)\.(\d+)$", self.version) + if match: + major, minor, patch = map(int, match.groups()) + + if increment_type == "major": + major += 1 + minor = 0 + patch = 0 + logger.info( + f"Incrementing MAJOR version from {self.version} to {major}.{minor}.{patch}" + ) + elif increment_type == "minor": + minor += 1 + patch = 0 + logger.info( + f"Incrementing MINOR version from {self.version} to {major}.{minor}.{patch}" + ) + else: # patch + patch += 1 + logger.info( + f"Incrementing PATCH version from {self.version} to {major}.{minor}.{patch}" + ) + + self.version = f"{major}.{minor}.{patch}" + else: + # Should never reach here after normalization, but handle gracefully + logger.error(f"Version '{self.version}' still invalid after normalization") + self.version = "1.0.1" + + logger.info(f"Incremented ontology version to {self.version}") + + def mark_as_updated(self, updates: list[GraphUpdate] | None = None) -> None: + """Mark the ontology version and update semantic version. + + Note: Ontologies are immutable - modifications create new versions. + This method only updates the semantic version number, not the creation timestamp. + The creation timestamp is set when a new version is created. + + Analyzes the updates to determine appropriate version increment type. + + Args: + updates: Optional list of GraphUpdate objects that were applied. + If provided, analyzes them to determine MAJOR/MINOR/PATCH increment. + """ + # Analyze updates to determine increment type + if updates: + increment_type, reason = self._analyze_version_increment_type(updates) + logger.info(f"Version increment analysis: {reason}") + self._increment_version(increment_type) + else: + # Default to patch increment if no updates provided + self._increment_version("patch") + + logger.info( + f"Updated semantic version for ontology {self.ontology_id} to {self.version}" + ) + + def _extract_ontology_id_from_prefixes(self) -> str | None: + """Extract ontology_id from namespace prefixes that match the ontology IRI. + + Looks for prefixes where the namespace URI matches the ontology IRI or namespace. + For example, if IRI is 'https://growgraph.dev/fcaont' and there's a prefix + 'fca' with namespace 'https://growgraph.dev/fcaont#', returns 'fca'. + + Returns: + str | None: The prefix name if found, None otherwise. + """ + if not self.graph or not self.iri or self.iri == ONTOLOGY_NULL_IRI: + return None + + # Try exact IRI match first + ontology_namespace = iri2namespace(self.iri, ontology=True) + + for prefix, namespace_uri in self.graph.namespaces(): + namespace_str = str(namespace_uri) + # Check if namespace matches ontology IRI or namespace + if namespace_str == self.iri or namespace_str == ontology_namespace: + if prefix and prefix not in [ + "rdf", + "rdfs", + "owl", + "xsd", + "dc", + "dcterms", + "skos", + "foaf", + "schema", + "prov", + ]: + logger.debug(f"Found prefix '{prefix}' matching IRI '{self.iri}'") + return prefix + + return None + + def _rebind_prefix_to_ontology_id(self, old_prefix: str, ontology_id: str) -> None: + """Rebind a prefix to match the ontology_id. + + If a prefix exists that matches the ontology IRI but has a different name + than the ontology_id, rebind it to use the ontology_id as the prefix name. + This ensures consistency between the prefix name and ontology_id. + + Args: + old_prefix: The existing prefix name that needs to be rebound. + ontology_id: The ontology_id that should be used as the new prefix name. + """ + if not self.graph or not self.iri or self.iri == ONTOLOGY_NULL_IRI: + return + + ontology_namespace = iri2namespace(self.iri, ontology=True) + + # Find the namespace URI for the old prefix + old_namespace_uri = None + for prefix, namespace_uri in self.graph.namespaces(): + if prefix == old_prefix: + old_namespace_uri = str(namespace_uri) + break + + if old_namespace_uri and old_namespace_uri == ontology_namespace: + # Only rebind if the namespace matches + # Bind the new prefix with ontology_id (this will override if it exists) + from rdflib import Namespace + + ns = Namespace(ontology_namespace) + self.graph.namespace_manager.bind(ontology_id, ns, override=True) + + # If old prefix is different, we can optionally remove it + # But keep it for now to avoid breaking existing references in the graph + # The new prefix will be used going forward + logger.debug( + f"Rebound prefix: '{old_prefix}' -> '{ontology_id}' " + f"for namespace '{ontology_namespace}'" + ) + + def sync_properties_from_graph(self): + """ + Update Ontology properties from the RDF graph if present, + but only if missing, and only for entities explicitly typed as owl:Ontology. + Optimized to avoid multiple loops over triples. + """ + g = self.graph + if not g or len(g) == 0: + return + + # Only proceed if this subject is explicitly typed as owl:Ontology + onto_triple = [ + subj + for subj, _, o in g.triples((None, RDF.type, None)) + if o == OWL.Ontology + ] + if not onto_triple: + # No owl:Ontology found - try to extract IRI from prefixes as fallback + if not self.iri or self.iri == ONTOLOGY_NULL_IRI: + # Look for prefixes that might indicate the ontology IRI + for prefix, namespace_uri in g.namespaces(): + namespace_str = str(namespace_uri).rstrip("#/") + # Skip standard prefixes + if prefix and prefix not in [ + "rdf", + "rdfs", + "owl", + "xsd", + "dc", + "dcterms", + "skos", + "foaf", + "schema", + "prov", + ]: + # Use this namespace as potential IRI + self.iri = namespace_str + self.ontology_id = prefix + logger.debug( + f"No owl:Ontology found, extracted IRI '{self.iri}' and " + f"ontology_id '{self.ontology_id}' from prefix '{prefix}'" + ) + return + return + + onto_iri = onto_triple[0] + iri_str = str(onto_iri) + + # Strip hash fragment from IRI to ensure simplified representation + # Hash fragments are long hex strings (64+ chars) used for versioning + if "#" in iri_str: + base_iri, fragment = iri_str.rsplit("#", 1) + # Check if fragment is a hash (long hex string) or version (v1.2.3) + if len(fragment) > 20 and all( + c in "0123456789abcdef" for c in fragment.lower() + ): + # Looks like a hash - use base IRI only + iri_str = base_iri + logger.debug( + f"Stripped hash fragment from IRI in graph: {fragment[:20]}..." + ) + elif fragment.startswith("v") and re.match(r"^v\d+\.\d+\.\d+$", fragment): + # Semantic version fragment - use base IRI only + iri_str = base_iri + logger.debug(f"Stripped version fragment from IRI in graph: {fragment}") + + # Set IRI from graph (this is authoritative) + if not self.iri or self.iri == ONTOLOGY_NULL_IRI: + self.iri = iri_str + elif self.iri != iri_str: + # Graph has different IRI - prefer graph IRI but log the difference + logger.debug( + f"Graph IRI '{iri_str}' differs from provided IRI '{self.iri}', " + f"using graph IRI" + ) + self.iri = iri_str + + # Extract ontology_id: prefer derivation from IRI over prefix + # If both exist, use IRI-derived ontology_id and rebind prefix to match + if not self.ontology_id: + # First try to derive from IRI (preferred) + derived_id = derive_ontology_id(self.iri) + prefix_id = self._extract_ontology_id_from_prefixes() + + if derived_id: + self.ontology_id = derived_id + # If prefix exists but doesn't match ontology_id, rebind it + if prefix_id and prefix_id != derived_id: + self._rebind_prefix_to_ontology_id(prefix_id, derived_id) + elif prefix_id: + # Fallback to prefix if IRI derivation fails + self.ontology_id = prefix_id + + # Collect all predicates and objects for this subject in one pass + pred_map = defaultdict(list) + for _, p, o in g.triples((onto_iri, None, None)): + pred_map[p].append(o) + + # Title: try rdfs:label, dcterms:title + if self.title is None: + title = None + if RDFS.label in pred_map: + title = str(pred_map[RDFS.label][0]) + elif DCTERMS.title in pred_map: + title = str(pred_map[DCTERMS.title][0]) + if title: + self.title = title + + # Description: try dcterms:description, rdfs:comment + if self.description is None: + description = None + if DCTERMS.description in pred_map: + description = str(pred_map[DCTERMS.description][0]) + elif RDFS.comment in pred_map: + description = str(pred_map[RDFS.comment][0]) + if description: + self.description = description + # Version + if self.version is None: + if OWL.versionInfo in pred_map: + version_str = str(pred_map[OWL.versionInfo][0]) + self.version = self._normalize_version(version_str) + # Created at - only read if not already set (preserve existing value) + if not getattr(self, "created_at", None): + if DCTERMS.created in pred_map: + # Get the first created date + created_str = str(pred_map[DCTERMS.created][0]) + # Try to parse as datetime + try: + self.created_at = datetime.fromisoformat( + created_str.replace("Z", "+00:00") + ) + except (ValueError, AttributeError): + # If parsing fails, keep it as None + pass + # Short name: try dcterms:title if not already used for title + if not getattr(self, "ontology_id", None): + if DCTERMS.title in pred_map: + self.ontology_id = str(pred_map[DCTERMS.title][0]) + # Hash: read from dcterms:identifier with "hash:" prefix if present + if self.hash is None: + if DCTERMS.identifier in pred_map: + for obj in pred_map[DCTERMS.identifier]: + obj_str = str(obj) + if obj_str.startswith("hash:"): + self.hash = obj_str[5:] # Remove "hash:" prefix + break + + # Parent_hashes: read all from prov:wasDerivedFrom if present + if len(self.parent_hashes) == 0: + if PROV.wasDerivedFrom in pred_map: + for parent_uri_obj in pred_map[PROV.wasDerivedFrom]: + parent_uri = str(parent_uri_obj) + # Extract hash from URN format: urn:hash: + if parent_uri.startswith("urn:hash:"): + parent_hash = parent_uri[9:] # Remove "urn:hash:" prefix + self.parent_hashes.append(parent_hash) + + def __iadd__(self, other: Union["Ontology", RDFGraph]) -> "Ontology": + """In-place addition operator for Ontology instances. + + Merges the RDF graphs and takes properties from the right-hand operand. + + Args: + other: The ontology or graph to add to this one. + + Returns: + Ontology: self after modification. + """ + if isinstance(other, Ontology): + self.graph += other.graph + self.title = other.title + self.ontology_id = other.ontology_id + self.description = other.description + self.iri = other.iri + self.version = other.version + self.created_at = other.created_at + self.initial_version = other.initial_version + self.hash = other.hash + self.parent_hashes = other.parent_hashes + else: + self.graph += other + return self + + @classmethod + def from_file(cls, file_path: pathlib.Path, format: str = "turtle", **kwargs): + """Create an Ontology instance by loading a graph from a file. + + Args: + file_path: Path to the ontology file. + format: Format of the input file (default: "turtle"). + **kwargs: Additional arguments to pass to the constructor. + + Returns: + Ontology: A new Ontology instance. + """ + graph: RDFGraph = RDFGraph() + graph.parse(file_path, format=format) + return cls(graph=graph, **kwargs) + + def describe(self) -> str: + """Get a human-readable description of the ontology. + + Returns: + str: A formatted description string. + """ + return ( + f"Ontology id: {self.ontology_id}\n" + f"Description: {self.description}\n" + f"Ontology IRI: {self.iri}\n" + ) + + def to_lineage_node(self) -> dict: + """Convert ontology to a lineage node representation. + + Returns a dictionary suitable for constructing a meta-graph representing + the ontology lineage. This representation can be used to build the full + ontology lineage graph. + + Returns: + dict: Lineage node with hash, parents, and metadata. + + Example: + >>> ont = Ontology(iri="https://example.org/ont", hash="abc123", parent_hashes=["def456"]) + >>> node = ont.to_lineage_node() + >>> node["hash"] + 'abc123' + >>> node["parents"] + ['def456'] + """ + return { + "hash": self.hash, + "parents": self.parent_hashes, + "iri": self.iri, + "title": self.title, + "version": self.version, + "created_at": self.created_at.isoformat() if self.created_at else None, + } + + @staticmethod + def build_lineage_graph(ontologies: list["Ontology"]): + """Build a NetworkX directed graph representing the lineage of all given ontologies. + + Constructs a directed graph where nodes represent ontologies (by their hash) + and edges represent parent-child relationships. Each node includes metadata + as node attributes (iri, title, version, created_at, etc.). + + Args: + ontologies: List of Ontology instances to include in the lineage graph. + + Returns: + networkx.DiGraph: A directed graph representing the full ontology lineage. + Nodes are identified by hash strings, with edges from children to parents. + Each node has attributes: iri, title, ontology_id, version, created_at. + + Example: + >>> import networkx as nx + >>> ont1 = Ontology(iri="https://example.org/ont1", hash="abc123") + >>> ont2 = Ontology(iri="https://example.org/ont2", hash="def456", parent_hashes=["abc123"]) + >>> lineage = Ontology.build_lineage_graph([ont1, ont2]) + >>> isinstance(lineage, nx.DiGraph) + True + >>> "def456" in lineage.nodes() + True + >>> "abc123" in lineage["def456"] # Check if edge exists + True + """ + import networkx as nx + + lineage_graph = nx.DiGraph() + + for ontology in ontologies: + if not ontology.hash: + logger.warning( + f"Skipping ontology {ontology.iri} in lineage graph: no hash" + ) + continue + + # Add node with metadata attributes + lineage_graph.add_node( + ontology.hash, + iri=ontology.iri, + title=ontology.title, + ontology_id=ontology.ontology_id, + version=ontology.version, + created_at=ontology.created_at.isoformat() + if ontology.created_at + else None, + ) + + # Add edges from this ontology to its parents + if ontology.parent_hashes: + for parent_hash in ontology.parent_hashes: + # Ensure parent node exists (even if not in the ontologies list) + if parent_hash not in lineage_graph: + lineage_graph.add_node(parent_hash) + lineage_graph.add_edge(ontology.hash, parent_hash) + + return lineage_graph + + def add_parent_hash(self, parent_hash: str) -> None: + """Add a parent hash to the ontology's parent list. + + Appends the given hash to parent_hashes if not already present, + and updates the RDF graph accordingly by adding a new prov:wasDerivedFrom triple. + + Args: + parent_hash: The hash of the parent ontology to add. + + Example: + >>> ont = Ontology(iri="https://example.org/ont", hash="abc123") + >>> ont.add_parent_hash("def456") + >>> "def456" in ont.parent_hashes + True + """ + if parent_hash not in self.parent_hashes: + self.parent_hashes.append(parent_hash) + # Update graph + if self.iri and not self.is_null(): + onto_iri = URIRef(self.iri) + parent_hash_uri = URIRef(f"urn:hash:{parent_hash}") + self.graph.add((onto_iri, PROV.wasDerivedFrom, parent_hash_uri)) + logger.debug( + f"Added parent hash {parent_hash} to ontology {self.ontology_id}" + ) + + def validate_lineage(self) -> list[str]: + """Validate the ontology lineage for integrity issues. + + Checks for cycles and ensures that self.hash is not in its own parent_hashes. + Returns a list of warning messages if any issues are found. + + Returns: + list[str]: List of warning messages describing any lineage issues found. + Empty list if lineage is valid. + + Example: + >>> ont = Ontology(iri="https://example.org/ont", hash="abc123", parent_hashes=["abc123"]) + >>> warnings = ont.validate_lineage() + >>> len(warnings) > 0 + True + """ + warnings = [] + + if not self.hash: + return warnings + + # Check if hash is in its own parent_hashes + if self.parent_hashes and self.hash in self.parent_hashes: + warnings.append( + f"Ontology {self.ontology_id} (hash: {self.hash[:8]}...) " + "has itself as a parent, which may indicate a cycle" + ) + + # Check for cycles using a simple depth-first search + visited = set() + to_visit = [(self.hash, [self.hash])] + + while to_visit: + current_hash, path = to_visit.pop() + if current_hash in visited: + continue + visited.add(current_hash) + + # Find ontology with this hash in the graph + # This is a simplified check - in practice, you'd need access to all ontologies + # For now, we just check immediate parents + if self.parent_hashes: + for parent_hash in self.parent_hashes: + if parent_hash == current_hash and len(path) > 1: + warnings.append( + f"Potential cycle detected in lineage: " + f"{' -> '.join(path)} -> {parent_hash}" + ) + elif parent_hash not in visited: + to_visit.append((parent_hash, path + [parent_hash])) + + if warnings: + for warning in warnings: + logger.warning(warning) + + return warnings diff --git a/ontology_platform/vendored/ontocast/ontocast/onto/ontology_operations.py b/ontology_platform/vendored/ontocast/ontocast/onto/ontology_operations.py new file mode 100644 index 0000000..df84750 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/onto/ontology_operations.py @@ -0,0 +1,296 @@ +import importlib +import logging +from datetime import datetime, timezone +from pathlib import Path + +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.tool import FusekiTripleStoreManager, OntologyManager + +logger = logging.getLogger(__name__) + + +def merge_ontologies(onto1: Ontology, onto2: Ontology) -> Ontology: + """Merge two ontologies algorithmically. + + This performs a union merge of the two ontology graphs, mapping contradictions. + The result has both ontologies as parents. This is similar to a git merge: + - Takes union of all triples from both ontologies + - Detects contradictions (same subject-predicate with different objects) + - Creates a new ontology with both parents + - Sets created_at to merge time + + Args: + onto1: First ontology to merge + onto2: Second ontology to merge + + Returns: + Ontology: Merged ontology with both parents + + Raises: + ValueError: If ontologies have different IRIs + """ + # Validate that both ontologies have the same IRI + if onto1.iri != onto2.iri: + raise ValueError( + f"Cannot merge ontologies with different IRIs: {onto1.iri} != {onto2.iri}" + ) + + # Ensure both ontologies have hashes + if not onto1.hash: + onto1._compute_and_set_hash() + if not onto2.hash: + onto2._compute_and_set_hash() + + if not onto1.hash or not onto2.hash: + raise ValueError("Cannot merge ontologies without hashes") + + # Create merged graph (union) - use RDFGraph's __add__ operator + merged_graph = onto1.graph + onto2.graph + + # Map contradictions (same subject-predicate with different objects) + contradictions = _find_contradictions(onto1.graph, onto2.graph) + if contradictions: + logger.warning(f"Found {len(contradictions)} contradictions in merge") + for (s, p), (obj1_set, obj2) in contradictions.items(): + logger.debug(f"Contradiction: {s} {p} -> {obj1_set} vs {obj2}") + # For now, keep both objects (RDF allows multiple values) + # In future LLM-based merge, this would be resolved intelligently + + # Create merged ontology + # Note: sync_properties_to_graph() will remove existing versionInfo triples, + # so we need to preserve them manually after creation + merged_ontology = Ontology( + graph=merged_graph, + iri=onto1.iri, # Use IRI from first ontology (should be same) + title=onto1.title or onto2.title, + description=onto1.description or onto2.description, + ontology_id=onto1.ontology_id or onto2.ontology_id, + version=onto1.version or onto2.version or "1.0.0", + parent_hashes=[onto1.hash, onto2.hash], + created_at=datetime.now(timezone.utc), + ) + + # Compute hash for merged ontology + # Note: Hash excludes metadata (version, title, description, created_at, hash, parent_hash) + # so it only reflects the actual ontology content (classes, properties, etc.) + merged_ontology._compute_and_set_hash() + + logger.info( + f"Merged ontologies {onto1.hash[:8]}... " + f"and {onto2.hash[:8]}... " + f"-> {merged_ontology.hash[:8] if merged_ontology.hash else 'None'}..." + ) + + return merged_ontology + + +def _find_contradictions(graph1: RDFGraph, graph2: RDFGraph) -> dict: + """Find contradictions between two graphs. + + Contradictions are triples with the same subject-predicate but different objects. + Note: RDF allows multiple values for the same property, so this detects potential + conflicts that might need resolution in an LLM-based merge. + + Args: + graph1: First graph + graph2: Second graph + + Returns: + dict: Dictionary mapping (subject, predicate) to (set of objects from graph1, object from graph2) tuples + """ + contradictions = {} + + # Build index of graph1: (subject, predicate) -> set of objects + graph1_index: dict[tuple, set] = {} + for s, p, o in graph1: + key = (s, p) + if key not in graph1_index: + graph1_index[key] = set() + # Use string representation for comparison + graph1_index[key].add(str(o)) + + # Check graph2 against graph1 + for s, p, o in graph2: + key = (s, p) + if key in graph1_index: + # Check if objects differ + graph1_objects = graph1_index[key] + obj2_str = str(o) + if obj2_str not in graph1_objects: + # Contradiction found - different object values + contradictions[key] = (graph1_objects, obj2_str) + + return contradictions + + +def plot_ontology_graph( + ontology_manager: OntologyManager, + output_path: Path, + iri: str | None = None, +) -> None: + """Plot the ontology version graph using pygraphviz. + + Args: + ontology_manager: The ontology manager containing ontologies + output_path: Path to save the graph image + iri: Optional IRI to plot (if None, plots all ontologies) + """ + try: + pgv = importlib.import_module("pygraphviz") + except ImportError: + logger.error("pygraphviz not installed. Cannot plot graph.") + logger.info("Install with: pip install pygraphviz") + return + + # Get ontologies to plot + if iri: + if iri not in ontology_manager.ontology_versions: + logger.warning(f"No ontologies found for IRI: {iri}") + return + ontologies = ontology_manager.ontology_versions[iri] + else: + ontologies = [ + o + for versions in ontology_manager.ontology_versions.values() + for o in versions + ] + + if not ontologies: + logger.warning("No ontologies to plot") + return + + # Create graph + viz = pgv.AGraph(directed=True, nodesep=0.7, ranksep=0.5) + + # Add nodes + for onto in ontologies: + if not onto.hash: + continue + node_id = onto.hash[:12] # Use first 12 chars of hash as node ID + label = f"{onto.ontology_id or 'ont'}\n{onto.hash[:8]}..." + if onto.created_at: + label += f"\n{onto.created_at.strftime('%Y-%m-%d')}" + viz.add_node( + node_id, + label=label, + style="filled", + fillcolor="#a9cca9", + fontsize=10, + ) + + # Add edges (parent relationships) + for onto in ontologies: + if not onto.hash: + continue + node_id = onto.hash[:12] + for parent_hash in onto.parent_hashes: + # Find parent node + parent_onto = None + for o in ontologies: + if o.hash == parent_hash: + parent_onto = o + break + if parent_onto: + parent_id = parent_hash[:12] + viz.add_edge(parent_id, node_id, style="solid") + + # Highlight terminal ontologies + terminals = ( + ontology_manager.get_terminal_ontologies_by_iri(iri) + if iri + else ontology_manager.get_terminal_ontologies_by_iri(None) + ) + for terminal in terminals: + if terminal.hash: + node_id = terminal.hash[:12] + node = viz.get_node(node_id) + if node: + node.attr["fillcolor"] = "#ffdb99" # Orange for terminals + + # Save graph + output_path.parent.mkdir(parents=True, exist_ok=True) + viz.draw(str(output_path), format="png", prog="dot", args="-Gdpi=300") + logger.info(f"Saved ontology graph to {output_path}") + + +async def merge_terminal_ontologies( + fuseki_manager: FusekiTripleStoreManager, + ontology_manager: OntologyManager, + iri: str, +) -> Ontology | None: + """Merge terminal ontologies for a given IRI. + + Fetches all terminal ontologies from Fuseki and merges them pair-wise + until only one remains. + + Args: + fuseki_manager: Fuseki triple store manager + ontology_manager: Ontology manager to add merged ontologies to + iri: IRI of the ontology to merge + + Returns: + Ontology: The final merged ontology, or None if no ontologies found + """ + # Fetch all ontologies from Fuseki + logger.info(f"Fetching ontologies for IRI: {iri}") + all_ontologies = await fuseki_manager.afetch_ontologies() + + # Filter by IRI and add to ontology manager + matching_ontologies = [o for o in all_ontologies if o.iri == iri] + if not matching_ontologies: + logger.warning(f"No ontologies found for IRI: {iri}") + return None + + logger.info(f"Found {len(matching_ontologies)} ontologies for IRI: {iri}") + + # Add all to ontology manager + for onto in matching_ontologies: + ontology_manager.add_ontology(onto) + + # Get terminal ontologies + terminals = ontology_manager.get_terminal_ontologies_by_iri(iri) + logger.info(f"Found {len(terminals)} terminal ontologies") + + # Merge pair-wise until only one remains + while len(terminals) > 1: + # Sort by created_at (oldest first) + terminals_with_time = [t for t in terminals if t.created_at is not None] + terminals_without_time = [t for t in terminals if t.created_at is None] + + # Sort terminals with time by created_at + terminals_with_time.sort(key=lambda x: x.created_at) + + # Combine: terminals with time (sorted) + terminals without time + sorted_terminals = terminals_with_time + terminals_without_time + + if len(sorted_terminals) < 2: + break + + # Take the two oldest + onto1 = sorted_terminals[0] + onto2 = sorted_terminals[1] + + logger.info( + f"Merging ontologies: {onto1.hash[:8] if onto1.hash else 'None'}... " + f"and {onto2.hash[:8] if onto2.hash else 'None'}..." + ) + + # Merge + merged = merge_ontologies(onto1, onto2) + + # Add merged ontology to manager + ontology_manager.add_ontology(merged) + + # Update terminals list + terminals = ontology_manager.get_terminal_ontologies_by_iri(iri) + logger.info(f"After merge: {len(terminals)} terminal ontologies remaining") + + if terminals: + logger.info( + f"Final terminal ontology: {terminals[0].hash[:8] if terminals[0].hash else 'None'}..." + ) + return terminals[0] + else: + logger.warning("No terminal ontologies remaining after merge") + return None diff --git a/ontology_platform/vendored/ontocast/ontocast/onto/rdfgraph.py b/ontology_platform/vendored/ontocast/ontocast/onto/rdfgraph.py new file mode 100644 index 0000000..a26f7b3 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/onto/rdfgraph.py @@ -0,0 +1,757 @@ +import json +import logging +import re +from collections import defaultdict +from collections.abc import Iterable, Mapping +from contextvars import ContextVar +from typing import Any, Union + +from pydantic import GetCoreSchemaHandler +from pydantic_core import core_schema +from pyld import jsonld +from rdflib import Graph, Literal, Namespace, URIRef +from rdflib.namespace import NamespaceManager + +from ontocast.onto.constants import COMMON_PREFIXES +from ontocast.util import render_text_hash + +logger = logging.getLogger(__name__) + +PREFIX_PATTERN = re.compile(r"@prefix\s+(\w+):\s+<[^>]+>\s+\.") +# Pattern to match prefix usage: prefix:something (not in @prefix declarations) +PREFIX_USAGE_PATTERN = re.compile(r"\b([a-zA-Z_][a-zA-Z0-9_]*):[^\s]") + +# Context variable to store known prefixes during parsing +_known_prefixes_context: ContextVar[dict[str, str] | None] = ContextVar[ + dict[str, str] | None +]("known_prefixes", default=None) + + +class RDFGraph(Graph): + """Subclass of rdflib.Graph with Pydantic schema support. + + This class extends rdflib.Graph to provide serialization and deserialization + capabilities for Pydantic models, with special handling for Turtle format. + """ + + @classmethod + def __get_pydantic_core_schema__(cls, _source_type, handler: GetCoreSchemaHandler): + """Get the Pydantic core schema for this class. + + Args: + _source_type: The source type. + handler: The core schema handler. + + Returns: + A union schema that handles both Graph instances and string conversion. + Supports both Turtle and JSON-LD string formats. + """ + return core_schema.union_schema( + [ + core_schema.is_instance_schema(cls), + core_schema.chain_schema( + [ + core_schema.str_schema(), + core_schema.no_info_plain_validator_function(cls._from_str), + ] + ), + ], + serialization=core_schema.plain_serializer_function_ser_schema( + cls._to_turtle_str, + info_arg=False, + return_schema=core_schema.str_schema(), + ), + ) + + def __add__(self, other: Union["RDFGraph", Graph, Iterable]) -> "RDFGraph": + """Addition operator for RDFGraph instances. + + Merges the RDF graphs while maintaining the RDFGraph type. + + Args: + other: The graph to add to this one. + + Returns: + RDFGraph: A new RDFGraph containing the merged triples. + """ + # Create a new RDFGraph instance + result = RDFGraph() + + # Copy all triples from both graphs + for triple in self: + result.add(triple) + for triple in other: + result.add(triple) + + # Copy namespace bindings from self + for prefix, uri in self.namespaces(): + result.bind(prefix, uri) + + # Copy namespace bindings from other if it's a Graph + if isinstance(other, Graph): + for prefix, uri in other.namespaces(): + result.bind(prefix, uri) + + return result + + def __iadd__(self, other: Union["RDFGraph", Graph, Iterable]) -> "RDFGraph": + """In-place addition operator for RDFGraph instances. + + Merges the RDF graphs while maintaining the RDFGraph type and binding prefixes. + + Args: + other: The graph to add to this one. + + Returns: + RDFGraph: self after modification. + """ + # Use __add__ to get the merged result with proper prefix binding + result = self.__add__(other) + + # Clear current graph and copy the result + self.remove((None, None, None)) # Remove all triples + + # Copy all triples from result + for triple in result: + self.add(triple) + + # Copy namespace bindings from result + for prefix, uri in result.namespaces(): + self.bind(prefix, uri) + + return self + + def copy(self) -> "RDFGraph": + """Create a copy of this RDFGraph. + + Returns: + RDFGraph: A new RDFGraph instance with all triples and namespace bindings copied. + """ + result = RDFGraph() + + # Copy all triples + for triple in self: + result.add(triple) + + # Copy namespace bindings + for prefix, uri in self.namespaces(): + result.bind(prefix, uri) + + return result + + @staticmethod + def _ensure_prefixes(turtle_str: str) -> str: + """Ensure all common prefixes and used custom prefixes are declared in the Turtle string. + + This method: + 1. Adds missing common prefixes (rdf, rdfs, owl, etc.) + 2. Detects prefixes that are used but not declared + 3. Adds declarations for used prefixes if they're available in the context + + Args: + turtle_str: The input Turtle string. + + Returns: + str: The Turtle string with all necessary prefixes declared. + """ + declared_prefixes = set( + match.group(1) for match in PREFIX_PATTERN.finditer(turtle_str) + ) + + # Add missing common prefixes + missing_common = { + prefix: uri + for prefix, uri in COMMON_PREFIXES.items() + if prefix not in declared_prefixes + } + + # Detect prefixes that are used but not declared + used_prefixes = set() + for match in PREFIX_USAGE_PATTERN.finditer(turtle_str): + prefix = match.group(1) + # Skip if already declared or is a common prefix we're about to add + if prefix not in declared_prefixes and prefix not in missing_common: + used_prefixes.add(prefix) + + # Get known prefixes from context (set by caller) + known_prefixes = _known_prefixes_context.get() + missing_custom = {} + if known_prefixes and used_prefixes: + for prefix in used_prefixes: + if prefix in known_prefixes: + namespace_uri = known_prefixes[prefix] + # Format as Turtle prefix declaration + if not namespace_uri.startswith("<"): + namespace_uri = f"<{namespace_uri}>" + missing_custom[prefix] = namespace_uri + + all_missing = {**missing_common, **missing_custom} + + if not all_missing: + return turtle_str + + prefix_block = ( + "\n".join( + f"@prefix {prefix}: {uri} ." for prefix, uri in all_missing.items() + ) + + "\n\n" + ) + + return prefix_block + turtle_str + + @staticmethod + def _is_jsonld_str(s: str) -> bool: + """Check if a string appears to be JSON-LD format. + + Args: + s: The string to check. + + Returns: + bool: True if the string appears to be JSON-LD. + """ + s = s.strip() + if not (s.startswith("{") or s.startswith("[")): + return False + try: + # Try to parse as JSON + data = json.loads(s) + # Check if it's a dict/object with @context or @id, or an array containing such objects + if isinstance(data, dict): + return "@context" in data or "@id" in data + elif isinstance(data, list): + return any( + isinstance(item, dict) and ("@context" in item or "@id" in item) + for item in data + ) + return False + except (json.JSONDecodeError, ValueError): + return False + + @classmethod + def _from_str(cls, data_str: str) -> "RDFGraph": + """Create an RDFGraph instance from a string (Turtle or JSON-LD). + + Automatically detects the format and parses accordingly. + + Args: + data_str: The input string in Turtle or JSON-LD format. + + Returns: + RDFGraph: A new RDFGraph instance. + """ + if cls._is_jsonld_str(data_str): + return cls._from_jsonld_str(data_str) + else: + return cls._from_turtle_str(data_str) + + @classmethod + def _from_turtle_str(cls, turtle_str: str) -> "RDFGraph": + """Create an RDFGraph instance from a Turtle string. + + This method uses context variables to access known prefixes that may be + needed to complete missing prefix declarations in the Turtle string. + + Args: + turtle_str: The input Turtle string. + + Returns: + RDFGraph: A new RDFGraph instance. + """ + turtle_str = bytes(turtle_str, "utf-8").decode("unicode_escape") + patched_turtle = cls._ensure_prefixes(turtle_str) + g = cls() + try: + g.parse(data=patched_turtle, format="turtle") + return g + except Exception as parse_error: + # Typical LLM truncation: dangling ';' or ',' at EOF in property list. + if "EOF found when expected verb in property list" not in str(parse_error): + raise + repaired_turtle = cls._repair_truncated_turtle(patched_turtle) + if repaired_turtle == patched_turtle: + raise + logger.warning( + "Recovering truncated Turtle by closing dangling property list punctuation." + ) + repaired_graph = cls() + repaired_graph.parse(data=repaired_turtle, format="turtle") + return repaired_graph + + @staticmethod + def _repair_truncated_turtle(turtle_str: str) -> str: + """Repair common LLM Turtle truncation patterns. + + This only applies a minimal fix when content ends with dangling property-list + punctuation (';' or ',') and no terminating '.'. + """ + stripped = turtle_str.rstrip() + if not stripped: + return turtle_str + if stripped.endswith(";") or stripped.endswith(","): + return f"{stripped[:-1].rstrip()} .\n" + return turtle_str + + @classmethod + def set_known_prefixes(cls, prefixes: dict[str, str] | None) -> None: + """Set known prefixes in the context for use during parsing. + + This should be called before parsing TTL strings that may use prefixes + from an ontology or other source. The prefixes will be automatically + added if they're used but not declared in the TTL string. + + Args: + prefixes: Dictionary mapping prefix names to namespace URIs. + Example: {"fcaont": "https://growgraph.dev/fcaont#"} + """ + _known_prefixes_context.set(prefixes) + + @classmethod + def get_known_prefixes(cls) -> dict[str, str] | None: + """Get currently known prefixes from context. + + Returns: + Dictionary mapping prefix names to namespace URIs, or None. + """ + return _known_prefixes_context.get() + + @classmethod + def _from_jsonld_str(cls, jsonld_str: str) -> "RDFGraph": + """Create an RDFGraph instance from a JSON-LD string. + + Args: + jsonld_str: The input JSON-LD string. + + Returns: + RDFGraph: A new RDFGraph instance with namespace prefixes extracted from @context. + """ + # Use pyld to convert JSON-LD to n-quads, then parse to avoid rdflib's deprecated ConjunctiveGraph + # This adapts to the new convention by using pyld directly instead of rdflib's JSON-LD parser + jsonld_data = json.loads(jsonld_str) + normalized = jsonld.normalize( + jsonld_data, + {"algorithm": "URDNA2015", "format": "application/n-quads"}, + ) + + # jsonld.normalize returns a string when format is "application/n-quads" + normalized_str = normalized if isinstance(normalized, str) else str(normalized) + + # Parse the normalized n-quads into RDFGraph + g = cls() + g.parse(data=normalized_str, format="nquads") + + # Extract prefixes from @context in JSON-LD and bind them + try: + context = None + + # Handle single object or array + if isinstance(jsonld_data, dict): + context = jsonld_data.get("@context") + elif isinstance(jsonld_data, list) and jsonld_data: + # For arrays, check first item for @context + first_item = jsonld_data[0] + if isinstance(first_item, dict): + context = first_item.get("@context") + + # Bind prefixes from @context + if context and isinstance(context, dict): + for prefix, uri in context.items(): + if isinstance(uri, str) and not prefix.startswith("@"): + # Skip JSON-LD keywords (starting with @) + try: + g.bind(prefix, uri) + except Exception as e: + logger.debug(f"Failed to bind prefix '{prefix}': {e}") + + except (json.JSONDecodeError, ValueError, AttributeError) as e: + logger.debug(f"Could not extract prefixes from JSON-LD @context: {e}") + + return g + + @staticmethod + def _to_turtle_str(g: Any) -> str: + """Convert an RDFGraph to a Turtle string. + + For graphs backed by the *oxigraph* store the serialisation is + delegated to ``pyoxigraph`` so that RDF 1.2 triple-term syntax + (``<<( s p o )>>``) is emitted correctly. + + Args: + g: The RDFGraph instance. + + Returns: + str: The Turtle (or Turtle-star) string representation. + """ + if hasattr(g, "store") and type(g.store).__name__ == "OxigraphStore": + return g.serialize_turtle_star() + return g.serialize(format="turtle") + + def serialize_turtle_star(self) -> str: + """Serialize an oxigraph-backed graph to Turtle-star via *pyoxigraph*. + + This method extracts all quads belonging to this graph's context + from the underlying ``pyoxigraph.Store`` and serialises them into + the default graph using ``pyoxigraph.serialize`` with the Turtle + format, which natively supports RDF 1.2 ``<<( … )>>`` syntax. + + Returns: + Turtle-star string. + + Raises: + RuntimeError: If the graph is not backed by an oxigraph store. + """ + try: + import pyoxigraph as ox + from oxrdflib._converter import to_ox + except ImportError as exc: + raise RuntimeError( + "pyoxigraph / oxrdflib must be installed for Turtle-star serialisation" + ) from exc + + inner_store: ox.Store = self.store._inner # type: ignore[attr-defined] + graph_ctx_raw = to_ox(self.identifier) + assert isinstance( + graph_ctx_raw, + (ox.NamedNode, ox.BlankNode, ox.DefaultGraph), + ) + graph_ctx: ox.NamedNode | ox.BlankNode | ox.DefaultGraph = graph_ctx_raw + + # Copy quads into a temporary store under the default graph so + # that ``ox.serialize`` can emit plain Turtle (Turtle-star). + tmp = ox.Store() + used_iri_terms: set[str] = set() + + def _collect_used_iris(term: Any) -> None: + if isinstance(term, ox.NamedNode): + used_iri_terms.add(term.value) + return + if isinstance(term, ox.Triple): + _collect_used_iris(term.subject) + _collect_used_iris(term.predicate) + _collect_used_iris(term.object) + + for quad in inner_store.quads_for_pattern( + None, + None, + None, + graph_ctx, + ): + _collect_used_iris(quad.subject) + _collect_used_iris(quad.predicate) + _collect_used_iris(quad.object) + tmp.add( + ox.Quad(quad.subject, quad.predicate, quad.object, ox.DefaultGraph()) + ) + + namespace_to_prefix: dict[str, str] = {} + for prefix, namespace in self.namespaces(): + if not prefix: + continue + prefix_str = str(prefix) + namespace_str = str(namespace) + current = namespace_to_prefix.get(namespace_str) + if current is None or (len(prefix_str), prefix_str) < ( + len(current), + current, + ): + namespace_to_prefix[namespace_str] = prefix_str + + prefixes = { + prefix: namespace + for namespace, prefix in namespace_to_prefix.items() + if any(iri.startswith(namespace) for iri in used_iri_terms) + } + raw: bytes = tmp.dump( + format=ox.RdfFormat.TURTLE, + from_graph=ox.DefaultGraph(), + prefixes=prefixes or None, + ) # type: ignore[assignment] + return raw.decode() + + def __new__(cls, *args, **kwargs): + """Create a new RDFGraph instance.""" + instance = super().__new__(cls) + return instance + + def serialize( + self, + destination: Any = None, + format: str = "turtle", + base: str | None = None, + encoding: str | None = None, + **args: Any, + ) -> Any: + """Serialize the graph, delegating to pyoxigraph for oxigraph stores. + + When the graph is backed by an *oxigraph* store and the requested + format is ``"turtle"`` (or ``"ttl"``), serialisation is handled by + ``pyoxigraph`` which natively supports RDF 1.2 triple terms. + For all other stores or formats the default rdflib serialiser is + used. + """ + is_ox = type(self.store).__name__ == "OxigraphStore" + if is_ox and format in ("turtle", "ttl"): + ttl = self.serialize_turtle_star() + if destination is not None: + enc = encoding or "utf-8" + with open(destination, "w", encoding=enc) as fh: + fh.write(ttl) + return None + return ttl + return super().serialize( + destination=destination, + format=format, + base=base, + encoding=encoding, + **args, + ) + + def update( + self, + update_object: Any, + processor: Any = "sparql", + initNs: Mapping[str, Any] | None = None, + initBindings: Mapping[str, Any] | None = None, + use_store_provided: bool = True, + **kwargs: Any, + ) -> None: + """Execute SPARQL update using a base Graph view. + + rdflib's SPARQL update engine has internal checks that branch on exact + ``Graph`` type, which can break for subclasses on ``INSERT/DELETE ... WHERE``. + Running updates through a base ``Graph`` view avoids that edge case while + still operating on the same underlying store/identifier. + """ + graph_view = Graph(store=self.store, identifier=self.identifier) + graph_view.namespace_manager = self.namespace_manager + graph_view.update( + update_object=update_object, + processor=processor, + initNs=initNs, + initBindings=initBindings, + use_store_provided=use_store_provided, + **kwargs, + ) + return None + + def sanitize_prefixes_namespaces(self): + """ + Rematches prefixes in an RDFLib graph to correct namespaces when a namespace + with the same URI exists. Handles cases where prefixes might not be bound + as namespaces. + + Args: + self (RDFGraph): The RDFLib graph to process + + Returns: + RDFGraph: The graph with corrected prefix-namespace mappings + """ + # Get the namespace manager + ns_manager = self.namespace_manager + + # Collect all current prefix-URI mappings + current_prefixes = dict(ns_manager.namespaces()) + + # Group URIs by their string representation to find duplicates + uri_to_prefixes = defaultdict(list) + for prefix, uri in current_prefixes.items(): + uri_to_prefixes[str(uri)].append((prefix, uri)) + + # Find the "canonical" namespace objects for each URI + # (the actual Namespace objects that might be registered) + canonical_namespaces = {} + + # Check if any of the URIs correspond to well-known namespaces + # by trying to create Namespace objects and seeing if they're already registered + for uri_str, prefix_uri_pairs in uri_to_prefixes.items(): + # Try to find if there's already a proper Namespace object for this URI + namespace_candidates = [] + + for prefix, uri_obj in prefix_uri_pairs: + # Check if this is already a proper Namespace object + if isinstance(uri_obj, Namespace): + namespace_candidates.append(uri_obj) + else: + # Try to create a Namespace and see if it matches existing ones + try: + ns = Namespace(uri_str) + namespace_candidates.append(ns) + except: + continue + + # Use the first valid namespace candidate as canonical + if namespace_candidates: + canonical_namespaces[uri_str] = namespace_candidates[0] + + # Now rebuild the namespace manager with corrected mappings + # Clear existing bindings first + new_ns_manager = NamespaceManager(self) + + # Track which prefixes we want to keep/reassign + final_mappings = {} + + for uri_str, prefix_uri_pairs in uri_to_prefixes.items(): + if len(prefix_uri_pairs) == 1: + # No duplicates, keep as-is but ensure we use canonical namespace + prefix, _ = prefix_uri_pairs[0] + canonical_ns = canonical_namespaces.get(uri_str) + if canonical_ns: + final_mappings[prefix] = canonical_ns + else: + # Fallback to creating a new Namespace + final_mappings[prefix] = Namespace(uri_str) + else: + # Multiple prefixes for same URI - need to decide which to keep + # Priority: 1) Proper Namespace objects, + # 2) Shorter prefixes, + # 3) Alphabetical + prefix_uri_pairs.sort( + key=lambda x: ( + not isinstance(x[1], Namespace), # Namespace objects first + len(x[0]), # Shorter prefixes next + x[0], # Alphabetical order + ) + ) + + # Keep the best prefix, map others to it if needed + best_prefix, _ = prefix_uri_pairs[0] + canonical_ns = canonical_namespaces.get(uri_str, Namespace(uri_str)) + final_mappings[best_prefix] = canonical_ns + + other_prefixes = [p for p, _ in prefix_uri_pairs[1:]] + if other_prefixes: + logger.debug( + f"Consolidating prefixes {other_prefixes} " + f"-> '{best_prefix}' for URI: {uri_str}" + ) + + # Apply the final mappings + for prefix, namespace in final_mappings.items(): + new_ns_manager.bind(prefix, namespace, override=True) + + # Replace the graph's namespace manager + self.namespace_manager = new_ns_manager + + def unbind_chunk_namespaces(self, chunk_pattern="/chunk/") -> "RDFGraph": + """ + Unbinds namespace prefixes that point to URIs containing a chunk pattern. + Returns a new graph with chunk namespaces dereferenced (expanded to full URIs). + + Args: + chunk_pattern (str): The pattern to look for in URIs (default: "/chunk/") + + Returns: + RDFGraph: New graph with chunk-related namespaces unbound + """ + current_prefixes = dict(self.namespace_manager.namespaces()) + + # Find prefixes that point to URIs containing the chunk pattern + chunk_prefixes = [] + for prefix, uri in current_prefixes.items(): + uri_str = str(uri) + if chunk_pattern in uri_str: + chunk_prefixes.append((prefix, uri_str)) + + # Create new graph + new_graph = RDFGraph() + + # Copy all triples (URIs are already expanded internally) + for triple in self: + new_graph.add(triple) + + # Bind only non-chunk namespace prefixes to the new graph + for prefix, uri in current_prefixes.items(): + uri_str = str(uri) + if chunk_pattern not in uri_str: + new_graph.bind(prefix, uri) + + # Log what was removed + if chunk_prefixes: + logger.debug(f"Unbound {len(chunk_prefixes)} chunk-related namespace(s):") + for prefix, uri in chunk_prefixes: + logger.debug(f" - '{prefix}': {uri}") + + return new_graph + + def remap_namespaces(self, old_namespace, new_namespace) -> None: + updates = {} + for s, p, o in self: + new_s, new_p, new_o = s, p, o + if isinstance(s, URIRef) and str(s).startswith(str(old_namespace)): + new_s = URIRef( + str(s).replace(str(old_namespace), str(new_namespace), 1) + ) + if isinstance(p, URIRef) and str(p).startswith(str(old_namespace)): + new_p = URIRef( + str(p).replace(str(old_namespace), str(new_namespace), 1) + ) + if isinstance(o, URIRef) and str(o).startswith(str(old_namespace)): + new_o = URIRef( + str(o).replace(str(old_namespace), str(new_namespace), 1) + ) + + if (new_s, new_p, new_o) != (s, p, o): + updates[(s, p, o)] = (new_s, new_p, new_o) + + for (s, p, o), (new_s, new_p, new_o) in updates.items(): + self.remove((s, p, o)) + self.add((new_s, new_p, new_o)) + + def add_triple(self, subject: str, predicate: str, object_: str) -> None: + """Add a triple to the graph. + + Args: + subject: Subject URI as string + predicate: Predicate URI as string + object_: Object URI as string or literal value + """ + # Convert strings to appropriate RDFLib objects + subj = URIRef(subject) + pred = URIRef(predicate) + + # Handle object - could be URI or literal + if object_.startswith("http://") or object_.startswith("https://"): + obj = URIRef(object_) + else: + # Treat as literal + obj = Literal(object_) + + self.add((subj, pred, obj)) + logger.debug(f"Added triple: {subj} {pred} {obj}") + + def remove_triple(self, subject: str, predicate: str, object_: str) -> None: + """Remove a triple from the graph. + + Args: + subject: Subject URI as string + predicate: Predicate URI as string + object_: Object URI as string or literal value + """ + # Convert strings to appropriate RDFLib objects + subj = URIRef(subject) + pred = URIRef(predicate) + + # Handle object - could be URI or literal + if object_.startswith("http://") or object_.startswith("https://"): + obj = URIRef(object_) + else: + # Treat as literal + obj = Literal(object_) + + self.remove((subj, pred, obj)) + logger.debug(f"Removed triple: {subj} {pred} {obj}") + + def hash(self: Graph) -> str: + # Serialize to JSON-LD + data = self.serialize(format="json-ld") + + # Parse the JSON string + doc = json.loads(data) + + # Canonicalize using URDNA2015 normalization + normalized = jsonld.normalize( + doc, + {"algorithm": "URDNA2015", "format": "application/n-quads"}, + ) + # jsonld.normalize returns a string when format is "application/n-quads" + normalized_str = normalized if isinstance(normalized, str) else str(normalized) + return render_text_hash(normalized_str, digits=None) diff --git a/ontology_platform/vendored/ontocast/ontocast/onto/sparql_models.py b/ontology_platform/vendored/ontocast/ontocast/onto/sparql_models.py new file mode 100644 index 0000000..a615cf8 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/onto/sparql_models.py @@ -0,0 +1,520 @@ +"""Pydantic models for SPARQL operations. + +This module provides Pydantic models for structured SPARQL queries +that can be used with PydanticOutputParser for LLM integration. +""" + +import logging +from typing import Annotated, Any +from typing import Literal as TypingLiteral + +from pydantic import BaseModel, BeforeValidator, Field +from rdflib import BNode, Literal, Node, URIRef + +from ontocast.onto.constants import COMMON_PREFIXES +from ontocast.onto.enum import SPARQLOperationType +from ontocast.onto.rdfgraph import RDFGraph + +logger = logging.getLogger(__name__) + +# Convert COMMON_PREFIXES from Turtle format (with angle brackets) to SPARQL format (without) +# Example: "" -> "http://example.org/" +STANDARD_PREFIXES = {prefix: uri.strip("<>") for prefix, uri in COMMON_PREFIXES.items()} + + +class SPARQLOperationModel(BaseModel): + """Pydantic model for a single SPARQL operation. + + Attributes: + operation_type: Type of SPARQL operation (INSERT, UPDATE, DELETE) + query: The SPARQL query string + description: Optional description of the operation + metadata: Optional metadata dictionary + """ + + operation_type: SPARQLOperationType = Field( + description="Type of SPARQL operation: INSERT, UPDATE, or DELETE" + ) + query: str = Field( + description="The complete SPARQL query string with proper syntax" + ) + description: str = Field( + default="", description="Optional description of the operation" + ) + metadata: dict[str, Any] = Field( + default_factory=dict, + description="Optional metadata dictionary for the operation", + ) + + +class StructuredSPARQLQueryModel(BaseModel): + """Pydantic model for structured SPARQL queries. + + Attributes: + operations: List of SPARQL operations (INSERT, UPDATE, DELETE) + namespaces: Dictionary mapping prefixes to URIs + """ + + operations: list[SPARQLOperationModel] = Field( + default_factory=list, description="List of SPARQL operations to execute" + ) + namespaces: dict[str, str] = Field( + default_factory=dict, + description="Dictionary mapping namespace prefixes to URIs", + ) + + def get_summary(self) -> str: + """Get a summary of the structured query.""" + add_count = len( + [ + op + for op in self.operations + if op.operation_type == SPARQLOperationType.INSERT + ] + ) + update_count = len( + [ + op + for op in self.operations + if op.operation_type == SPARQLOperationType.UPDATE + ] + ) + remove_count = len( + [ + op + for op in self.operations + if op.operation_type == SPARQLOperationType.DELETE + ] + ) + + return ( + f"Structured SPARQL Query: " + f"{add_count} ADD operations, " + f"{update_count} UPDATE operations, " + f"{remove_count} REMOVE operations" + ) + + def get_all_operations(self) -> list[SPARQLOperationModel]: + """Get all operations in execution order (INSERT, UPDATE, DELETE).""" + # Sort operations by type: INSERT first, then UPDATE, then DELETE + type_order = { + SPARQLOperationType.INSERT: 0, + SPARQLOperationType.UPDATE: 1, + SPARQLOperationType.DELETE: 2, + } + return sorted(self.operations, key=lambda op: type_order[op.operation_type]) + + def get_add_operations(self) -> list[SPARQLOperationModel]: + """Get all INSERT operations.""" + return [ + op + for op in self.operations + if op.operation_type == SPARQLOperationType.INSERT + ] + + def get_update_operations(self) -> list[SPARQLOperationModel]: + """Get all UPDATE operations.""" + return [ + op + for op in self.operations + if op.operation_type == SPARQLOperationType.UPDATE + ] + + def get_remove_operations(self) -> list[SPARQLOperationModel]: + """Get all DELETE operations.""" + return [ + op + for op in self.operations + if op.operation_type == SPARQLOperationType.DELETE + ] + + +class OntologyUpdateReport(BaseModel): + """Report from ontology update process using structured SPARQL. + + Attributes: + update_success: True if the ontology update was performed successfully + structured_query: The structured SPARQL query used for the update + add_count: Number of ADD operations + update_count: Number of UPDATE operations + remove_count: Number of REMOVE operations + critique: Optional critique of the update process + """ + + update_success: bool = Field( + description="True if the ontology update was performed successfully, False otherwise" + ) + structured_query: StructuredSPARQLQueryModel = Field( + description="The structured SPARQL query used for the update" + ) + add_count: int = Field( + description="Number of ADD operations in the structured query" + ) + update_count: int = Field( + description="Number of UPDATE operations in the structured query" + ) + remove_count: int = Field( + description="Number of REMOVE operations in the structured query" + ) + critique: str | None = Field( + None, description="Optional critique or explanation of the update process" + ) + + +class FactsUpdateReport(BaseModel): + """Report from facts update process using structured SPARQL. + + Attributes: + update_success: True if the facts update was performed successfully + structured_query: The structured SPARQL query used for the update + add_count: Number of ADD operations + update_count: Number of UPDATE operations + remove_count: Number of REMOVE operations + critique: Optional critique of the update process + """ + + update_success: bool = Field( + description="True if the facts update was performed successfully, False otherwise" + ) + structured_query: StructuredSPARQLQueryModel = Field( + description="The structured SPARQL query used for the update" + ) + add_count: int = Field( + description="Number of ADD operations in the structured query" + ) + update_count: int = Field( + description="Number of UPDATE operations in the structured query" + ) + remove_count: int = Field( + description="Number of REMOVE operations in the structured query" + ) + critique: str | None = Field( + None, description="Optional critique or explanation of the update process" + ) + + +class FreshOntologyReport(BaseModel): + """Report from fresh ontology generation process. + + Attributes: + generation_success: True if the ontology was generated successfully + ontology_graph: The generated ontology as an RDFGraph + ontology_score: Score 0-100 for ontology quality + critique: Optional critique of the ontology generation + """ + + generation_success: bool = Field( + description="True if the ontology was generated successfully, False otherwise" + ) + ontology_graph: RDFGraph = Field( + default_factory=RDFGraph, + description="The generated ontology as an RDFGraph in Turtle format", + ) + ontology_score: float | None = Field( + None, description="Score 0-100 for ontology quality and completeness" + ) + critique: str | None = Field( + None, description="Optional critique or explanation of the ontology generation" + ) + + +class FreshFactsReport(BaseModel): + """Report from fresh facts generation process. + + Attributes: + generation_success: True if the facts were generated successfully + facts_graph: The generated facts as an RDFGraph + facts_score: Score 0-100 for facts quality + critique: Optional critique of the facts generation + """ + + generation_success: bool = Field( + description="True if the facts were generated successfully, False otherwise" + ) + facts_graph: RDFGraph = Field( + default_factory=RDFGraph, + description="The generated facts as an RDFGraph in Turtle format", + ) + facts_score: float | None = Field( + None, description="Score 0-100 for facts quality and completeness" + ) + critique: str | None = Field( + None, description="Optional critique or explanation of the facts generation" + ) + + +class TripleOp(BaseModel): + """Operation to modify triples in the RDF graph. + + This operation can insert or delete triples. Prefixes are automatically extracted + from the RDFGraph's namespace bindings (from @prefix declarations in Turtle). + """ + + type: TypingLiteral["insert", "delete"] = Field( + description="Type of operation: 'insert' to add triples, 'delete' to remove triples" + ) + graph: Annotated[ + RDFGraph, + BeforeValidator( + lambda v: RDFGraph._from_turtle_str(v) if isinstance(v, str) else v + ), + ] = Field( + default_factory=RDFGraph, + description="RDF graph containing triples to insert or delete. " + "Must be provided as a Turtle format string or RDFGraph instance. " + 'Example Turtle: "@prefix ex: . ex:John a ex:Person ; rdfs:label "John Doe" ."', + ) + prefixes: dict[str, str] = Field( + default_factory=dict, + description="Optional: Additional or override prefixes. " + "Prefixes are automatically extracted from the RDFGraph's namespace bindings. " + "Standard prefixes from COMMON_PREFIXES in constants.py (rdf, rdfs, owl, xsd, dc, dcterms, skos, foaf, schema, prov, ex) are automatically available. " + "This field can be used to add or override prefixes if needed. " + "Mapping format: {'prefix_name': 'namespace_uri'}. Example: {'fca': 'http://example.org/ontologies/fca#'}", + ) + + +class GenericSparqlQuery(BaseModel): + """Operation for custom SPARQL queries that go beyond basic insert/delete operations. + + This operation allows for complex SPARQL queries that cannot be expressed + using the structured operations. Use this when you need custom SPARQL syntax, + complex WHERE clauses, or operations that don't fit the basic patterns. + """ + + type: TypingLiteral["sparql_query"] = Field( + default="sparql_query", + description="Type of operation - always 'sparql_query' for this operation", + ) + query: str = Field( + description="The complete SPARQL query string with proper syntax" + ) + + +class GraphUpdate(BaseModel): + """Structured representation of RDF graph updates for LLM output. + + This model represents ontology updates as a structured set of operations. + Each operation in the list is executed in order to modify the graph. + """ + + triple_operations: list[TripleOp] = Field( + default_factory=list, + description="List of graph update operations in execution order. " + "Each operation should be a TripleOp (for insert/delete) with RDFGraph containing triples in Turtle format." + "Example: [TripleOp(type='insert', graph='@prefix ex: . ex:John a ex:Person .', prefixes={'ex': 'http://example.org/'})]", + ) + + sparql_operations: list[GenericSparqlQuery] = Field( + default_factory=list, + description="List of graph update operations in execution order. " + "Each operation should be a GenericSparqlQuery for complex custom queries. ", + ) + + def generate_sparql_queries(self) -> list[str]: + """Generate a list of SPARQL queries to execute the graph update. + + Returns: + List of SPARQL query strings that can be executed to perform the update. + The queries are generated in the exact order of operations in the operations list. + """ + queries = [] + + # Process triple operations first + for op in self.triple_operations: + if len(op.graph) > 0: # Only generate query if there are triples + # Build prefix block for this operation + # Start with standard prefixes from COMMON_PREFIXES + prefixes = STANDARD_PREFIXES.copy() + + # Extract prefixes from RDFGraph's namespace bindings + for prefix, uri in op.graph.namespaces(): + if prefix: # Skip empty prefix + prefixes[prefix] = str(uri) + + # Add custom prefixes declared in this operation (may override standard ones) + prefixes.update(op.prefixes) + + # Generate PREFIX declarations block + if prefixes: + prefix_declarations = [] + for prefix, uri in prefixes.items(): + prefix_declarations.append(f"PREFIX {prefix}: <{uri}>") + prefix_block = "\n".join(prefix_declarations) + else: + prefix_block = "" + + # Generate query based on operation type + if op.type == "insert": + triple_query = self._generate_insert_query(op.graph, prefix_block) + else: # delete + triple_query = self._generate_delete_query(op.graph, prefix_block) + queries.append(triple_query) + + # Process SPARQL operations + for op in self.sparql_operations: + if op.query.strip(): # Only generate query if there's content + # For custom SPARQL queries, use them as-is + queries.append(op.query) + + return queries + + def count_total_triples(self) -> tuple[int, int]: + """Count total triples across all operations. + + Returns: + Tuple of (total_operations, total_triples) where: + - total_operations: Number of operations + - total_triples: Total number of triples across all TripleOp operations + """ + total_triples = 0 + for op in self.triple_operations: + if isinstance(op, TripleOp): + total_triples += len(op.graph) + return (len(self.triple_operations), total_triples) + + def extract_insert_graph(self) -> RDFGraph: + """Extract RDFGraph of all insert triples from triple_operations. + + Only TripleOps with type='insert' are included. sparql_operations + are not extractable as triples and are skipped. + + Returns: + RDFGraph containing the union of all insert triples. + """ + result = RDFGraph() + for op in self.triple_operations: + if isinstance(op, TripleOp) and op.type == "insert" and len(op.graph) > 0: + for triple in op.graph: + result.add(triple) + for prefix, uri in op.graph.namespaces(): + if prefix: + result.bind(prefix, uri) + for prefix, uri in op.prefixes.items(): + result.bind(prefix, uri) + return result + + def generate_diff_summary(self) -> str: + """Generate a human-readable diff summary of all operations for LLM consumption. + + Returns: + String representation of all operations showing what will be added, removed, and modified. + Returns empty string if no operations to perform. + """ + if not self.triple_operations: + return "" + + diff_parts = [] + operation_count = 0 + + for i, op in enumerate(self.triple_operations, 1): + if isinstance(op, TripleOp): + if len(op.graph) > 0: + op_type = op.type.upper() + diff_parts.append(f"{i}. {op_type} {len(op.graph)} triple(s):") + + # Show prefixes from graph and explicit prefixes + graph_prefixes = { + prefix: str(uri) + for prefix, uri in op.graph.namespaces() + if prefix + } + all_prefixes = {**graph_prefixes, **op.prefixes} + if all_prefixes: + prefix_list = ", ".join( + [f"{k}: {v}" for k, v in all_prefixes.items()] + ) + diff_parts.append(f" Prefixes: {prefix_list}") + + for subject, predicate, obj in op.graph: + symbol = "+" if op.type == "insert" else "-" + diff_parts.append( + f" {symbol} {self._serialize_rdf_term(subject)} {self._serialize_rdf_term(predicate)} {self._serialize_rdf_term(obj)}" + ) + operation_count += 1 + + elif isinstance(op, GenericSparqlQuery): + if op.query.strip(): + # Truncate long queries for readability + query_preview = op.query.strip() + if len(query_preview) > 100: + query_preview = query_preview[:97] + "..." + diff_parts.append(f"{i}. CUSTOM SPARQL QUERY:") + diff_parts.append(f" {query_preview}") + operation_count += 1 + + if operation_count == 0: + return "" + + summary = f"Ontology Update Summary ({operation_count} operation(s)):\n\n" + summary += "\n".join(diff_parts) + + return summary + + def _generate_insert_query(self, graph: RDFGraph, prefix_block: str) -> str: + """Generate a SPARQL INSERT query for the given RDFGraph.""" + if len(graph) == 0: + return "" + + # Format triples for SPARQL using proper RDF term serialization + triple_patterns = [] + for subject, predicate, obj in graph: + triple_patterns.append( + f" {self._serialize_rdf_term(subject)} {self._serialize_rdf_term(predicate)} {self._serialize_rdf_term(obj)} ." + ) + + triples_block = "\n".join(triple_patterns) + + query_parts = [] + if prefix_block: + query_parts.append(prefix_block) + query_parts.append("INSERT DATA {") + query_parts.append(triples_block) + query_parts.append("}") + + return "\n".join(query_parts) + + def _generate_delete_query(self, graph: RDFGraph, prefix_block: str) -> str: + """Generate a SPARQL DELETE query for the given RDFGraph.""" + if len(graph) == 0: + return "" + + # Format triples for SPARQL using proper RDF term serialization + triple_patterns = [] + for subject, predicate, obj in graph: + triple_patterns.append( + f" {self._serialize_rdf_term(subject)} {self._serialize_rdf_term(predicate)} {self._serialize_rdf_term(obj)} ." + ) + + triples_block = "\n".join(triple_patterns) + + query_parts = [] + if prefix_block: + query_parts.append(prefix_block) + query_parts.append("DELETE DATA {") + query_parts.append(triples_block) + query_parts.append("}") + + return "\n".join(query_parts) + + def _serialize_rdf_term(self, term: Node) -> str: + """Serialize an RDF term to its SPARQL string representation.""" + if isinstance(term, URIRef): + # Check if it's already a prefixed name (contains ':') + if ":" in str(term) and not str(term).startswith("http"): + return str(term) + else: + return f"<{term}>" + elif isinstance(term, BNode): + return f"_:{term}" + elif isinstance(term, Literal): + # Handle language-tagged literals first + if term.language: + return f'"{term}"@{term.language}' + elif term.datatype: + return f'"{term}"^^<{term.datatype}>' + else: + return f'"{term}"' + else: + # Fallback to string representation + return str(term) diff --git a/ontology_platform/vendored/ontocast/ontocast/onto/state.py b/ontology_platform/vendored/ontocast/ontocast/onto/state.py new file mode 100644 index 0000000..e93408d --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/onto/state.py @@ -0,0 +1,713 @@ +import os +from collections import defaultdict +from typing import Any + +from pydantic import ConfigDict, Field +from rdflib import URIRef + +from ontocast.onto.constants import CHUNK_NULL_IRI, DEFAULT_DOMAIN, ONTOLOGY_NULL_IRI +from ontocast.onto.content_unit import ContentUnit +from ontocast.onto.context import AgentContext, AgentType, ContextManager +from ontocast.onto.enum import FailureStage, RenderMode, Status, WorkflowNode +from ontocast.onto.model import BasePydanticModel, Suggestions +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.sparql_models import GraphUpdate, TripleOp +from ontocast.util import iri2namespace, render_text_hash + + +class BudgetTracker(BasePydanticModel): + """Lightweight tracker for LLM usage statistics and generated triples.""" + + chars_sent: int = Field(default=0, description="Total characters sent to LLM") + chars_received: int = Field( + default=0, description="Total characters received from LLM" + ) + calls_count: int = Field(default=0, description="Total number of LLM API calls") + + # Triple generation tracking + ontology_triples_generated: int = Field( + default=0, description="Total number of triples generated for ontology updates" + ) + facts_triples_generated: int = Field( + default=0, description="Total number of triples generated for facts" + ) + ontology_operations_count: int = Field( + default=0, description="Total number of ontology update operations" + ) + facts_operations_count: int = Field( + default=0, description="Total number of facts update operations" + ) + + def add_usage(self, chars_sent: int, chars_received: int) -> None: + """Add usage statistics.""" + self.chars_sent += chars_sent + self.chars_received += chars_received + self.calls_count += 1 + + def add_ontology_update(self, num_operations: int, num_triples: int) -> None: + """Add ontology update statistics. + + Args: + num_operations: Number of update operations generated + num_triples: Number of triples in these operations + """ + self.ontology_operations_count += num_operations + self.ontology_triples_generated += num_triples + + def add_facts_update(self, num_operations: int, num_triples: int) -> None: + """Add facts update statistics. + + Args: + num_operations: Number of update operations generated + num_triples: Number of triples in these operations + """ + self.facts_operations_count += num_operations + self.facts_triples_generated += num_triples + + def get_summary(self) -> str: + """Get a summary of LLM usage and generated triples.""" + parts = [ + f"LLM: {self.calls_count} calls, " + f"{self.chars_sent:,} sent, " + f"{self.chars_received:,} received", + ] + + if self.ontology_triples_generated > 0 or self.facts_triples_generated > 0: + parts.append( + f"Triples: {self.ontology_triples_generated} ontology, " + f"{self.facts_triples_generated} facts" + ) + + return " | ".join(parts) + + +class AgentState(BasePydanticModel): + """State for the ontology-based knowledge graph agent. + + This class maintains the state of the agent during document processing, + including input text, content units, ontologies, and workflow status. + + Attributes: + input_text: Input text to process. + current_domain: IRI used for forming document namespace. + doc_hid: An almost unique hash/id for the parent document. + files: Files to process. + current_ontology: Current ontology object. + ontology_addendum: Additional ontology content. + failure_stage: Stage where failure occurred. + failure_reason: Reason for failure. + success_score: Score indicating success level. + status: Current workflow status. + node_visits: Number of visits per node. + max_visits: Maximum number of visits allowed per node. + max_chunks: Maximum number of source content units to split and process. + """ + + input_text: str = Field(description="Input text", default="") + current_domain: str = Field( + description="IRI used for forming document namespace", default=DEFAULT_DOMAIN + ) + doc_hid: str = Field( + description="An almost unique hash / id for the parent document of the current unit", + default="default_doc", + ) + files: dict[str, bytes] = Field( + default_factory=lambda: dict(), description="Files to process" + ) + content_units: list[ContentUnit] = Field( + default_factory=list, + description="Pending content units to process.", + ) + current_content_unit: ContentUnit = Field( + default_factory=lambda: ContentUnit( + text="", + index=0, + doc_iri=URIRef(CHUNK_NULL_IRI), + ), + alias="current_chunk", + description="Current content unit under processing.", + ) + current_ontology: Ontology = Field( + default_factory=lambda: Ontology( + ontology_id=None, + title=None, + description=None, + graph=RDFGraph(), + iri=ONTOLOGY_NULL_IRI, + ), + description="Ontology object that contain the semantic graph " + "as well as the description, name, short name, version, " + "and IRI of the ontology", + ) + aggregated_facts: RDFGraph = Field( + description="RDF triples representing aggregated facts " + "from the current document", + default_factory=RDFGraph, + ) + ontology_user_instruction: str = Field( + description="Specific user instructions for ontology extraction, e.g. `Focus on extracting places`", + default="", + ) + + facts_user_instruction: str = Field( + description="Specific user instructions for facts extraction, e.g. `Focus on extracting places`", + default="", + ) + + dataset: str | None = Field( + description="Fuseki dataset name for this request (optional)", + default=None, + ) + + graph_uri_override: str | None = Field(default=None) + + source_url: str | None = Field( + description="Source URL from JSON input file (for provenance tracking)", + default=None, + ) + + ontology_updates: list[GraphUpdate] = Field( + default_factory=list, + description="A list of graph update that improve the current ontology", + ) + + ontology_updates_applied: list[GraphUpdate] = Field( + default_factory=list, + description="A list of graph update that improve the current ontology", + ) + + facts_updates: list[GraphUpdate] = Field( + default_factory=list, + description="A list of graph update that improve the current graph of facts (pending)", + ) + + facts_updates_applied: list[GraphUpdate] = Field( + default_factory=list, + description="A list of graph update that improve the current graph of facts (applied)", + ) + + parallel_facts_units: list[ContentUnit] = Field( + default_factory=list, + description="Successful per-unit facts outputs collected during parallel map phase", + ) + + ontology_units: list[ContentUnit] = Field( + default_factory=list, + description="Successful per-unit ontology outputs collected during parallel map phase", + ) + ontology_provenance_artifact: RDFGraph = Field( + default_factory=RDFGraph, + description="Provenance/reification triples stripped from normalized ontology.", + ) + + ontology_addendum: Ontology = Field( + default_factory=lambda: Ontology( + ontology_id=None, + title=None, + description=None, + graph=RDFGraph(), + iri=ONTOLOGY_NULL_IRI, + ), + description="Ontology object that contain the semantic graph " + "as well as the description, name, short name, version, " + "and IRI of the ontology", + ) + failure_stage: FailureStage | None = None + failure_reason: str | None = None + + improvements_suggestions: list[str] = Field( + description="Itemized concrete and actionable instructions for improvements of extraction of facts/ontology", + default_factory=list, + ) + + success_score: float = 0.0 + status: Status = Status.SUCCESS + statuses: dict[WorkflowNode, Status] = Field( + default_factory=dict, description="Status of each node" + ) + node_visits: defaultdict[WorkflowNode, int] = Field( + default_factory=lambda: defaultdict(int), + description="Number of visits per node", + ) + max_visits: int = Field( + default=3, description="Maximum number of visits allowed per node" + ) + max_chunks: int | None = None + model_config = ConfigDict(arbitrary_types_allowed=True, populate_by_name=True) + render_mode: RenderMode = Field( + default=RenderMode.ONTOLOGY_AND_FACTS, + description=("Rendering mode: ontology, facts, or ontology_and_facts."), + ) + ontology_max_triples: int | None = Field( + default=50000, + description="Maximum number of triples allowed in ontology graph. " + "Updates that would exceed this limit are skipped with a warning. " + "Set to None for unlimited.", + ) + context_manager: ContextManager = Field( + default_factory=ContextManager, + description="Context manager for passing information between agents", + ) + suggestions: Suggestions = Field( + default_factory=Suggestions, + description="Context manager for passing information between agents", + ) + + # Budget Tracking + budget_tracker: BudgetTracker = Field( + default_factory=BudgetTracker, + description="Budget statistics tracker (LLM usage and generated triples)", + ) + + def model_post_init(self, __context): + """Post-initialization hook for the model.""" + pass + + def __init__(self, **kwargs): + """Initialize the agent state with given keyword arguments.""" + super().__init__(**kwargs) + self.current_domain = os.getenv("CURRENT_DOMAIN", DEFAULT_DOMAIN) + + def get_node_status(self, node: WorkflowNode) -> Status: + """Get the status of a workflow node, returning NOT_VISITED if not set.""" + return self.statuses.get(node, Status.NOT_VISITED) + + @property + def render_ontology(self) -> bool: + """Whether ontology rendering should run.""" + return self.render_mode in ( + RenderMode.ONTOLOGY, + RenderMode.ONTOLOGY_AND_FACTS, + ) + + @property + def render_facts(self) -> bool: + """Whether facts rendering should run.""" + return self.render_mode in ( + RenderMode.FACTS, + RenderMode.ONTOLOGY_AND_FACTS, + ) + + def set_node_status(self, node: WorkflowNode, status: Status) -> None: + """Set the status of a workflow node.""" + self.statuses[node] = status + + def get_content_unit_progress_info(self) -> tuple[int, int]: + """Get current content unit number and total content units.""" + from ontocast.onto.constants import CHUNK_NULL_IRI + + has_current_content_unit = CHUNK_NULL_IRI not in self.current_content_unit.iri + current_content_unit_number = 1 if has_current_content_unit else 0 + total_content_units = len(self.content_units) + return current_content_unit_number, total_content_units + + def get_content_unit_progress_string(self) -> str: + """Get a formatted string showing content unit progress.""" + current, total = self.get_content_unit_progress_info() + if total == 0: + return "no content units" + return f"content unit {current}/{total}" + + def get_chunk_progress_info(self) -> tuple[int, int]: + """Backward-compatible wrapper for content unit progress. + + Returns: + tuple[int, int]: (current_chunk_number, total_chunks) + """ + return self.get_content_unit_progress_info() + + def get_chunk_progress_string(self) -> str: + """Backward-compatible wrapper for content unit progress. + + Returns: + str: Formatted string like "chunk 3/10" + """ + return self.get_content_unit_progress_string() + + @classmethod + def render_updated_graph( + cls, graph: RDFGraph, updates: list[GraphUpdate], max_triples: int | None = None + ) -> tuple[RDFGraph, bool]: + """Create a copy of the given graph with all GraphUpdate objects applied. + + This method: + 1. Creates a copy of the input graph + 2. Generates SPARQL queries from all GraphUpdate objects + 3. Executes the queries on the copied graph + 4. Checks if the updated graph exceeds max_triples limit + 5. Returns the updated graph copy, or original if limit exceeded + + Args: + graph: The RDFGraph to update + updates: List of GraphUpdate objects to apply + max_triples: Maximum number of triples allowed. If None, no limit enforced. + + Returns: + Tuple of (RDFGraph, bool): The updated graph (or original if limit exceeded), + and a boolean indicating if the update was applied (True) or skipped (False) + """ + if not updates: + return graph, True + + # Create a copy of the input graph + # Use RDFGraph's copy method to preserve type + updated_graph = RDFGraph() + for triple in graph: + updated_graph.add(triple) + # Copy namespace bindings + for prefix, namespace in graph.namespaces(): + updated_graph.bind(prefix, namespace) + + all_prefixes = {} + for graph_update in updates: + for op in graph_update.triple_operations: + # Extract prefixes from TripleOp operations + if isinstance(op, TripleOp) and op.prefixes: + all_prefixes.update(op.prefixes) + + # Bind prefixes to the copied graph + for prefix, uri in all_prefixes.items(): + updated_graph.bind(prefix, uri) + + # Apply each GraphUpdate to the copied graph + for graph_update in updates: + # Generate SPARQL queries from the GraphUpdate + queries = graph_update.generate_sparql_queries() + + # Execute each query on the copied graph + for query in queries: + cls._apply_update_query(updated_graph, query) + + # Check if updated graph exceeds max_triples limit + if max_triples is not None and len(updated_graph) > max_triples: + import logging + + logger = logging.getLogger(__name__) + logger.warning( + f"Ontology update skipped: would exceed limit " + f"({len(updated_graph)} > {max_triples} triples). " + f"Original size: {len(graph)} triples." + ) + return graph, False # Return original, unchanged + + return updated_graph, True + + @classmethod + def _apply_update_query(cls, graph: RDFGraph, query: str) -> None: + """Apply one SPARQL update query, recovering common LLM compound outputs.""" + try: + graph.update(query) + return + except Exception as exc: + split_queries = cls._split_compound_sparql_updates(query, exc) + if not split_queries: + raise + for split_query in split_queries: + graph.update(split_query) + + @staticmethod + def _split_compound_sparql_updates( + query: str, parse_error: Exception + ) -> list[str] | None: + """Split concatenated top-level UPDATE statements if parser rejects input. + + LLM outputs sometimes concatenate multiple update statements (e.g. two + top-level INSERT blocks) into a single custom SPARQL string without a + separator accepted by the parser. We only recover in that specific case. + """ + message = str(parse_error) + if "Expected end of text, found" not in message: + return None + + prefixes: list[str] = [] + body_lines: list[str] = [] + for raw_line in query.splitlines(): + stripped = raw_line.strip() + if stripped.upper().startswith("PREFIX "): + prefixes.append(stripped) + elif stripped: + body_lines.append(stripped) + + if not body_lines: + return None + + top_level_ops = ("INSERT", "DELETE", "WITH") + segments: list[list[str]] = [] + current: list[str] = [] + for line in body_lines: + if current and line.upper().startswith(top_level_ops): + segments.append(current) + current = [line] + else: + current.append(line) + if current: + segments.append(current) + + if len(segments) <= 1: + return None + + prefix_block = "\n".join(prefixes) + rebuilt = [] + for segment in segments: + segment_block = "\n".join(segment) + rebuilt.append( + f"{prefix_block}\n{segment_block}" if prefix_block else segment_block + ) + return rebuilt + + def render_uptodate_ontology(self) -> Ontology: + """Create a copy of the current ontology with all GraphUpdate objects applied. + + This method: + 1. Creates a copy of the current ontology + 2. Generates SPARQL queries from all GraphUpdate objects + 3. Executes the queries on the copied ontology graph + 4. Checks if the updated graph exceeds max_triples limit + 5. Sets the current hash as parent_hash in the updated ontology + 6. Computes a new hash for the updated ontology + 7. Syncs properties to ensure object fields are updated + 8. Returns the updated ontology copy, or original if limit exceeded + + Returns: + Ontology: A copy of the current ontology with all updates applied and + a new hash generated, with the previous hash set as parent. + Returns original ontology if update would exceed max_triples limit. + """ + if not self.ontology_updates: + return self.current_ontology + + # Use the generalized function to update the graph + updated_graph, was_applied = self.render_updated_graph( + self.current_ontology.graph, + self.ontology_updates, + max_triples=self.ontology_max_triples, + ) + + # If graph wasn't updated (limit exceeded), return original ontology + if not was_applied: + return self.current_ontology + + return self.current_ontology.derive_updated_version(updated_graph) + + def update_ontology(self) -> None: + """Update the current ontology with all GraphUpdate objects and clear the updates list. + + This method: + 1. Uses render_uptodate_ontology() to get an updated copy + 2. Replaces the current ontology with the updated copy + 3. Clears the ontology_updates list + + Note: Version update is deferred to aggregate_serialize() to update only once at the end. + """ + if not self.ontology_updates: + return + + # Get the updated ontology copy + updated_ontology = self.render_uptodate_ontology() + + # Replace the current ontology with the updated copy + self.current_ontology = updated_ontology + + # Clear the updates list + self.ontology_updates_applied += self.ontology_updates + self.ontology_updates = [] + + def render_uptodate_facts(self) -> RDFGraph: + """Create a copy of the current content unit graph with facts updates applied. + + This method: + 1. Creates a copy of the current content unit's graph + 2. Generates SPARQL queries from all facts GraphUpdate objects + 3. Executes the queries on the copied graph + 4. Returns the updated graph copy + + Returns: + RDFGraph: A copy of the current chunk's graph with all facts updates applied + """ + if not self.facts_updates: + return self.current_content_unit.graph + + # Use the generalized function to update the graph + updated_graph, _ = self.render_updated_graph( + self.current_content_unit.graph, self.facts_updates, max_triples=None + ) + return updated_graph + + def update_facts(self) -> None: + """Update current content unit graph with facts updates and clear the updates list. + + This method: + 1. Uses render_uptodate_facts() to get an updated copy + 2. Replaces the current content unit graph with the updated copy + 3. Clears the facts_updates list + """ + if not self.facts_updates: + return + + # Get the updated graph copy + updated_graph = self.render_uptodate_facts() + + # Replace the current chunk's graph with the updated copy + self.current_content_unit.graph = updated_graph + + # Clear the updates list + self.facts_updates_applied += self.facts_updates + self.facts_updates = [] + + def generate_ontology_updates_markdown(self) -> str: + """Generate a markdown string representing the chain of ontology updates. + + Returns: + Markdown-formatted string showing all pending ontology updates. + Returns empty string if no updates are pending. + """ + if not self.ontology_updates: + return "" + + markdown_parts = [] + for i, graph_update in enumerate(self.ontology_updates, 1): + diff_summary = graph_update.generate_diff_summary() + if diff_summary: + markdown_parts.append(f"## Update {i}") + markdown_parts.append(diff_summary) + + markdown_parts.append("") + + # Add separator between updates (except for the last one) + if i < len(self.ontology_updates): + markdown_parts.append("---") + markdown_parts.append("") + + return "\n".join(markdown_parts) + + def set_text(self, text): + """Set the input text and generate document hash. + + Args: + text: The input text to set. + """ + self.input_text = text + self.doc_hid = render_text_hash(self.input_text) + + def set_failure(self, stage: FailureStage, reason: str, success_score: float = 0.0): + """Set failure state with stage and reason. + + Args: + stage: The stage where the failure occurred. + reason: The reason for the failure. + success_score: The success score at failure (default: 0.0). + """ + self.failure_stage = stage + self.failure_reason = reason + self.success_score = success_score + self.status = Status.FAILED + + def clear_failure(self): + """Clear failure state and set status to success.""" + self.failure_stage = None + self.failure_reason = None + self.success_score = 0.0 + self.status = Status.SUCCESS + + @property + def doc_iri(self) -> URIRef: + """Get the document IRI. + + Returns: + str: The document IRI. + """ + return URIRef(f"{self.current_domain}/doc/{self.doc_hid}") + + @property + def doc_namespace(self): + """Get the document namespace. + + Returns: + str: The document namespace. + """ + return iri2namespace(self.doc_iri, ontology=False) + + @property + def graph_uri(self): + if self.graph_uri_override is not None: + return self.graph_uri_override + return self.doc_namespace + + @property + def ontology_id(self): + """Get the document namespace. + + Returns: + str: The document namespace. + """ + return self.current_ontology.ontology_id + + def get_context_for_agent(self, agent_type: AgentType) -> AgentContext: + """Get or create context for a specific agent. + + Args: + agent_type: Type of agent (renderer, critic, etc.). + + Returns: + AgentContext: The context for the agent. + """ + existing_context = self.context_manager.get_latest_context_by_agent(agent_type) + + if existing_context: + return existing_context + + # Create new context if none exists + return self.context_manager.create_context(agent_type=agent_type) + + def update_context_for_agent( + self, + agent_type: AgentType, + ontology_version: Any | None = None, + facts_version: Any | None = None, + ontology_operations: list[Any] | None = None, + facts_operations: list[Any] | None = None, + ontology_critique: dict[str, Any] | None = None, + facts_critique: dict[str, Any] | None = None, + metadata: dict[str, Any] | None = None, + ) -> AgentContext: + """Update context for a specific agent. + + Args: + agent_type: Name of the agent updating context. + ontology_version: New ontology version if available. + facts_version: New facts version if available. + ontology_operations: New ontology operations if available. + facts_operations: New facts operations if available. + ontology_critique: New ontology critique if available. + facts_critique: New facts critique if available. + metadata: Additional metadata for the context. + + Returns: + AgentContext: The updated context. + """ + return self.context_manager.update_context( + agent_type=agent_type, + ontology_version=ontology_version, + facts_version=facts_version, + ontology_operations=ontology_operations, + facts_operations=facts_operations, + ontology_critique=ontology_critique, + facts_critique=facts_critique, + metadata=metadata, + ) + + def get_context_summary_for_agent(self, agent_type: AgentType) -> str: + """Get a context summary for a specific agent. + + Args: + agent_type: Name of the agent requesting context summary. + + Returns: + str: A formatted context summary. + """ + context = self.context_manager.get_latest_context_by_agent(agent_type) + if not context: + return "No context available for this agent." + + return context.get_full_context_summary() diff --git a/ontology_platform/vendored/ontocast/ontocast/onto/unit_states.py b/ontology_platform/vendored/ontocast/ontocast/onto/unit_states.py new file mode 100644 index 0000000..63648f8 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/onto/unit_states.py @@ -0,0 +1,198 @@ +"""Dedicated state models for parallel unit loops.""" + +from collections import defaultdict +from copy import deepcopy + +from pydantic import Field + +from ontocast.onto.constants import DEFAULT_DOMAIN +from ontocast.onto.content_unit import ContentUnit, SourceUnit +from ontocast.onto.enum import FailureStage, Status, WorkflowNode +from ontocast.onto.model import ( + BasePydanticModel, + ExternalEvidenceCacheEntry, + ExternalEvidenceHit, + ExternalEvidencePlan, + ExternalEvidenceRequest, + Suggestions, +) +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.sparql_models import GraphUpdate +from ontocast.onto.state import AgentState, BudgetTracker + + +def _render_updated_graph( + graph: RDFGraph, updates: list[GraphUpdate], max_triples: int | None = None +) -> tuple[RDFGraph, bool]: + """Apply GraphUpdate objects to a graph. Delegates to AgentState implementation.""" + return AgentState.render_updated_graph(graph, updates, max_triples=max_triples) + + +class UnitState(BasePydanticModel): + """Common per-unit workflow state.""" + + ontology_snapshot: Ontology = Field(description="Immutable ontology snapshot") + suggestions: Suggestions = Field(default_factory=Suggestions) + budget_tracker: BudgetTracker = Field(default_factory=BudgetTracker) + max_visits_per_node: int = Field(default=1, ge=1) + + status: Status = Field(default=Status.NOT_VISITED) + failure_stage: FailureStage | None = Field(default=None) + failure_reason: str | None = Field(default=None) + node_visits: dict[WorkflowNode, int] = Field( + default_factory=lambda: defaultdict(int), + ) + external_evidence_plan: ExternalEvidencePlan = Field( + default_factory=ExternalEvidencePlan + ) + external_evidence_hits: list[ExternalEvidenceHit] = Field(default_factory=list) + external_evidence_text: str = Field(default="") + external_evidence_source_count: int = Field(default=0, ge=0) + external_evidence_domains: list[str] = Field(default_factory=list) + external_evidence_planned_at_node: WorkflowNode | None = Field(default=None) + external_evidence_used_by_nodes: list[WorkflowNode] = Field(default_factory=list) + external_evidence_requests: dict[WorkflowNode, ExternalEvidenceRequest] = Field( + default_factory=dict + ) + external_evidence_cache: dict[WorkflowNode, ExternalEvidenceCacheEntry] = Field( + default_factory=dict + ) + + def get_content_unit_progress_string(self) -> str: + """Progress string for logging (single unit context).""" + return "content unit" + + def set_node_status(self, node: WorkflowNode, status: Status) -> None: + """Set workflow node status (for logging).""" + self.status = status + + def set_failure(self, stage: FailureStage, reason: str) -> None: + """Record failure stage and reason.""" + self.failure_stage = stage + self.failure_reason = reason + self.status = Status.FAILED + + def clear_failure(self) -> None: + """Clear failure state.""" + self.failure_stage = None + self.failure_reason = None + + def clear_external_evidence(self) -> None: + """Reset evidence plan, retrieved hits, and rendered evidence block.""" + self.external_evidence_plan = ExternalEvidencePlan() + self.external_evidence_hits = [] + self.external_evidence_text = "" + self.external_evidence_source_count = 0 + self.external_evidence_domains = [] + self.external_evidence_planned_at_node = None + self.external_evidence_cache = {} + + def get_external_evidence_request( + self, node: WorkflowNode + ) -> ExternalEvidenceRequest: + """Return node-scoped search request, defaulting to disabled.""" + return self.external_evidence_requests.get(node, ExternalEvidenceRequest()) + + def set_external_evidence_request( + self, node: WorkflowNode, request: ExternalEvidenceRequest + ) -> None: + """Store node-scoped search request.""" + self.external_evidence_requests[node] = request + + def clear_external_evidence_request(self, node: WorkflowNode) -> None: + """Clear node-scoped search request.""" + self.external_evidence_requests.pop(node, None) + + def set_external_evidence_cache_entry( + self, node: WorkflowNode, entry: ExternalEvidenceCacheEntry + ) -> None: + """Persist node-scoped evidence plan/fetch result cache.""" + self.external_evidence_cache[node] = entry + + def get_external_evidence_cache_entry( + self, node: WorkflowNode + ) -> ExternalEvidenceCacheEntry: + """Return node-scoped evidence cache entry.""" + return self.external_evidence_cache.get(node, ExternalEvidenceCacheEntry()) + + def load_external_evidence_for_node(self, node: WorkflowNode) -> None: + """Load node-scoped evidence cache into active prompt fields.""" + entry = self.get_external_evidence_cache_entry(node) + self.external_evidence_plan = entry.plan + self.external_evidence_hits = entry.hits + self.external_evidence_text = entry.text + self.external_evidence_source_count = entry.source_count + self.external_evidence_domains = entry.domains + self.external_evidence_planned_at_node = node + + def mark_external_evidence_used(self, node: WorkflowNode) -> None: + """Record that a workflow node consumed prepared external evidence.""" + if node not in self.external_evidence_used_by_nodes: + self.external_evidence_used_by_nodes.append(node) + + +class UnitFactsState(UnitState): + """Independent per-unit state for facts extraction and critique.""" + + content_unit: ContentUnit = Field(description="Unit under processing (mutable)") + facts_user_instruction: str = Field(default="") + facts_updates: list[GraphUpdate] = Field(default_factory=list) + + def get_content_unit_progress_string(self) -> str: + """Progress string for logging with content unit index.""" + return f"content unit {self.content_unit.index + 1}" + + def update_facts(self) -> None: + """Apply facts_updates to content_unit.graph and clear the list.""" + if not self.facts_updates: + return + updated_graph, _ = _render_updated_graph( + self.content_unit.graph, self.facts_updates, max_triples=None + ) + self.content_unit.graph = updated_graph + self.facts_updates = [] + + +class UnitOntologyState(UnitState): + """Independent per-unit state for ontology improvement loop.""" + + content_unit: SourceUnit = Field(description="Unit under processing") + ontology_user_instruction: str = Field(default="") + current_ontology: Ontology = Field( + default_factory=Ontology, description="Current ontology under refinement" + ) + ontology_updates: list[GraphUpdate] = Field(default_factory=list) + ontology_updates_applied: list[GraphUpdate] = Field(default_factory=list) + current_domain: str = Field(default=DEFAULT_DOMAIN) + ontology_max_triples: int | None = Field(default=None) + + def get_content_unit_progress_string(self) -> str: + """Progress string for logging with content unit index.""" + return f"content unit {self.content_unit.index + 1}" + + def model_post_init(self, __context) -> None: + """Initialize mutable ontology state from immutable snapshot.""" + self.current_ontology = deepcopy(self.ontology_snapshot) + + @property + def all_updates(self) -> list[GraphUpdate]: + """All ontology updates produced by this unit (applied and pending).""" + return [*self.ontology_updates_applied, *self.ontology_updates] + + def update_ontology(self) -> None: + """Apply ontology_updates to current_ontology and clear the list.""" + if not self.ontology_updates: + return + updated_graph, was_applied = _render_updated_graph( + self.current_ontology.graph, + self.ontology_updates, + max_triples=self.ontology_max_triples, + ) + if not was_applied: + return + + updated_ontology = self.current_ontology.derive_updated_version(updated_graph) + self.ontology_updates_applied += self.ontology_updates + self.current_ontology = updated_ontology + self.ontology_updates = [] diff --git a/ontology_platform/vendored/ontocast/ontocast/onto/util.py b/ontology_platform/vendored/ontocast/ontocast/onto/util.py new file mode 100644 index 0000000..7662323 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/onto/util.py @@ -0,0 +1,35 @@ +import re +from urllib.parse import urlparse + +from ontocast.util import CONVENTIONAL_MAPPINGS + + +def derive_ontology_id(iri: str) -> str | None: + if not isinstance(iri, str) or not iri.strip(): + return None + + normalized_iri = iri.strip().rstrip("/#") + + if normalized_iri in CONVENTIONAL_MAPPINGS: + return CONVENTIONAL_MAPPINGS[normalized_iri] + + parsed = urlparse(normalized_iri) + + candidate = ( + parsed.path.rsplit("/", 1)[-1] + if parsed.path and "/" in parsed.path + else parsed.netloc.split(".")[0] + if parsed.netloc + else normalized_iri + ) + + return _clean_derived_id(candidate) + + +def _clean_derived_id(value: str) -> str | None: + value = re.sub(r"\.(owl|ttl|rdf|xml)$", "", value, flags=re.IGNORECASE) + match = re.match(r"^(.*?)\.(org|com|net|io|edu|gov|int|mil)$", value, re.IGNORECASE) + if match: + value = match.group(1) + result = re.sub(r"[^a-zA-Z0-9_-]", "", value).lower() + return result if result else None diff --git a/ontology_platform/vendored/ontocast/ontocast/prompt/__init__.py b/ontology_platform/vendored/ontocast/ontocast/prompt/__init__.py new file mode 100644 index 0000000..057d968 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/prompt/__init__.py @@ -0,0 +1,15 @@ +"""Prompt templates for OntoCast. + +This package contains prompt templates used by the OntoCast framework +for LLM interactions. These prompts are designed to guide language +models in performing specific tasks within the ontology processing +workflow. + +Available prompts: +- render_ontology: Generate ontology triples from text +- render_facts: Extract facts from text using ontologies +- select_ontology: Choose appropriate ontologies for text +- criticise_ontology: Evaluate and critique ontology quality +- criticise_facts: Validate and critique extracted facts +- common: Shared prompt templates and components +""" diff --git a/ontology_platform/vendored/ontocast/ontocast/prompt/common.py b/ontology_platform/vendored/ontocast/ontocast/prompt/common.py new file mode 100644 index 0000000..40daffc --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/prompt/common.py @@ -0,0 +1,80 @@ +"""Common prompt templates and components shared across the application. + +This module contains reusable prompt templates and components to avoid +duplication across different prompt modules. +""" + +system_preamble_semantic = """ +# SYSTEM INSTRUCTION + +You are an expert in semantic technologies, SPARQL and triple extraction. +""" + +system_preamble_ontology = """ +# SYSTEM INSTRUCTION + +You are an expert in semantic technologies and ontology engineering. +""" + +ontology_template = """\n\n +# ONTOLOGY + +```ttl +{ontology_ttl} +``` +""" + +text_template = """\n\n +# TEXT + +``` +{text} +``` +""" + +facts_template = """\n\n +# SEMANTIC GRAPH OF FACTS +The following facts were extracted + +```ttl +{facts_ttl} +``` +""" + + +output_instruction_empty = """\n\n +# OUTPUT INSTRUCTION + +""" + +output_instruction_ttl = """\n\n +# OUTPUT INSTRUCTION + +1. ontology must be provided in turtle format as a single string +2. define all prefixes for all namespaces used in the ontology, etc rdf, rdfs, owl, schema, etc. +""" + +output_instruction_sparql = """\n\n +# OUTPUT INSTRUCTION + +Generate SPARQL operations that modify the existing ontology, not replace it entirely. +Follow the Pydantic schema definitions exactly - they fully specify the output structure. +""" + +user_template = """\n\n +# USER INSTRUCTION + +{user_instruction} +""" + +suggestion_general_template = """\n\n +## GENERAL + +{general_suggestion} +""" + +suggestion_concrete_template = """\n\n +## CONCRETE + +{suggestion_str} +""" diff --git a/ontology_platform/vendored/ontocast/ontocast/prompt/criticise_facts.py b/ontology_platform/vendored/ontocast/ontocast/prompt/criticise_facts.py new file mode 100644 index 0000000..8065bbc --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/prompt/criticise_facts.py @@ -0,0 +1,78 @@ +"""Enhanced facts criticism prompts. + +This module provides enhanced prompt templates for facts criticism that support +with improved critique quality. +""" + +from ontocast.onto.constants import DEFAULT_IRI + +from .common import system_preamble_semantic + +template_prompt = """ +{preamble} + +{evaluation_instruction} + +{user_instruction} + +{ontology_chapter} + +{facts_chapter} + +{text_chapter} + +{format_instructions} +""" + +preamble = f""" +{system_preamble_semantic} +You are given an ontology, a text and a semantic graph of facts, generated from the text (guided by ontology). +Following evaluation guidelines provide concrete suggestions for improvement of the extracted facts graph with respect to provided text and ontology. +""" + + +evaluation_instruction = f"""\n\n +# EVALUATION GUIDELINES + +1. Appropriateness: Are the facts appropriate for the document? + +2. Completeness: Are all possible facts extracted from the text given the ontology? + +3. Concreteness: Only concrete facts should be extracted. + +4. Structure: Are all concrete entities linked to abstract classes via relations? + +5. Namespace Consistency: + - Facts MUST use `cd:` with fixed namespace `<{DEFAULT_IRI}>` + - Flag any fact entity that uses a different "facts-like" namespace as error + - `cd:` is reserved for concrete instances/facts only (not ontology classes/properties) + +6. Ontology Validity: Verify that every non-cd: entity exists in either the provided domain ontology or standard ontologies (RDFS, OWL, schema.org, etc.). + - Every class, property, and individual using ontology prefixes (fca:, onto:, schema:, etc.) must be defined in its respective ontology + - Invented entities using ontology prefixes are errors + - Fix: REMOVE invented entities or REPLACE with semantically similar existing entities from available ontologies + - Treat morphology/casing variants of ontology terms as likely mistakes (e.g. `AppealCourt_Rouen` vs `AppealCourtRouen`) unless the variant exists explicitly in ontology + - For multilingual variants, prefer the exact canonical ontology IRI that exists in ontology; do not allow translated/reformatted ontology-prefixed IRIs as new entities + +# VERIFICATION CHECKLIST + +Before finalizing your critique: + +1. For every triple using a non-cd: prefix, confirm the entity exists in the corresponding ontology + - If it does not exist exactly, flag and propose replacement with the closest existing canonical ontology IRI + +2. For every fact entity, confirm it uses `cd:` with `<{DEFAULT_IRI}>` + +3. When you find invented entities: + - Flag as error + - Search available ontologies for semantically similar entities + - Suggest REPLACE if found, or REMOVE if no suitable replacement exists + +# SEARCH DECISION OUTPUT + +Include `external_evidence_request` in your structured response: +- Set `initiate_search=true` only if external web evidence is necessary to resolve uncertainty + that blocks a confident critique. +- Keep `initiate_search=false` when the source text + ontology are sufficient. +- When true, provide concise `rationale` and optional focused `query_hints`. +""" diff --git a/ontology_platform/vendored/ontocast/ontocast/prompt/criticise_ontology.py b/ontology_platform/vendored/ontocast/ontocast/prompt/criticise_ontology.py new file mode 100644 index 0000000..41cd235 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/prompt/criticise_ontology.py @@ -0,0 +1,67 @@ +from ontocast.onto.constants import DEFAULT_IRI + +template_prompt = """ +{preamble} + +{intro_instruction} + +{ontology_criteria} + +{user_instruction} + +{ontology_chapter} + +{text_chapter} + +{external_evidence} + +{format_instructions} +""" + +intro_instruction = """ +You are given a text and an ontology. +You task is to evaluate the quality of the ontology with respect to the provided doc and provide a constructive critique of the ontology with respect to provided text. +""" + + +ontology_criteria = f""" +# TASK +Provide a constructive, actionable critique following these priorities: + +## PRIMARY EVALUATION CRITERIA (in order of importance): +1. **Consistency**: No logical contradictions, proper use of OWL semantics +2. **Completeness**: All key domain concepts from text are represented +3. **Correctness**: Accurate relationships, proper datatypes, valid syntax +4. **Structure**: Appropriate class hierarchies and property definitions +5. **Abstraction**: Uses abstract classes/properties (no instances) +6. **Domain Coverage**: Includes implicit domain knowledge beyond literal text + +## SCORING: +- 90-100: Excellent - minor refinements only +- 70-89: Good - some improvements needed +- 50-69: Adequate - significant gaps or errors +- 30-49: Poor - major structural issues +- 0-29: Inadequate - fundamental problems + +## OUTPUT REQUIREMENTS: +1. Start with what works well (2-3 strengths) +2. Group fixes by severity: critical → important → minor + - Use severity: "critical" (breaks semantic graph), "important" (significant gap), or "minor" (polish) +3. For each fix, provide: + - Exact text evidence (quote from source) + - Clear before/after using Turtle syntax + - Actionable explanation +4. Systemic summary should identify patterns, not repeat individual fixes + +## SPECIAL INSTRUCTIONS: +- For missing concepts: specify WHERE in the hierarchy they belong +- For relationship errors: explain the correct domain/range constraints +- For redundancies: suggest consolidation strategy +- Prioritize fixes that have cascading impact +- Enforce namespace hygiene: ontology classes/properties MUST NOT be modeled in `cd:` (`{DEFAULT_IRI}`), since `cd:` is reserved for facts/instances +- Treat external web evidence as optional support only. If evidence conflicts with source text or ontology context, prioritize source text and ontology context. +- Include `external_evidence_request` in your structured response: + - Set `initiate_search=true` only when external web evidence is needed to resolve ambiguity. + - Keep `initiate_search=false` when source text + ontology are sufficient. + - Provide concise `rationale` and optional focused `query_hints` when search is requested. +""" diff --git a/ontology_platform/vendored/ontocast/ontocast/prompt/render_facts.py b/ontology_platform/vendored/ontocast/ontocast/prompt/render_facts.py new file mode 100644 index 0000000..e10d21e --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/prompt/render_facts.py @@ -0,0 +1,101 @@ +from .common import system_preamble_semantic + +template_prompt = """ +{preamble} + +{facts_instruction} + +{user_instruction} + +{ontology_chapter} + +{text_chapter} + +{fact_chapter} + +{improvement_instruction} + +{output_instruction} + +{format_instructions} +""" + +preamble = f""" +{system_preamble_semantic} +Generate semantic triples representing facts (not abstract entities) based on provided domain ontology. +""" + +facts_instruction_template = """\n\n +# OPERATIONAL GUIDELINES + +1. Facts MUST use the fixed namespace `{facts_namespace}` with the prefix `cd:` (declare exactly: `@prefix cd: <{facts_namespace}> .`). +2. Use the provided domain ontology <{ontology_namespace}> (below) and standard ontologies (RDFS, OWL, schema.org, etc.) to identify/infer entities, classes, types, and relationships +3. Thoroughly Extract and Link: extract all possible text mentions that correspond to entities, classes, types, or relationships defined in the domain ontology <{ontology_namespace}>. When referring to the domain ontology, use the prefix `{ontology_prefix}:` +4. Enforce typing: all `cd:` entities (facts) must be linked (e.g. using rdf:type) to entities from either the DOMAIN ONTOLOGY <{ontology_namespace}> or basic ontologies (RDFS, OWL, etc), e.g. rdfs:Class, rdf:Property, schema:Person, schema:Organization, etc. +5. Define all prefixes for all namespaces used rdf, rdfs, owl, schema, etc +6. CRITICAL - Entity Matching Protocol: + - BEFORE creating any `cd:` entity, you MUST search the domain ontology for existing entities that match the concept semantically + - Match by meaning, not just exact label matching + - Check all language variants of `rdfs:label` and alternative names + - If a matching entity exists in the domain ontology, use its IRI directly - DO NOT create a duplicate in the `cd:` namespace + - Only create `cd:` entities for NEW facts not already defined in the ontology + - NEVER mint new entities under the ontology prefix (`{ontology_prefix}:`) unless that exact IRI already exists in the provided ontology + - Preserve canonical ontology IRIs exactly as given (character-for-character): no translation, no transliteration, no snake_case/camelCase changes, no suffix/prefix changes + - Cross-lingual mentions (e.g. French/English variants) MUST be linked to the existing canonical ontology IRI when semantically equivalent + - If no ontology entity can be verified, create a `cd:` entity instead of inventing a new ontology-prefixed IRI +7. Maximize atomicity: decompose complex facts and complex literals into simple subject-predicate-object statements (e.g. decompose person's first name and last name). +8. Literals Handling: + - Use appropriate XSD datatypes: xsd:integer, xsd:decimal, xsd:float, xsd:date, xsd:dateTime + - Dates: Use ISO 8601 format (e.g., "2024-01-15"^^xsd:date) + - Numbers: Always use typed literals (e.g., "42"^^xsd:integer, "99.95"^^xsd:decimal) + - Currencies: Include currency codes (e.g., "1000"^^xsd:decimal with schema:priceCurrency "USD") +9. To extract data from tables, use CSV on the Web (CSVW) to describe tables +10. No comments in Turtle: Output must contain only @prefix declarations and triples. Do not include comments (lines starting with #) +11. Decide whether external evidence is needed for a retry and set `external_evidence_request`: + - Set `initiate_search=true` only when ambiguity/term disambiguation/standards lookup materially blocks quality. + - Otherwise keep `initiate_search=false`. + - Provide concise `rationale` and optional focused `query_hints` when search is requested. +""" + +improvement_instruction_template = """\n\n +# IMPROVEMENT INSTRUCTION + +The current iteration of the graph of factual triples has been reviewed by Critic, who provided suggestions for improvement. + +CRITICAL: You are the final decision-maker. Critic's suggestions are advisory, not mandatory. Think independently. + +Your task is to critically evaluate and improve the triples: + +1. Independently verify each suggestion - Before implementing ANY suggestion, verify it against: + - The original source text (does it accurately reflect what's written?) + - The OPERATIONAL GUIDELINES (does it follow the rules?) + - The domain ontology (does it use entities correctly?) + - Logical consistency (does it make semantic sense?) + +2. Implement only valid improvements - Apply suggestions that are demonstrably correct and enhance accuracy or completeness. If uncertain, prioritize faithfulness to the source text. + +3. Actively reject flawed suggestions - If a suggestion is: + - Factually incorrect (contradicts the source text) + - Violates OPERATIONAL GUIDELINES + - Would introduce errors or degrade quality + - Based on misunderstanding of the ontology + + Then REJECT it and briefly explain why in your response. + +4. Think beyond the critique - Critic may have: + - Missed issues entirely + - Identified patterns but not all instances + - Focused on some aspects while overlooking others + + Proactively identify and fix additional problems not mentioned in the critique. + +5. Verify every change - Before finalizing, double-check that: + - Each triple accurately represents information from the source text + - Existing ontology entities are used instead of creating new cd: entities + - No ontology-prefixed entity was invented or renamed + - All OPERATIONAL GUIDELINES are satisfied + - The overall graph is more complete and accurate than before + +Your goal: Produce the most accurate representation of the source text, not to satisfy Critic. +{suggestions_instruction} +""" diff --git a/ontology_platform/vendored/ontocast/ontocast/prompt/render_ontology.py b/ontology_platform/vendored/ontocast/ontocast/prompt/render_ontology.py new file mode 100644 index 0000000..820771f --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/prompt/render_ontology.py @@ -0,0 +1,103 @@ +from ontocast.onto.constants import DEFAULT_IRI + +template_prompt = """ +{preamble} + +{intro_instruction} + +{ontology_instruction} + +{user_instruction} + +{improvement_instruction} + +{ontology_ttl} + +{text} + +{external_evidence} + +{output_instruction} + +{format_instructions} +""" + +intro_instruction_fresh = """ +1. Develop a new domain ontology based on the provided document. When deciding on the name and scope, remember that the document you are given is an example, so the ontology name, ontology identifier and scope should be at least one level of abstraction above the scope of the document. +2. Propose a domain specific and succinct specifier if for the new ontology, which should be an abbreviation, consistent with the Ontology property `ontology_id`, for example it could be `abc` for a hypothetical A... B... of C... Ontology. +3. From the proposed `ontology_id` derive an IRI (URI) using domain {current_domain}, for example `{current_domain}/abc` +""" + + +intro_instruction_update = """ +Update/modify the domain ontology {ontology_iri} provided below with abstract entities and relations that can be inferred from the document or known to hold in the domain the document pertains to. + +{ontology_desc} + +Feel free to update the description of the ontology to make it more accurate and complete, do not change ontology IRI, prefix, or ontology_id. +""" + +prefix_instruction = """Use prefix `{ontology_prefix}` for entities/properties placed in the current domain ontology. DECLARE the prefix in preamble!""" +prefix_instruction_fresh = """Define a new prefix for the current domain ontology. DECLARE the prefix in preamble!""" + + +general_ontology_instruction = f""" +### GENERAL + +1. **Only model abstract concepts — no instances or facts from the document** (e.g., no specific case names, dates, or people). + +2. **All abstract entities (classes/properties) must connect to:** + - **Standard vocabularies (RDFS, OWL, schema.org, SKOS) via rdfs:subClassOf, rdf:type, rdfs:subPropertyOf, etc.** + - **OR other entities within this ontology** + - **Example: `legal:CourtDecision rdfs:subClassOf schema:Event .`** + +3. **Every new entity must have:** + - **rdfs:label (required)** + - **rdfs:comment describing its purpose (required)** + - **At least one relationship to existing classes/properties** + +4. **Ensure ontology faithfully represents domain semantics from the document.** Use **domain knowledge to add implicit relationships** not explicitly stated but clearly implied. + +5. **Maintain consistency with existing conventions:** + - **Language: Use same language for labels/comments as existing ontology** + - **Naming: Follow existing PascalCase/camelCase patterns** + - **Structure: Respect existing hierarchy depth and property usage patterns** + +6. {prefix_instruction} + +7. **Define property characteristics when applicable:** + - **owl:FunctionalProperty** — property has at most one value (e.g., `foaf:homepage`, `dcterms:identifier`) + - **owl:InverseFunctionalProperty** — value uniquely identifies the subject (e.g., `foaf:mbox`, `schema:email`) + - **owl:TransitiveProperty** — if A→B and B→C, then A→C (e.g., `skos:broader`, `org:subOrganizationOf`) + - **owl:SymmetricProperty** — if A→B, then B→A (e.g., `foaf:knows`, `schema:relatedTo`) + - **owl:AsymmetricProperty** — if A→B, then NOT B→A (e.g., `org:hasSubOrganization`, `prov:wasDerivedFrom`) + - **owl:ReflexiveProperty** — every entity relates to itself (e.g., `owl:sameAs`) + - **owl:IrreflexiveProperty** — no entity relates to itself (e.g., `owl:differentFrom`) + +8. **For measurable properties, specify units using schema:unitCode, rdfs:comment, or explicit unit classes** (e.g., `schema:duration schema:unitCode "DAY"` or `time:numericDuration rdfs:comment "Duration measured in days"`). + +9. **Never model ontology classes/properties under `cd:`.** The `cd:` namespace (`{DEFAULT_IRI}`) is reserved for factual instances only. + +10. **When introducing entities from other domain ontologies, declare their namespace prefixes** (e.g., `@prefix foaf: .` or `@prefix dcterms: .`). + +11. **External evidence is optional and advisory**: + - Use web evidence only to resolve ambiguity, terminology, standards, or domain conventions. + - If external snippets conflict with source text or ontology context, prioritize source text and ontology context. + - Avoid adding entities/relations that are only weakly supported by web snippets. + +12. **Search decision output**: + - Include `external_evidence_request` in your structured response. + - Set `initiate_search=true` only when web evidence is necessary to resolve uncertainty + before a better retry can be produced. + - Otherwise keep `initiate_search=false`. + - When true, provide short `rationale` and optional focused `query_hints`. +""" + + +improvement_instruction_template = """\n\n +# IMPROVEMENT INSTRUCTION + +The current iteration of the ontology was not deemed accurate by Critic, who left the following suggestions for improvement: + +{suggestions_instruction} +""" diff --git a/ontology_platform/vendored/ontocast/ontocast/prompt/select_ontology.py b/ontology_platform/vendored/ontocast/ontocast/prompt/select_ontology.py new file mode 100644 index 0000000..bab9ab4 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/prompt/select_ontology.py @@ -0,0 +1,17 @@ +template_prompt = """ +You are a helpful assistant that decides which ontology to use for a given document. +You are given a numbered list of ontologies and a document excerpt. +You need to select which ontology can be used for the document to create a semantic graph. + +Select from the following options: + +0. No suitable ontology available + +{ontologies_list} + + +Here is an excerpt from the document: +{excerpt} + +{format_instructions} +""" diff --git a/ontology_platform/vendored/ontocast/ontocast/stategraph/__init__.py b/ontology_platform/vendored/ontocast/ontocast/stategraph/__init__.py new file mode 100644 index 0000000..8617a64 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/stategraph/__init__.py @@ -0,0 +1,13 @@ +"""Agent module for OntoCast. + +This module provides a collection of agents that handle various aspects of ontology +processing, including document conversion, text chunking, fact aggregation, and +ontology management. Each agent is designed to perform a specific task in the +ontology processing pipeline. +""" + +from .create import create_agent_graph + +__all__ = [ + "create_agent_graph", +] diff --git a/ontology_platform/vendored/ontocast/ontocast/stategraph/atomic.py b/ontology_platform/vendored/ontocast/ontocast/stategraph/atomic.py new file mode 100644 index 0000000..e385478 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/stategraph/atomic.py @@ -0,0 +1,248 @@ +"""Reusable per-unit render/critic retry loops. + +These loops are designed for map/reduce execution where each content unit +is processed independently. They deep-copy the incoming unit state, then run +render -> critic until success or retry exhaustion. + +Minimal tools contract: +- A ``ToolBox`` instance is still expected by type, but the loop itself only + relies on downstream agent calls that resolve an LLM via + ``tools.get_llm_tool(state.budget_tracker)``. +- No triple-store, chunker, converter, or aggregator capabilities are required + for these atomic loops. +""" + +import logging + +from ontocast.agent.criticise_facts import criticise_facts +from ontocast.agent.criticise_ontology import criticise_ontology +from ontocast.agent.external_evidence import ( + fetch_external_evidence_for_node, + plan_external_evidence_for_node, +) +from ontocast.agent.render_facts import render_facts +from ontocast.agent.render_ontology import render_ontology +from ontocast.onto.enum import Status, WorkflowNode +from ontocast.onto.model import ExternalEvidenceCacheEntry, ExternalEvidenceRequest +from ontocast.onto.unit_states import UnitFactsState, UnitOntologyState +from ontocast.tool.atomic import AtomicToolBox + +logger = logging.getLogger(__name__) + + +def _resolve_max_visits_limit(state_visits: int, override: int | None) -> int: + """Return a safe visit limit while respecting explicit overrides.""" + visits = state_visits if override is None else override + return max(1, visits) + + +def _reset_node_evidence_context( + state: UnitFactsState | UnitOntologyState, node: WorkflowNode +) -> None: + """Start node execution in no-search mode with empty evidence context.""" + state.set_external_evidence_request(node, ExternalEvidenceRequest()) + state.set_external_evidence_cache_entry(node, ExternalEvidenceCacheEntry()) + state.load_external_evidence_for_node(node) + + +async def facts_loop( + state: UnitFactsState, tools: AtomicToolBox, max_visits_per_node: int | None = None +) -> UnitFactsState: + """Run facts render/critic loop for one content unit. + + Ontology is selected once per document in the main workflow; ontology_snapshot + is always provided by the caller. + """ + unit_state = state.model_copy(deep=True) + max_visits = _resolve_max_visits_limit( + unit_state.max_visits_per_node, max_visits_per_node + ) + unit_state.max_visits_per_node = max_visits + + for render_attempt in range(1, max_visits + 1): + unit_state.node_visits[WorkflowNode.TEXT_TO_FACTS] += 1 + _reset_node_evidence_context(unit_state, WorkflowNode.TEXT_TO_FACTS) + unit_state = await render_facts(unit_state, tools) + if unit_state.status != Status.SUCCESS: + render_request = unit_state.get_external_evidence_request( + WorkflowNode.TEXT_TO_FACTS + ) + if render_request.initiate_search: + unit_state = await plan_external_evidence_for_node( + unit_state, tools, WorkflowNode.TEXT_TO_FACTS + ) + unit_state = await fetch_external_evidence_for_node( + unit_state, tools, WorkflowNode.TEXT_TO_FACTS + ) + unit_state = await render_facts(unit_state, tools) + if unit_state.status == Status.SUCCESS: + logger.info( + "Unit facts render recovered with search at attempt %s/%s", + render_attempt, + max_visits, + ) + # Continue to critic tier below. + else: + logger.info( + "Unit facts render failed at attempt %s/%s (with search)", + render_attempt, + max_visits, + ) + continue + else: + logger.info( + "Unit facts render failed at attempt %s/%s (no search request)", + render_attempt, + max_visits, + ) + continue + + for critic_attempt in range(1, max_visits + 1): + unit_state.node_visits[WorkflowNode.CRITICISE_FACTS] += 1 + _reset_node_evidence_context(unit_state, WorkflowNode.CRITICISE_FACTS) + unit_state = await criticise_facts(unit_state, tools) + if unit_state.status == Status.SUCCESS: + logger.info( + "Unit facts loop converged at render %s/%s critic %s/%s", + render_attempt, + max_visits, + critic_attempt, + max_visits, + ) + return unit_state + + critic_request = unit_state.get_external_evidence_request( + WorkflowNode.CRITICISE_FACTS + ) + if not critic_request.initiate_search: + logger.info( + "Unit facts critic failed at render %s/%s critic %s/%s without search request", + render_attempt, + max_visits, + critic_attempt, + max_visits, + ) + break + + unit_state = await plan_external_evidence_for_node( + unit_state, tools, WorkflowNode.CRITICISE_FACTS + ) + unit_state = await fetch_external_evidence_for_node( + unit_state, tools, WorkflowNode.CRITICISE_FACTS + ) + unit_state = await criticise_facts(unit_state, tools) + if unit_state.status == Status.SUCCESS: + logger.info( + "Unit facts loop converged with critic search at render %s/%s critic %s/%s", + render_attempt, + max_visits, + critic_attempt, + max_visits, + ) + return unit_state + + continue + + logger.info("Unit facts loop exhausted retries") + return unit_state + + +async def ontology_loop( + state: UnitOntologyState, + tools: AtomicToolBox, + max_visits_per_node: int | None = None, +) -> UnitOntologyState: + """Run ontology render/critic loop for one content unit. + + Ontology is selected once per document in the main workflow; ontology_snapshot + is always provided by the caller (may be null for fresh-ontology builds). + """ + unit_state = state.model_copy(deep=True) + max_visits = _resolve_max_visits_limit( + unit_state.max_visits_per_node, max_visits_per_node + ) + unit_state.max_visits_per_node = max_visits + + for render_attempt in range(1, max_visits + 1): + unit_state.node_visits[WorkflowNode.TEXT_TO_ONTOLOGY] += 1 + _reset_node_evidence_context(unit_state, WorkflowNode.TEXT_TO_ONTOLOGY) + unit_state = await render_ontology(unit_state, tools) + if unit_state.status != Status.SUCCESS: + render_request = unit_state.get_external_evidence_request( + WorkflowNode.TEXT_TO_ONTOLOGY + ) + if render_request.initiate_search: + unit_state = await plan_external_evidence_for_node( + unit_state, tools, WorkflowNode.TEXT_TO_ONTOLOGY + ) + unit_state = await fetch_external_evidence_for_node( + unit_state, tools, WorkflowNode.TEXT_TO_ONTOLOGY + ) + unit_state = await render_ontology(unit_state, tools) + if unit_state.status == Status.SUCCESS: + logger.info( + "Unit ontology render recovered with search at attempt %s/%s", + render_attempt, + max_visits, + ) + else: + logger.info( + "Unit ontology render failed at attempt %s/%s (with search)", + render_attempt, + max_visits, + ) + continue + else: + logger.info( + "Unit ontology render failed at attempt %s/%s (no search request)", + render_attempt, + max_visits, + ) + continue + + for critic_attempt in range(1, max_visits + 1): + unit_state.node_visits[WorkflowNode.CRITICISE_ONTOLOGY] += 1 + _reset_node_evidence_context(unit_state, WorkflowNode.CRITICISE_ONTOLOGY) + unit_state = await criticise_ontology(unit_state, tools) + if unit_state.status == Status.SUCCESS: + logger.info( + "Unit ontology loop converged at render %s/%s critic %s/%s", + render_attempt, + max_visits, + critic_attempt, + max_visits, + ) + return unit_state + + critic_request = unit_state.get_external_evidence_request( + WorkflowNode.CRITICISE_ONTOLOGY + ) + if not critic_request.initiate_search: + logger.info( + "Unit ontology critic failed at render %s/%s critic %s/%s without search request", + render_attempt, + max_visits, + critic_attempt, + max_visits, + ) + break + + unit_state = await plan_external_evidence_for_node( + unit_state, tools, WorkflowNode.CRITICISE_ONTOLOGY + ) + unit_state = await fetch_external_evidence_for_node( + unit_state, tools, WorkflowNode.CRITICISE_ONTOLOGY + ) + unit_state = await criticise_ontology(unit_state, tools) + if unit_state.status == Status.SUCCESS: + logger.info( + "Unit ontology loop converged with critic search at render %s/%s critic %s/%s", + render_attempt, + max_visits, + critic_attempt, + max_visits, + ) + return unit_state + + logger.info("Unit ontology loop exhausted retries") + return unit_state diff --git a/ontology_platform/vendored/ontocast/ontocast/stategraph/create.py b/ontology_platform/vendored/ontocast/ontocast/stategraph/create.py new file mode 100644 index 0000000..f13d8bc --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/stategraph/create.py @@ -0,0 +1,95 @@ +from functools import partial + +from langgraph.constants import END, START +from langgraph.graph import StateGraph +from langgraph.graph.state import CompiledStateGraph + +from ontocast.agent import chunk_text, convert_document, select_ontology +from ontocast.agent.serialize import serialize +from ontocast.onto.enum import WorkflowNode +from ontocast.onto.state import AgentState +from ontocast.stategraph.node_factories import ( + make_bootstrap_ontology_node, + make_consolidate_ontology_node, + make_merge_facts_node, + make_normalize_ontology_node, + make_render_facts_node, + make_render_ontology_node, +) +from ontocast.stategraph.routing import ( + route_after_ontology_consolidation, + route_after_ontology_selection, +) +from ontocast.toolbox import ToolBox + + +def create_agent_graph(tools: ToolBox) -> CompiledStateGraph: + """Create the parallel map/reduce agent graph. + + Flow: CONVERT -> CHUNK -> (conditional) + - ontology null: SELECT_ONTOLOGY -> (ontology or facts map) + - ontology set: PARALLEL_ONTOLOGY_MAP or PARALLEL_FACTS_MAP + - render_ontology: PARALLEL_ONTOLOGY_MAP -> REDUCE_ONTOLOGY -> + [PARALLEL_FACTS_MAP -> REDUCE_FACTS]? -> SERIALIZE + - render_facts only: PARALLEL_FACTS_MAP -> REDUCE_FACTS -> SERIALIZE + + One ontology is selected per document in the main workflow (SELECT_ONTOLOGY). + """ + workflow = StateGraph(AgentState) + + convert_document_node = partial(convert_document, tools=tools) + chunk_text_node = partial(chunk_text, tools=tools) + select_ontology_node = partial(select_ontology, tools=tools) + serialize_node = partial(serialize, tools=tools) + + bootstrap_ontology_node = make_bootstrap_ontology_node(tools) + render_ontology_node = make_render_ontology_node(tools) + normalize_ontology_node = make_normalize_ontology_node(tools) + consolidate_ontology_node = make_consolidate_ontology_node(tools) + render_facts_node = make_render_facts_node(tools) + merge_facts_node = make_merge_facts_node(tools) + + workflow.add_node(WorkflowNode.CONVERT_TO_MD, convert_document_node) + workflow.add_node(WorkflowNode.CHUNK, chunk_text_node) + workflow.add_node(WorkflowNode.SELECT_ONTOLOGY, select_ontology_node) + workflow.add_node(WorkflowNode.BOOTSTRAP_ONTOLOGY, bootstrap_ontology_node) + workflow.add_node(WorkflowNode.RENDER_ONTOLOGY_UPDATE, render_ontology_node) + workflow.add_node(WorkflowNode.NORMALIZE_ONTOLOGY_UPDATES, normalize_ontology_node) + workflow.add_node(WorkflowNode.CONSOLIDATE_ONTOLOGY, consolidate_ontology_node) + workflow.add_node(WorkflowNode.RENDER_FACTS, render_facts_node) + workflow.add_node(WorkflowNode.MERGE_FACTS, merge_facts_node) + workflow.add_node(WorkflowNode.SERIALIZE, serialize_node) + workflow.add_edge(WorkflowNode.CHUNK, WorkflowNode.SELECT_ONTOLOGY) + workflow.add_conditional_edges( + WorkflowNode.SELECT_ONTOLOGY, + route_after_ontology_selection, + { + WorkflowNode.BOOTSTRAP_ONTOLOGY: WorkflowNode.BOOTSTRAP_ONTOLOGY, + WorkflowNode.RENDER_ONTOLOGY_UPDATE: WorkflowNode.RENDER_ONTOLOGY_UPDATE, + WorkflowNode.RENDER_FACTS: WorkflowNode.RENDER_FACTS, + }, + ) + workflow.add_edge( + WorkflowNode.BOOTSTRAP_ONTOLOGY, WorkflowNode.RENDER_ONTOLOGY_UPDATE + ) + workflow.add_edge(START, WorkflowNode.CONVERT_TO_MD) + workflow.add_edge(WorkflowNode.CONVERT_TO_MD, WorkflowNode.CHUNK) + workflow.add_edge( + WorkflowNode.RENDER_ONTOLOGY_UPDATE, WorkflowNode.NORMALIZE_ONTOLOGY_UPDATES + ) + workflow.add_edge( + WorkflowNode.NORMALIZE_ONTOLOGY_UPDATES, WorkflowNode.CONSOLIDATE_ONTOLOGY + ) + workflow.add_conditional_edges( + WorkflowNode.CONSOLIDATE_ONTOLOGY, + route_after_ontology_consolidation, + { + WorkflowNode.RENDER_FACTS: WorkflowNode.RENDER_FACTS, + WorkflowNode.SERIALIZE: WorkflowNode.SERIALIZE, + }, + ) + workflow.add_edge(WorkflowNode.RENDER_FACTS, WorkflowNode.MERGE_FACTS) + workflow.add_edge(WorkflowNode.MERGE_FACTS, WorkflowNode.SERIALIZE) + workflow.add_edge(WorkflowNode.SERIALIZE, END) + + return workflow.compile() diff --git a/ontology_platform/vendored/ontocast/ontocast/stategraph/helpers.py b/ontology_platform/vendored/ontocast/ontocast/stategraph/helpers.py new file mode 100644 index 0000000..9d4112b --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/stategraph/helpers.py @@ -0,0 +1,57 @@ +import logging + +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.state import AgentState +from ontocast.onto.unit_states import UnitOntologyState + +logger = logging.getLogger(__name__) + + +def build_ontology_delta_graph(result: UnitOntologyState) -> RDFGraph: + """Build a delta graph from a unit ontology result. + + If update operations exist, only inserted triples are aggregated. + Otherwise, the current ontology snapshot is used as the delta. + """ + if result.all_updates: + delta_graph = RDFGraph() + for graph_update in result.all_updates: + insert_graph = graph_update.extract_insert_graph() + for triple in insert_graph: + delta_graph.add(triple) + for prefix, namespace_uri in insert_graph.namespaces(): + if prefix: + delta_graph.bind(prefix, namespace_uri) + return delta_graph + + return result.current_ontology.graph.copy() + + +def build_document_excerpt(state: AgentState) -> str: + """Create a representative excerpt from sampled source units.""" + excerpt_parts: list[str] = [] + + if state.content_units: + unit_count = len(state.content_units) + if unit_count == 1: + sample_indices = [0] + elif unit_count == 2: + sample_indices = [0, 1] + else: + sample_indices = [0, 1, unit_count // 2, unit_count - 1] + + visited_indices: set[int] = set() + for index in sample_indices: + if index in visited_indices or index < 0 or index >= unit_count: + continue + visited_indices.add(index) + unit_text = state.content_units[index].text.strip() + if not unit_text: + continue + excerpt_parts.append(unit_text) + + if excerpt_parts: + return "\n\n[...]\n\n".join(excerpt_parts) + if state.input_text: + return state.input_text + return "" diff --git a/ontology_platform/vendored/ontocast/ontocast/stategraph/node_factories.py b/ontology_platform/vendored/ontocast/ontocast/stategraph/node_factories.py new file mode 100644 index 0000000..d228a05 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/stategraph/node_factories.py @@ -0,0 +1,329 @@ +import asyncio +import logging + +from rdflib import DCTERMS, URIRef + +from ontocast.agent.normalize_ontology import normalize_ontology_units +from ontocast.agent.render_ontology import render_ontology_update +from ontocast.onto.content_unit import ContentUnit, OutputType, SourceUnit +from ontocast.onto.enum import Status +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.state import AgentState +from ontocast.onto.unit_states import UnitFactsState, UnitOntologyState +from ontocast.stategraph.atomic import facts_loop, ontology_loop +from ontocast.stategraph.helpers import ( + build_document_excerpt, + build_ontology_delta_graph, +) +from ontocast.toolbox import ToolBox + +logger = logging.getLogger(__name__) + + +def make_bootstrap_ontology_node(tools: ToolBox): + atomic_tools = tools.get_atomic_tools() + + async def bootstrap_ontology(state: AgentState) -> AgentState: + """Create one seed ontology for null-selection flow.""" + if not state.render_ontology or not state.current_ontology.is_null(): + state.status = Status.SUCCESS + return state + if not state.content_units: + state.status = Status.SUCCESS + return state + + excerpt = build_document_excerpt(state).strip() + if not excerpt: + logger.warning( + "Skipping ontology bootstrap: no usable excerpt was produced from content units." + ) + state.status = Status.SUCCESS + return state + + bootstrap_unit = SourceUnit( + text=excerpt, + index=0, + doc_iri=URIRef(state.doc_iri), + type=OutputType.ONTOLOGIES, + ) + bootstrap_state = UnitOntologyState( + content_unit=bootstrap_unit, + ontology_snapshot=Ontology(), + ontology_user_instruction=state.ontology_user_instruction, + budget_tracker=state.budget_tracker, + max_visits_per_node=tools.config.server.max_visits_per_node, + current_domain=state.current_domain, + ontology_max_triples=tools.config.server.ontology_max_triples, + ) + result = await ontology_loop(bootstrap_state, atomic_tools) + if result.status == Status.SUCCESS and not result.current_ontology.is_null(): + state.current_ontology = result.current_ontology + logger.info( + f"Bootstrapped ontology anchor: {state.current_ontology.iri} " + f"({len(state.current_ontology.graph)} triples)" + ) + else: + logger.warning( + "Ontology bootstrap did not yield a usable seed ontology; " + "continuing with fallback normalization behavior." + ) + state.status = Status.SUCCESS + return state + + return bootstrap_ontology + + +def make_render_ontology_node(tools: ToolBox): + atomic_tools = tools.get_atomic_tools() + + async def render_ontology_updates(state: AgentState) -> AgentState: + if not state.content_units: + state.ontology_units = [] + state.status = Status.SUCCESS + return state + + worker_limit = max(1, tools.config.server.parallel_workers) + semaphore = asyncio.Semaphore(worker_limit) + + async def process_unit(unit_index: int) -> tuple[int, UnitOntologyState]: + async with semaphore: + base_state = state.model_copy(deep=True) + ontology_state = UnitOntologyState( + content_unit=state.content_units[unit_index], + ontology_snapshot=state.current_ontology, + ontology_user_instruction=state.ontology_user_instruction, + budget_tracker=base_state.budget_tracker, + max_visits_per_node=tools.config.server.max_visits_per_node, + current_domain=state.current_domain, + ontology_max_triples=tools.config.server.ontology_max_triples, + ) + result = await ontology_loop(ontology_state, atomic_tools) + return unit_index, result + + tasks = [process_unit(i) for i, _ in enumerate(state.content_units)] + raw_results = await asyncio.gather(*tasks) + ordered_results = sorted(raw_results, key=lambda item: item[0]) + + ontology_units: list[ContentUnit] = [] + failed_without_output_count = 0 + salvaged_failed_count = 0 + for _, result in ordered_results: + has_output = bool(result.all_updates) or ( + result.current_ontology.hash != result.ontology_snapshot.hash + ) + if not has_output: + failed_without_output_count += 1 + continue + + content_unit = result.content_unit + delta_graph = build_ontology_delta_graph(result) + ontology_units.append( + ContentUnit( + text=content_unit.text, + index=content_unit.index, + doc_iri=content_unit.doc_iri, + graph=delta_graph, + type=OutputType.ONTOLOGIES, + ) + ) + if result.status != Status.SUCCESS: + salvaged_failed_count += 1 + + if failed_without_output_count: + logger.warning( + "Parallel ontology map failed without usable output for " + f"{failed_without_output_count}/{len(state.content_units)} unit(s)" + ) + if salvaged_failed_count: + logger.warning( + "Parallel ontology map salvaged output from non-converged loop(s): " + f"{salvaged_failed_count}/{len(state.content_units)} unit(s)" + ) + + state.ontology_units = ontology_units + state.status = Status.SUCCESS + return state + + return render_ontology_updates + + +def make_normalize_ontology_node(tools: ToolBox): + def normalize_ontology_updates(state: AgentState) -> AgentState: + if not state.ontology_units: + state.ontology_provenance_artifact = RDFGraph() + state.status = Status.SUCCESS + return state + + ontology, applied_updates, provenance_artifact = normalize_ontology_units( + units=state.ontology_units, + tools=tools, + base_ontology=state.current_ontology + if not state.current_ontology.is_null() + else None, + require_base=True, + ) + state.current_ontology = ontology + state.ontology_updates_applied = applied_updates + state.ontology_provenance_artifact = provenance_artifact + state.status = Status.SUCCESS + return state + + return normalize_ontology_updates + + +def make_consolidate_ontology_node(tools: ToolBox): + atomic_tools = tools.get_atomic_tools() + + async def consolidate_ontology(state: AgentState) -> AgentState: + """Optional post-normalization ontology consolidation pass.""" + if not tools.config.server.enable_ontology_consolidation: + logger.info( + "Skipping ontology consolidation: enable_ontology_consolidation is false" + ) + state.status = Status.SUCCESS + return state + if not state.render_ontology or state.current_ontology.is_null(): + logger.info( + "Skipping ontology consolidation: no rendered ontology snapshot available" + ) + state.status = Status.SUCCESS + return state + + excerpt = build_document_excerpt(state).strip() + if not excerpt: + logger.info( + "Skipping ontology consolidation: no usable document excerpt was produced" + ) + state.status = Status.SUCCESS + return state + + consolidation_unit = SourceUnit( + text=excerpt, + index=0, + doc_iri=state.doc_iri, + type=OutputType.ONTOLOGIES, + ) + consolidation_instruction = ( + "Consolidation pass: keep ontology IRI, ontology_id, and prefix unchanged. " + "Harmonize duplicated or semantically overlapping classes/properties, " + "normalize naming consistency, and improve hierarchy coherence." + ) + ontology_user_instruction = ( + f"{state.ontology_user_instruction}\n\n{consolidation_instruction}".strip() + ) + consolidation_state = UnitOntologyState( + content_unit=consolidation_unit, + ontology_snapshot=state.current_ontology, + ontology_user_instruction=ontology_user_instruction, + budget_tracker=state.budget_tracker, + max_visits_per_node=1, + current_domain=state.current_domain, + ontology_max_triples=tools.config.server.ontology_max_triples, + ) + result = await render_ontology_update(consolidation_state, atomic_tools) + if result.status == Status.SUCCESS and not result.current_ontology.is_null(): + state.current_ontology = result.current_ontology + state.ontology_updates_applied.extend(result.ontology_updates_applied) + logger.info( + f"Ontology consolidation applied {len(result.ontology_updates_applied)} " + "update operation(s)." + ) + else: + logger.warning( + "Ontology consolidation was enabled but no update was applied." + ) + state.status = Status.SUCCESS + return state + + return consolidate_ontology + + +def make_render_facts_node(tools: ToolBox): + atomic_tools = tools.get_atomic_tools() + + async def render_facts(state: AgentState) -> AgentState: + if not state.content_units: + state.parallel_facts_units = [] + state.status = Status.SUCCESS + return state + + worker_limit = max(1, tools.config.server.parallel_workers) + semaphore = asyncio.Semaphore(worker_limit) + + async def process_unit(unit_index: int) -> tuple[int, UnitFactsState]: + async with semaphore: + base_state = state.model_copy(deep=True) + facts_state = UnitFactsState( + content_unit=state.content_units[unit_index], + ontology_snapshot=state.current_ontology, + facts_user_instruction=state.facts_user_instruction, + budget_tracker=base_state.budget_tracker, + max_visits_per_node=tools.config.server.max_visits_per_node, + ) + result = await facts_loop(facts_state, atomic_tools) + return unit_index, result + + tasks = [process_unit(i) for i, _ in enumerate(state.content_units)] + raw_results = await asyncio.gather(*tasks) + ordered_results = sorted(raw_results, key=lambda item: item[0]) + + facts_units: list[ContentUnit] = [] + failed_without_output_count = 0 + salvaged_failed_count = 0 + for _, result in ordered_results: + has_output = len(result.content_unit.graph) > 0 + if not has_output: + failed_without_output_count += 1 + continue + + facts_units.append(result.content_unit) + if result.status != Status.SUCCESS: + salvaged_failed_count += 1 + + if failed_without_output_count: + logger.warning( + "Parallel facts map failed without usable output for " + f"{failed_without_output_count}/{len(state.content_units)} unit(s)" + ) + if salvaged_failed_count: + logger.warning( + "Parallel facts map salvaged output from non-converged loop(s): " + f"{salvaged_failed_count}/{len(state.content_units)} unit(s)" + ) + + state.parallel_facts_units = facts_units + state.status = Status.SUCCESS + return state + + return render_facts + + +def make_merge_facts_node(tools: ToolBox): + def merge_facts(state: AgentState) -> AgentState: + if not state.parallel_facts_units: + state.aggregated_facts = RDFGraph() + state.status = Status.SUCCESS + return state + + for unit in state.parallel_facts_units: + unit.sanitize() + state.aggregated_facts = tools.aggregator.aggregate_graphs( + units=state.parallel_facts_units, + ontology_graph=state.current_ontology.graph + if not state.current_ontology.is_null() + else None, + ) + if len(state.aggregated_facts) == 0: + logger.warning( + "Facts aggregation produced an empty graph from " + f"{len(state.parallel_facts_units)} successful unit(s)." + ) + if state.source_url and state.doc_namespace: + state.aggregated_facts.add( + (URIRef(state.doc_namespace), DCTERMS.source, URIRef(state.source_url)) + ) + state.status = Status.SUCCESS + return state + + return merge_facts diff --git a/ontology_platform/vendored/ontocast/ontocast/stategraph/routing.py b/ontology_platform/vendored/ontocast/ontocast/stategraph/routing.py new file mode 100644 index 0000000..1ca0407 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/stategraph/routing.py @@ -0,0 +1,18 @@ +from ontocast.onto.enum import WorkflowNode +from ontocast.onto.state import AgentState + + +def route_after_ontology_selection(state: AgentState) -> str: + """Route after ontology selection.""" + if not state.render_ontology: + return WorkflowNode.RENDER_FACTS + if state.current_ontology.is_null(): + return WorkflowNode.BOOTSTRAP_ONTOLOGY + return WorkflowNode.RENDER_ONTOLOGY_UPDATE + + +def route_after_ontology_consolidation(state: AgentState) -> str: + """Route after ontology stage: facts map if needed, else serialize.""" + if state.render_facts: + return WorkflowNode.RENDER_FACTS + return WorkflowNode.SERIALIZE diff --git a/ontology_platform/vendored/ontocast/ontocast/stategraph/util.py b/ontology_platform/vendored/ontocast/ontocast/stategraph/util.py new file mode 100644 index 0000000..3f5e5b4 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/stategraph/util.py @@ -0,0 +1,75 @@ +import asyncio +import logging +from functools import wraps +from typing import Callable + +from ontocast.onto.enum import Status, WorkflowNode +from ontocast.onto.state import AgentState + +logger = logging.getLogger(__name__) + + +def count_visits_conditional_success( + state: AgentState, current_node: WorkflowNode +) -> AgentState: + """Track node visits and handle success/failure conditions. + + This function increments the visit counter for a node and manages the state + based on success/failure conditions and maximum visit limits. + + Args: + state: The current agent state. + current_node: The node being visited. + + Returns: + AgentState: Updated agent state after processing visit conditions. + """ + state.node_visits[current_node] += 1 + if state.status == Status.SUCCESS: + logger.info(f"For {current_node}: status is SUCCESS, proceeding to next node") + state.clear_failure() + elif state.node_visits[current_node] >= state.max_visits: + logger.info(f"For {current_node}: maximum visits exceeded") + # Don't set failure stage since we're continuing with SUCCESS status + # Just log the reason and continue + state.failure_reason = f"Maximum visits exceeded for {current_node}" + state.status = Status.SUCCESS + return state + + +def wrap_with(func, node_name, post_func) -> tuple[WorkflowNode, Callable]: + """Add a visit counter to a function. + + This function wraps a given function with logging and post-processing + functionality, typically used for workflow node execution. + + Args: + func: The function to wrap (can be sync or async). + node_name: The name of the node. + post_func: Function to execute after the main function. + + Returns: + tuple[WorkflowNode, Callable]: A tuple containing the node name and + the wrapped function. + """ + # Check if the function is async + if asyncio.iscoroutinefunction(func): + + @wraps(func) + async def async_wrapper(state: AgentState): + logger.info(f"Starting to execute {node_name}") + state = await func(state) + state = post_func(state, node_name) + return state + + return node_name, async_wrapper + else: + + @wraps(func) + def sync_wrapper(state: AgentState): + logger.info(f"Starting to execute {node_name}") + state = func(state) + state = post_func(state, node_name) + return state + + return node_name, sync_wrapper diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/__init__.py b/ontology_platform/vendored/ontocast/ontocast/tool/__init__.py new file mode 100644 index 0000000..457c453 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/__init__.py @@ -0,0 +1,52 @@ +"""Tool package for OntoCast. + +This package provides a collection of tools that support the OntoCast workflow, +including document processing, ontology management, triple store operations, +and LLM interactions. + +The package includes: +- LLMTool: Language model interaction and prompting +- OntologyManager: Ontology loading and management +- TripleStoreManager: Abstract interface for triple store operations +- FusekiTripleStoreManager: Fuseki-specific triple store implementation (preferred) +- Neo4jTripleStoreManager: Neo4j-specific triple store implementation +- FilesystemTripleStoreManager: Filesystem-based triple store implementation +- ConverterTool: Document format conversion utilities +- ChunkerTool: Text chunking and segmentation + +All tools inherit from the base Tool class and provide standardized +interfaces for integration into the OntoCast workflow. + +Example: + >>> from ontocast.tool import LLMTool, OntologyManager + >>> llm = LLMTool.create(provider="openai", model="gpt-4") + >>> om = OntologyManager() +""" + +from ontocast.tool.chunk.chunker import ChunkerTool + +from .atomic import AtomicToolBox, SearchHit +from .converter import ConverterTool +from .llm import LLMTool +from .onto import Tool +from .ontology_manager import OntologyManager +from .triple_manager import ( + FilesystemTripleStoreManager, + FusekiTripleStoreManager, + Neo4jTripleStoreManager, + TripleStoreManager, +) + +__all__ = [ + "LLMTool", + "OntologyManager", + "TripleStoreManager", + "FusekiTripleStoreManager", + "Neo4jTripleStoreManager", + "FilesystemTripleStoreManager", + "ConverterTool", + "ChunkerTool", + "Tool", + "AtomicToolBox", + "SearchHit", +] diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/agg/README.md b/ontology_platform/vendored/ontocast/ontocast/tool/agg/README.md new file mode 100644 index 0000000..c655da7 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/agg/README.md @@ -0,0 +1,44 @@ +## Naming & Normalization Conventions + +We follow standard RDF / Semantic Web naming conventions: + +- Classes (entities / types) use PascalCase + +```ttl +ex:Case +ex:JudicialDecision +``` + + +- Properties (predicates) use lower camelCase + +```ttl +ex:hasDecision +ex:datePublished +``` + + +- Individuals with natural names use PascalCase + +```ttl +ex:FrenchCourtOfCassation +``` + +Individuals with structured or external identifiers preserve their structure +(underscores and digits are allowed and encouraged) + +```shell +ex:Case_2023_456 +ex:Decision_2021_09_15 +``` + + +## Notes + +- Underscores are avoided in ontology terms (classes, properties). +- Underscores are acceptable for instances derived from external IDs. +- Prefer stable, readable IRIs; store human-facing identifiers explicitly when needed: + +```ttl +ex:Case_2023_456 ex:caseNumber "2023/456" . +``` diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/agg/__init__.py b/ontology_platform/vendored/ontocast/ontocast/tool/agg/__init__.py new file mode 100644 index 0000000..dade7d6 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/agg/__init__.py @@ -0,0 +1,16 @@ +"""Embedding-based aggregation pipeline for RDF content unit graphs.""" + +from .aggregate import ( + EmbeddingBasedAggregator, + aggregate_chunk_graphs, + aggregate_content_unit_graphs, +) +from .uri_builder import EntityRole, URIBuilder + +__all__ = [ + "EmbeddingBasedAggregator", + "EntityRole", + "URIBuilder", + "aggregate_content_unit_graphs", + "aggregate_chunk_graphs", +] diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/agg/aggregate.py b/ontology_platform/vendored/ontocast/ontocast/tool/agg/aggregate.py new file mode 100644 index 0000000..482143c --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/agg/aggregate.py @@ -0,0 +1,833 @@ +"""Embedding-based RDF graph aggregator. + +This module provides the main aggregator class that orchestrates entity +disambiguation using embedding-based clustering. + +Pipeline: +1. Collect entities from all content units +2. Normalize entities: e -> r(e) (string representation with semantic context) +3. Generate embedding-based identity candidates +4. Validate candidate merges with symbolic identity checks +5. Select canonical identity per validated cluster +6. Assign final URIs from canonical identity + document namespace policy +7. Rewrite graphs: apply mapping e -> e' to all triples +""" + +import logging +from difflib import SequenceMatcher +from enum import StrEnum +from itertools import combinations +from typing import cast + +import numpy as np +from rdflib import URIRef +from rdflib.namespace import OWL, RDF, RDFS, XSD + +from ontocast.onto.constants import DEFAULT_IRI, PROV, SCHEMA +from ontocast.onto.content_unit import ContentUnit, OutputType +from ontocast.onto.rdfgraph import RDFGraph + +from .clustering import ClusterRepresentativeSelector, EntityClusterer +from .normalizer import EntityNormalizer, EntityRepresentation +from .rewriter import GraphRewriter +from .uri_builder import EntityRole, URIBuilder + +logger = logging.getLogger(__name__) + + +class EntityClassification(StrEnum): + """Classification of entities during aggregation.""" + + FACT = "fact" + KNOWN_ONTOLOGY = "known_ontology" + TENTATIVE_ONTOLOGY = "tentative_ontology" + + +_STANDARD_NAMESPACES = ( + str(RDF), + str(RDFS), + str(OWL), + str(XSD), + str(SCHEMA), + str(PROV), +) + + +class EmbeddingBasedAggregator: + """Main aggregator using embedding-based entity disambiguation. + + Pipeline stages: + 1. Entity normalisation (with semantic context) + 2. Parallel embedding + 3. Similarity-based clustering + 4. Representative selection (prefer ontology, then simplicity) + 5. URI normalisation (PascalCase/camelCase under DEFAULT_IRI) + 6. Graph rewriting + + ContentUnit types are handled as follows: + - ``facts``: entities under ``base_iri`` are normalised. + - ``ontology``: all other entities are considered ontology entities and preserved. + """ + + def __init__( + self, + embedding_model: str = "paraphrase-multilingual-MiniLM-L12-v2", + similarity_threshold: float = 0.80, + candidate_similarity_threshold: float = 0.70, + add_sameas_links: bool = True, + base_iri: str = DEFAULT_IRI, + ): + """Initialise the embedding-based aggregator. + + Args: + embedding_model: Name of sentence transformer model. + similarity_threshold: Cosine similarity threshold for clustering (0-1). + candidate_similarity_threshold: Lower cosine threshold used to + generate permissive merge candidates before symbolic validation. + add_sameas_links: Whether to add owl:sameAs for merged entities. + base_iri: Base IRI for fact entity URIs (default: DEFAULT_IRI). + Entities under this namespace are facts; everything else is + treated as an ontology entity and left unchanged. + """ + self.base_iri = base_iri + self.candidate_similarity_threshold = candidate_similarity_threshold + + # Pipeline components + self.normalizer = EntityNormalizer(facts_iri=self.base_iri) + self.clusterer = EntityClusterer( + embedding_model=embedding_model, + similarity_threshold=similarity_threshold, + ) + self.selector = ClusterRepresentativeSelector() + self.uri_builder = URIBuilder(base_iri=self.base_iri) + self.rewriter = GraphRewriter( + add_sameas_links=add_sameas_links, + blocked_sameas_namespaces=(self.base_iri,), + ) + + @staticmethod + def _entity_in_namespace(entity: URIRef, namespace: URIRef | str | None) -> bool: + """Return True when *entity* is under the provided namespace.""" + if namespace is None: + return False + entity_str = str(entity) + namespace_str = str(namespace) + + # Accept exact prefix namespaces (e.g. ``.../facts`` used with Turtle + # ``@prefix cd: <.../facts>`` → ``.../factsConviction1``) and slash/hash + # namespace variants. + if entity_str.startswith(namespace_str): + return True + + slash_variant = namespace_str.rstrip("/") + "/" + hash_variant = namespace_str.rstrip("#") + "#" + return entity_str.startswith(slash_variant) or entity_str.startswith( + hash_variant + ) + + def _is_fact_entity_in_unit(self, entity: URIRef, unit: ContentUnit) -> bool: + """Classify whether an entity should be treated as a fact in this unit. + + Facts are entities in either: + - the configured base facts namespace (``base_iri``), or + - the unit document namespace (``unit.doc_iri``). + """ + return self._entity_in_namespace( + entity, self.base_iri + ) or self._entity_in_namespace(entity, unit.doc_iri) + + @staticmethod + def _is_standard_ontology_entity(entity: URIRef) -> bool: + """Return True for entities from built-in standard RDF vocabularies.""" + entity_str = str(entity) + return any(entity_str.startswith(prefix) for prefix in _STANDARD_NAMESPACES) + + def _build_known_ontology_entities( + self, ontology_graph: RDFGraph | None + ) -> set[URIRef]: + """Build a set of known ontology entities from ontology and std vocabularies.""" + known_entities: set[URIRef] = set() + + if ontology_graph is not None: + for s, p, o in ontology_graph: + if isinstance(s, URIRef): + known_entities.add(s) + if isinstance(p, URIRef): + known_entities.add(p) + if isinstance(o, URIRef): + known_entities.add(o) + + return known_entities + + @staticmethod + def _tokenize(text: str) -> set[str]: + return {token for token in text.split() if len(token) > 2} + + @staticmethod + def _role_key(representation: EntityRepresentation) -> str: + role = ( + representation.role + if representation.role is not None + else EntityRole.INSTANCE + ) + return str(role) + + @staticmethod + def _jaccard(left: set[str], right: set[str]) -> float: + if not left and not right: + return 1.0 + union = left | right + return len(left & right) / len(union) + + def _are_roles_compatible( + self, + left: URIRef, + right: URIRef, + representations: dict[URIRef, EntityRepresentation], + ) -> bool: + left_rep = representations.get(left) + right_rep = representations.get(right) + if left_rep is None or right_rep is None: + return False + return self._role_key(left_rep) == self._role_key(right_rep) + + def _are_types_compatible( + self, + left: URIRef, + right: URIRef, + representations: dict[URIRef, EntityRepresentation], + ) -> bool: + left_rep = representations.get(left) + right_rep = representations.get(right) + if left_rep is None or right_rep is None: + return False + left_types = set(left_rep.types) + right_types = set(right_rep.types) + if not left_types or not right_types: + return True + return bool(left_types & right_types) + + def _are_lexical_aliases( + self, + left: URIRef, + right: URIRef, + representations: dict[URIRef, EntityRepresentation], + ) -> bool: + left_rep = representations.get(left) + right_rep = representations.get(right) + if left_rep is None or right_rep is None: + return False + if left_rep.normal_form == right_rep.normal_form: + return True + + left_label_tokens = { + self.normalizer.normalize_string(label) + for label in left_rep.labels + if label.strip() + } + right_label_tokens = { + self.normalizer.normalize_string(label) + for label in right_rep.labels + if label.strip() + } + if left_label_tokens & right_label_tokens: + return True + if left_label_tokens and right_label_tokens: + max_label_overlap = 0.0 + for left_label in left_label_tokens: + left_tokens = self._tokenize(left_label) + for right_label in right_label_tokens: + right_tokens = self._tokenize(right_label) + overlap = self._jaccard(left_tokens, right_tokens) + max_label_overlap = max(max_label_overlap, overlap) + if max_label_overlap >= 0.2: + return True + + ratio = SequenceMatcher( + None, left_rep.normal_form, right_rep.normal_form + ).ratio() + if ratio >= 0.82: + return True + + left_tokens = self._tokenize(left_rep.normal_form) + right_tokens = self._tokenize(right_rep.normal_form) + if len(left_tokens) >= 2 and len(right_tokens) >= 2: + if self._jaccard(left_tokens, right_tokens) >= 0.75: + return True + + return False + + def _can_merge_as_identity( + self, + left: URIRef, + right: URIRef, + representations: dict[URIRef, EntityRepresentation], + ) -> bool: + return ( + self._are_roles_compatible(left, right, representations) + and self._are_types_compatible(left, right, representations) + and self._are_lexical_aliases(left, right, representations) + ) + + def _cluster_entities_by_role( + self, representations: dict[URIRef, EntityRepresentation] + ) -> tuple[list[list[URIRef]], dict[URIRef, np.ndarray]]: + grouped_entities: dict[str, dict[URIRef, EntityRepresentation]] = {} + for entity, representation in representations.items(): + grouped_entities.setdefault(self._role_key(representation), {})[entity] = ( + representation + ) + + all_clusters: list[list[URIRef]] = [] + all_embeddings: dict[URIRef, np.ndarray] = {} + original_threshold = self.clusterer.similarity_threshold + self.clusterer.similarity_threshold = self.candidate_similarity_threshold + try: + for role_representations in grouped_entities.values(): + role_clusters, role_embeddings = self.clusterer.cluster_entities( + role_representations + ) + all_clusters.extend(role_clusters) + all_embeddings.update(role_embeddings) + finally: + self.clusterer.similarity_threshold = original_threshold + return all_clusters, all_embeddings + + @staticmethod + def _candidate_similarity( + left: URIRef, + right: URIRef, + embeddings: dict[URIRef, np.ndarray], + ) -> float | None: + left_embedding = embeddings.get(left) + right_embedding = embeddings.get(right) + if left_embedding is None or right_embedding is None: + return None + + denominator = float( + np.linalg.norm(left_embedding) * np.linalg.norm(right_embedding) + ) + if denominator == 0: + return None + return float(np.dot(left_embedding, right_embedding) / denominator) + + def _merge_validation_failures( + self, + left: URIRef, + right: URIRef, + representations: dict[URIRef, EntityRepresentation], + ) -> list[str]: + failures: list[str] = [] + if not self._are_roles_compatible(left, right, representations): + failures.append("role") + if not self._are_types_compatible(left, right, representations): + failures.append("type") + if not self._are_lexical_aliases(left, right, representations): + failures.append("lexical") + return failures + + def _build_identity_clusters( + self, + candidate_clusters: list[list[URIRef]], + representations: dict[URIRef, EntityRepresentation], + embeddings: dict[URIRef, np.ndarray], + ) -> tuple[ + list[list[URIRef]], list[tuple[URIRef, URIRef, float | None, tuple[str, ...]]] + ]: + validated_clusters: list[list[URIRef]] = [] + rejected_merges: list[tuple[URIRef, URIRef, float | None, tuple[str, ...]]] = [] + + for candidate_cluster in candidate_clusters: + if len(candidate_cluster) <= 1: + validated_clusters.append(candidate_cluster) + continue + + parents: dict[URIRef, URIRef] = { + entity: entity for entity in candidate_cluster + } + + def find(entity: URIRef) -> URIRef: + root = parents[entity] + if root != entity: + parents[entity] = find(root) + return parents[entity] + + def union(left: URIRef, right: URIRef) -> None: + left_root = find(left) + right_root = find(right) + if left_root == right_root: + return + if str(left_root) <= str(right_root): + parents[right_root] = left_root + else: + parents[left_root] = right_root + + for left, right in combinations(candidate_cluster, 2): + score = self._candidate_similarity(left, right, embeddings) + if score is not None and score < self.candidate_similarity_threshold: + continue + if self._can_merge_as_identity(left, right, representations): + union(left, right) + continue + rejected_merges.append( + ( + left, + right, + score, + tuple( + self._merge_validation_failures( + left, right, representations + ) + ), + ) + ) + + grouped: dict[URIRef, list[URIRef]] = {} + for entity in candidate_cluster: + grouped.setdefault(find(entity), []).append(entity) + + for group in grouped.values(): + sorted_group = cast(list[URIRef], sorted(group, key=str)) + validated_clusters.append(sorted_group) + + return validated_clusters, rejected_merges + + def _select_ontology_anchor_candidates( + self, + tentative_entities: list[URIRef], + tentative_representations: dict[URIRef, EntityRepresentation], + tentative_doc_iris: dict[URIRef, URIRef], + ontology_graph: RDFGraph | None, + known_ontology_entities: set[URIRef], + ) -> dict[URIRef, URIRef]: + """Pick ontology anchors and preserve the triggering document IRI.""" + if ( + ontology_graph is None + or not tentative_entities + or not known_ontology_entities + ): + return {} + + ontology_entities = [ + entity + for entity in known_ontology_entities + if not self._is_standard_ontology_entity(entity) + ] + if not ontology_entities: + return {} + + ontology_graphs = {entity: ontology_graph for entity in ontology_entities} + ontology_representations = self.normalizer.create_representations_batch( + ontology_entities, ontology_graphs + ) + + token_index: dict[str, set[URIRef]] = {} + for entity, representation in ontology_representations.items(): + for token in self._tokenize(representation.representation): + token_index.setdefault(token, set()).add(entity) + + selected: dict[URIRef, URIRef] = {} + for tentative_entity in tentative_entities: + tentative_representation = tentative_representations.get(tentative_entity) + if tentative_representation is None: + continue + tentative_doc_iri = tentative_doc_iris.get(tentative_entity) + if tentative_doc_iri is None: + continue + tentative_tokens = self._tokenize(tentative_representation.representation) + if not tentative_tokens: + continue + + candidate_pool: set[URIRef] = set() + for token in tentative_tokens: + candidate_pool.update(token_index.get(token, set())) + + if not candidate_pool: + continue + + scored: list[tuple[int, URIRef]] = [] + for candidate in candidate_pool: + candidate_representation = ontology_representations.get(candidate) + if candidate_representation is None: + continue + candidate_tokens = self._tokenize( + candidate_representation.representation + ) + overlap = len(tentative_tokens & candidate_tokens) + if overlap >= 2: + scored.append((overlap, candidate)) + + scored.sort(key=lambda item: (-item[0], str(item[1]))) + for _, candidate in scored[:3]: + selected.setdefault(candidate, tentative_doc_iri) + + return selected + + def _classify_entity_for_unit( + self, + entity: URIRef, + unit: ContentUnit, + known_ontology_entities: set[URIRef], + ) -> EntityClassification: + """Classify an entity as fact, known ontology, or tentative ontology.""" + if unit.type == OutputType.ONTOLOGIES: + return EntityClassification.KNOWN_ONTOLOGY + + if self._is_fact_entity_in_unit(entity, unit): + return EntityClassification.FACT + + if entity in known_ontology_entities or self._is_standard_ontology_entity( + entity + ): + return EntityClassification.KNOWN_ONTOLOGY + + return EntityClassification.TENTATIVE_ONTOLOGY + + @staticmethod + def _classification_priority(classification: EntityClassification) -> int: + """Return priority for multi-unit classification merging.""" + if classification == EntityClassification.KNOWN_ONTOLOGY: + return 3 + if classification == EntityClassification.TENTATIVE_ONTOLOGY: + return 2 + return 1 + + @staticmethod + def _merge_into_context_graph(target: RDFGraph, source: RDFGraph) -> None: + """Merge source triples/namespaces into a per-entity context graph.""" + target += source + + def _register_entity( + self, + *, + entity: URIRef, + unit: ContentUnit, + known_entities: set[URIRef], + entities: set[URIRef], + source_entities: set[URIRef], + entity_graphs: dict[URIRef, RDFGraph], + entity_doc_iris: dict[URIRef, URIRef], + entity_classification: dict[URIRef, EntityClassification], + ) -> None: + """Register one URI entity with merged context and stable classification.""" + entities.add(entity) + source_entities.add(entity) + if entity not in entity_graphs: + entity_graphs[entity] = unit.graph.copy() + else: + self._merge_into_context_graph(entity_graphs[entity], unit.graph) + entity_doc_iris.setdefault(entity, unit.doc_iri) + current = entity_classification.get(entity, EntityClassification.FACT) + candidate = self._classify_entity_for_unit(entity, unit, known_entities) + entity_classification[entity] = ( + candidate + if self._classification_priority(candidate) + >= self._classification_priority(current) + else current + ) + + def _collect_all_entities( + self, + units: list[ContentUnit], + known_ontology_entities: set[URIRef] | None = None, + ) -> tuple[ + list[URIRef], + set[URIRef], + dict[URIRef, RDFGraph], + dict[URIRef, URIRef], + dict[URIRef, EntityClassification], + ]: + """Collect all entities from all content unit graphs. + + Each entity is associated with the graph it was found in and the + ``doc_iri`` of the :class:`ContentUnit` that produced it. When an + entity appears in several units the *last-seen* ``doc_iri`` wins (in + practice most pipelines aggregate chunks of the same document, so all + ``doc_iri`` values are identical). + + Args: + units: List of content units to aggregate. + + Returns: + Tuple of ( + entities, + entity_to_graph, + entity_to_doc_iri, + entity_to_is_ontology, + ). + """ + entities: set[URIRef] = set() + source_entities: set[URIRef] = set() + entity_graphs: dict[URIRef, RDFGraph] = {} + entity_doc_iris: dict[URIRef, URIRef] = {} + entity_classification: dict[URIRef, EntityClassification] = {} + known_entities = known_ontology_entities or set() + + for unit in units: + if unit.graph is None: + continue + # Keep collection in the same URI space that rewrite/merge consumes + # (unit.graph). Using graph_absolute here causes mapping keys to miss + # during rewrite, because unit.graph still contains the original terms. + for s, p, o in unit.graph: + if isinstance(s, URIRef): + self._register_entity( + entity=s, + unit=unit, + known_entities=known_entities, + entities=entities, + source_entities=source_entities, + entity_graphs=entity_graphs, + entity_doc_iris=entity_doc_iris, + entity_classification=entity_classification, + ) + if isinstance(p, URIRef): + self._register_entity( + entity=p, + unit=unit, + known_entities=known_entities, + entities=entities, + source_entities=source_entities, + entity_graphs=entity_graphs, + entity_doc_iris=entity_doc_iris, + entity_classification=entity_classification, + ) + if isinstance(o, URIRef): + self._register_entity( + entity=o, + unit=unit, + known_entities=known_entities, + entities=entities, + source_entities=source_entities, + entity_graphs=entity_graphs, + entity_doc_iris=entity_doc_iris, + entity_classification=entity_classification, + ) + + return ( + list(entities), + source_entities, + entity_graphs, + entity_doc_iris, + entity_classification, + ) + + def aggregate_graphs( + self, + units: list[ContentUnit], + ontology_graph: RDFGraph | None = None, + ) -> RDFGraph: + """Aggregate multiple content unit graphs with embedding-based disambiguation. + + Args: + units: List of ContentUnits to aggregate. + ontology_graph: Optional selected ontology graph used to distinguish + known ontology entities from tentative ontology-like aliases. + + Returns: + Merged RDF graph with provenance annotations. + """ + logger.info(f"Starting aggregation with metadata for {len(units)} units") + + if not units: + return RDFGraph() + + # Steps 1-3: Collect, normalise, candidate clustering + known_ontology_entities = self._build_known_ontology_entities(ontology_graph) + ( + entities, + source_entities, + entity_graphs, + entity_doc_iris, + entity_classification, + ) = self._collect_all_entities(units, known_ontology_entities) + representations = self.normalizer.create_representations_batch( + entities, entity_graphs + ) + tentative_entities = [ + entity + for entity, classification in entity_classification.items() + if classification == EntityClassification.TENTATIVE_ONTOLOGY + ] + anchor_candidates = self._select_ontology_anchor_candidates( + tentative_entities=tentative_entities, + tentative_representations=representations, + tentative_doc_iris=entity_doc_iris, + ontology_graph=ontology_graph, + known_ontology_entities=known_ontology_entities, + ) + if anchor_candidates and ontology_graph is not None: + for ontology_entity, anchor_doc_iri in anchor_candidates.items(): + if ontology_entity in entity_graphs: + continue + entities.append(ontology_entity) + entity_graphs[ontology_entity] = ontology_graph + entity_doc_iris[ontology_entity] = anchor_doc_iri + entity_classification[ontology_entity] = ( + EntityClassification.KNOWN_ONTOLOGY + ) + representations[ontology_entity] = ( + self.normalizer.create_representation( + ontology_entity, ontology_graph + ) + ) + entity_is_known_ontology = { + entity: classification == EntityClassification.KNOWN_ONTOLOGY + for entity, classification in entity_classification.items() + } + + # Representative selection should prefer known ontology entities only. + for entity, is_known_ontology in entity_is_known_ontology.items(): + representation = representations.get(entity) + if representation is not None: + representation.is_ontology_entity = is_known_ontology + candidate_clusters, embeddings = self._cluster_entities_by_role(representations) + clusters, rejected_merges = self._build_identity_clusters( + candidate_clusters=candidate_clusters, + representations=representations, + embeddings=embeddings, + ) + if rejected_merges: + logger.info( + "Rejected %d candidate merges after symbolic validation", + len(rejected_merges), + ) + for left, right, score, failed_checks in rejected_merges: + logger.debug( + "Rejected candidate merge: %s <-> %s (score=%s, failed=%s)", + left, + right, + f"{score:.3f}" if score is not None else "n/a", + ",".join(failed_checks) if failed_checks else "unknown", + ) + + # Step 4: Canonical identity mapping (no URI policy yet) + identity_mapping = self.selector.create_mapping(clusters, representations) + + # Keep known ontology entities stable. Tentative ontology-like entities are: + # - mapped to known ontology representatives when present in a mixed cluster + # - preserved as-is when only tentative entities are present + ontology_sameas_links: dict[URIRef, set[URIRef]] = {} + suppress_sameas_origins: set[URIRef] = set() + for cluster in clusters: + known_ontology_entities_in_cluster = [ + entity + for entity in cluster + if entity_classification.get(entity) + == EntityClassification.KNOWN_ONTOLOGY + ] + tentative_entities_in_cluster = [ + entity + for entity in cluster + if entity_classification.get(entity) + == EntityClassification.TENTATIVE_ONTOLOGY + ] + fact_entities_in_cluster = [ + entity + for entity in cluster + if entity_classification.get(entity) == EntityClassification.FACT + ] + + for entity in known_ontology_entities_in_cluster: + identity_mapping[entity] = entity + + if known_ontology_entities_in_cluster: + canonical_known_ontology = self.selector.select_representative( + known_ontology_entities_in_cluster, + representations, + ) + for tentative_entity in tentative_entities_in_cluster: + if self._can_merge_as_identity( + tentative_entity, + canonical_known_ontology, + representations, + ): + identity_mapping[tentative_entity] = canonical_known_ontology + suppress_sameas_origins.add(tentative_entity) + else: + identity_mapping[tentative_entity] = tentative_entity + for fact_entity in fact_entities_in_cluster: + identity_mapping[fact_entity] = fact_entity + + elif tentative_entities_in_cluster: + for tentative_entity in tentative_entities_in_cluster: + identity_mapping[tentative_entity] = tentative_entity + + if len(known_ontology_entities_in_cluster) > 1: + canonical = self.selector.select_representative( + known_ontology_entities_in_cluster, + representations, + ) + aliases = { + entity + for entity in known_ontology_entities_in_cluster + if entity != canonical + and entity in source_entities + and canonical in source_entities + and self._can_merge_as_identity(entity, canonical, representations) + } + if aliases: + ontology_sameas_links.setdefault(canonical, set()).update(aliases) + + # Step 5: URI assignment from canonical identity + namespace policy + non_fact_entities = { + entity + for entity, classification in entity_classification.items() + if classification != EntityClassification.FACT + } + final_mapping = self.uri_builder.create_entity_uri_mapping( + identity_mapping=identity_mapping, + representations=representations, + entity_doc_iris=entity_doc_iris, + entity_is_ontology={ + entity: entity in non_fact_entities for entity in representations + }, + ) + final_mapping = { + entity: mapped + for entity, mapped in final_mapping.items() + if entity in source_entities + } + + # Step 7: Rewrite and merge with provenance + active_units = [u for u in units if u.graph is not None] + merged_graph = self.rewriter.merge_graphs_with_provenance( + active_units, + final_mapping, + extra_sameas_links=ontology_sameas_links, + suppress_sameas_origins=suppress_sameas_origins, + ) + + logger.info("Aggregation with metadata complete") + return merged_graph + + +# Convenience function for backward compatibility +def aggregate_content_unit_graphs( + units: list[ContentUnit], + similarity_threshold: float = 0.80, +) -> RDFGraph: + """Convenience function to aggregate content unit graphs. + + Args: + units: List of content units to aggregate. + similarity_threshold: Cosine similarity threshold for clustering. + + Returns: + Aggregated RDF graph. + """ + aggregator = EmbeddingBasedAggregator( + similarity_threshold=similarity_threshold, + ) + return aggregator.aggregate_graphs(units) + + +def aggregate_chunk_graphs( + units: list[ContentUnit], + similarity_threshold: float = 0.80, +) -> RDFGraph: + """Backward-compatible alias for :func:`aggregate_content_unit_graphs`.""" + return aggregate_content_unit_graphs( + units=units, + similarity_threshold=similarity_threshold, + ) diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/agg/clustering.py b/ontology_platform/vendored/ontocast/ontocast/tool/agg/clustering.py new file mode 100644 index 0000000..5a1068f --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/agg/clustering.py @@ -0,0 +1,291 @@ +"""Embedding-based entity clustering for disambiguation. + +This module handles the embedding and clustering of entity representations +to identify groups of similar entities. +""" + +import importlib +import logging +from typing import Any + +import numpy as np +from rdflib import URIRef +from sklearn.cluster import DBSCAN +from sklearn.metrics.pairwise import cosine_similarity + +from .normalizer import EntityRepresentation + +logger = logging.getLogger(__name__) + + +class EntityClusterer: + """Clusters entities based on embedding similarity. + + This class handles the embedding of entity representations and + grouping them into clusters of similar entities. + """ + + def __init__( + self, + embedding_model: str = "paraphrase-multilingual-MiniLM-L12-v2", + similarity_threshold: float = 0.80, + min_cluster_size: int = 1, + ): + """Initialize the entity clusterer. + + Args: + embedding_model: Name of the sentence transformer model to use + similarity_threshold: Minimum cosine similarity for grouping (0-1) + min_cluster_size: Minimum size for a cluster (1 allows singletons) + """ + self.embedding_model = embedding_model + self.similarity_threshold = similarity_threshold + self.min_cluster_size = min_cluster_size + self._embedder: Any | None = None + + @property + def embedder(self) -> Any: + if self._embedder is None: + try: + st = importlib.import_module("sentence_transformers") + except ImportError as e: + raise ImportError( + "Entity clustering requires the sentence-transformers package. " + "Install it with: uv add sentence-transformers" + ) from e + + self._embedder = st.SentenceTransformer(self.embedding_model) + return self._embedder + + def embed_representations( + self, representations: dict[URIRef, EntityRepresentation] + ) -> dict[URIRef, np.ndarray]: + """Embed all entity representations in parallel. + + This is much faster than embedding one at a time. + + Args: + representations: Dictionary mapping entities to their representations + + Returns: + Dictionary mapping entities to their embedding vectors + """ + if not representations: + return {} + + # Prepare batch of texts + entities = list(representations.keys()) + texts = [representations[e].representation for e in entities] + + logger.info(f"Embedding {len(texts)} entity representations in parallel...") + + # Batch embedding (much faster!) + embeddings = self.embedder.encode( + texts, convert_to_numpy=True, show_progress_bar=len(texts) > 100 + ) + + # Create mapping + entity_embeddings = { + entity: embedding for entity, embedding in zip(entities, embeddings) + } + + logger.info(f"Embedded {len(entity_embeddings)} entities") + return entity_embeddings + + def cluster_by_similarity( + self, + embeddings: dict[URIRef, np.ndarray], + representations: dict[URIRef, EntityRepresentation], + ) -> list[list[URIRef]]: + """Cluster entities based on embedding similarity. + + Args: + embeddings: Dictionary mapping entities to embeddings + representations: Dictionary mapping entities to their representations + + Returns: + List of clusters (each cluster is a list of entity URIs) + """ + if not embeddings: + return [] + + entities = list(embeddings.keys()) + embedding_matrix = np.array([embeddings[e] for e in entities]) + + logger.info(f"Clustering {len(entities)} entities...") + + # Compute pairwise cosine similarity + similarity_matrix = cosine_similarity(embedding_matrix) + + # Convert similarity to distance for DBSCAN (must be non-negative) + # DBSCAN uses epsilon as maximum distance, so we use 1 - similarity + distance_matrix = np.maximum(0.0, 1.0 - similarity_matrix) + + # Use DBSCAN for clustering + # eps is the maximum distance between two samples for them to be in same cluster + # We want high similarity (low distance), so eps = 1 - threshold + eps = 1 - self.similarity_threshold + + clusterer = DBSCAN( + eps=eps, min_samples=self.min_cluster_size, metric="precomputed" + ) + + cluster_labels = clusterer.fit_predict(distance_matrix) + + # Group entities by cluster + clusters_dict: dict[int, list[URIRef]] = {} + for entity, label in zip(entities, cluster_labels): + if label not in clusters_dict: + clusters_dict[label] = [] + clusters_dict[label].append(entity) + + # Convert to list of clusters + clusters = list(clusters_dict.values()) + + # Log statistics + singleton_count = sum(1 for c in clusters if len(c) == 1) + multi_count = sum(1 for c in clusters if len(c) > 1) + max_size = max(len(c) for c in clusters) if clusters else 0 + + logger.info( + f"Formed {len(clusters)} clusters: " + f"{singleton_count} singletons, " + f"{multi_count} multi-entity clusters, " + f"max cluster size: {max_size}" + ) + + return clusters + + def cluster_entities( + self, representations: dict[URIRef, EntityRepresentation] + ) -> tuple[list[list[URIRef]], dict[URIRef, np.ndarray]]: + """Complete clustering pipeline: embed and cluster. + + Args: + representations: Dictionary mapping entities to their representations + + Returns: + Tuple of (clusters, embeddings) + - clusters: List of entity groups + - embeddings: Dictionary mapping entities to their embeddings + """ + # Step 1: Embed all representations in parallel + embeddings = self.embed_representations(representations) + + # Step 2: Cluster based on similarity + clusters = self.cluster_by_similarity(embeddings, representations) + + return clusters, embeddings + + +class ClusterRepresentativeSelector: + """Selects the best representative entity from a cluster. + + The selection criteria are: + 1. Prefer ontology entities over fact entities + 2. Among ontology entities (or fact entities), prefer simpler URIs + """ + + def __init__(self): + """Initialize the representative selector.""" + pass + + def compute_simplicity_score(self, entity: URIRef) -> float: + """Compute simplicity score for an entity URI. + + Lower score = simpler = better + + Args: + entity: Entity URI + + Returns: + Simplicity score (lower is better) + """ + uri_str = str(entity) + + # Factors that increase complexity (decrease simplicity) + score = 0.0 + + # Length penalty (longer URIs are more complex) + score += len(uri_str) * 0.1 + + # Path depth penalty (more / means deeper hierarchy) + score += uri_str.count("/") * 5 + + # Underscore/hyphen penalty (more complex names) + score += uri_str.count("_") * 2 + score += uri_str.count("-") * 2 + + # Number penalty (URIs with numbers are often auto-generated) + score += sum(c.isdigit() for c in uri_str) * 1 + + return score + + def select_representative( + self, cluster: list[URIRef], representations: dict[URIRef, EntityRepresentation] + ) -> URIRef: + """Select the best representative entity from a cluster. + + Selection criteria: + 1. Prefer ontology entities + 2. Among same category, prefer simpler URIs + + Args: + cluster: List of entity URIs in the cluster + representations: Dictionary mapping entities to their representations + + Returns: + The selected representative entity URI + """ + if len(cluster) == 1: + return cluster[0] + + # Separate ontology entities from fact entities + ontology_entities = [ + e for e in cluster if representations[e].is_ontology_entity + ] + fact_entities = [ + e for e in cluster if not representations[e].is_ontology_entity + ] + + # Prefer ontology entities + candidates = ontology_entities if ontology_entities else fact_entities + + # Among candidates, select the simplest + best = min(candidates, key=self.compute_simplicity_score) + + logger.debug( + f"Selected representative {best} from cluster of {len(cluster)} entities " + f"({len(ontology_entities)} ontology, {len(fact_entities)} facts)" + ) + + return best + + def create_mapping( + self, + clusters: list[list[URIRef]], + representations: dict[URIRef, EntityRepresentation], + ) -> dict[URIRef, URIRef]: + """Create mapping from all entities to their cluster representatives. + + Args: + clusters: List of entity clusters + representations: Dictionary mapping entities to their representations + + Returns: + Dictionary mapping each entity to its representative (e -> e') + """ + mapping = {} + + for cluster in clusters: + representative = self.select_representative(cluster, representations) + + for entity in cluster: + mapping[entity] = representative + + logger.info( + f"Created mapping for {len(mapping)} entities " + f"to {len(set(mapping.values()))} representatives" + ) + + return mapping diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/agg/normalizer.py b/ontology_platform/vendored/ontocast/ontocast/tool/agg/normalizer.py new file mode 100644 index 0000000..77454d6 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/agg/normalizer.py @@ -0,0 +1,279 @@ +"""Entity normalization for disambiguation. + +This module handles the preparation of entities for embedding-based disambiguation. +It creates normalized string representations r(e) that include: +- Normalized form of the entity URI +- Semantic neighbors (types, properties) +""" + +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass, field +from typing import TYPE_CHECKING + +from rdflib import RDF, RDFS, Literal, URIRef + +from ontocast.onto.constants import DEFAULT_IRI +from ontocast.onto.rdfgraph import RDFGraph + +if TYPE_CHECKING: + from ontocast.tool.agg.uri_builder import EntityRole + + +@dataclass +class EntityRepresentation: + """Normalized representation of an entity for embedding. + + Attributes: + entity: Original entity URI + normal_form: Normalized string (lowercase, no diacritics, etc.) + types: List of type URIs for this entity + properties: List of property URIs used with this entity + labels: List of labels found for this entity + representation: Combined string representation r(e) for embedding + is_ontology_entity: Whether this entity is from an ontology namespace + role: Detected entity role (class / property / instance) + """ + + entity: URIRef + normal_form: str + types: list[URIRef] + properties: list[URIRef] + labels: list[str] + representation: str + is_ontology_entity: bool + role: EntityRole | None = field(default=None) + + +class EntityNormalizer: + """Normalizes entities and creates string representations for embedding. + + This class is responsible for transforming entity URIs into normalized + string representations that can be embedded and compared. + """ + + def __init__(self, facts_iri: str = DEFAULT_IRI): + """Initialize the entity normalizer. + + Args: + facts_iri: Base IRI for fact entities. Entities under this namespace + are facts; all other entities are considered ontology entities. + """ + self.facts_iri = facts_iri.rstrip("/") + "/" + + def normalize_string(self, text: str) -> str: + """Normalize a string: lowercase, remove diacritics, clean special chars. + + CamelCase is split so that it yields the same logical tokens as snake_case + (e.g. 'PLRedShift' -> 'pl red shift'). + + Args: + text: Input string to normalize + + Returns: + Normalized string suitable for comparison + + Examples: + 'PLRedShift' -> 'pl red shift' + 'PL_red_shift_value' -> 'pl red shift value' + 'Café' -> 'cafe' + """ + # Remove diacritics + text = "".join( + c + for c in unicodedata.normalize("NFD", text) + if unicodedata.category(c) != "Mn" + ) + + # Insert space before capitals that start a word (followed by lowercase) + # so e.g. PLRedShift -> PL Red Shift -> pl red shift (like snake_case) + text = re.sub(r"(?=[A-Z][a-z])", " ", text) + + # Convert to lowercase + text = text.lower() + + # Replace underscores and hyphens with spaces + text = text.replace("_", " ").replace("-", " ") + + # Collapse multiple spaces and strip + return re.sub(r"\s+", " ", text).strip() + + def normalize_uri(self, uri: URIRef) -> str: + """Extract and normalize the local part of a URI. + + Args: + uri: URI to normalize + + Returns: + Normalized local name + + Examples: + 'http://example.org/PLRedShift' -> 'pl red shift' + 'http://example.org/PL_red_shift_value' -> 'pl red shift value' + """ + uri_str = str(uri) + + # Extract local name from fragment or path + if "#" in uri_str: + local = uri_str.rsplit("#", 1)[-1] + else: + trimmed = uri_str.rstrip("/") + local = trimmed.rsplit("/", 1)[-1] if "/" in trimmed else trimmed + + # Handle camelCase before normalization + # Insert spaces before uppercase letters + local = re.sub(r"([a-z])([A-Z])", r"\1 \2", local) + local = re.sub(r"([A-Z]+)([A-Z][a-z])", r"\1 \2", local) + + return self.normalize_string(local) + + def is_ontology_entity(self, entity: URIRef) -> bool: + """Check if an entity belongs to an ontology namespace. + + Facts live under ``facts_iri``; everything else is an ontology entity. + + Args: + entity: Entity URI to check + + Returns: + True if entity is **not** from the facts namespace + """ + return not str(entity).startswith(self.facts_iri) + + def extract_entity_context( + self, entity: URIRef, graph: RDFGraph + ) -> tuple[list[URIRef], list[URIRef], list[str], bool]: + """Extract semantic context for an entity from the graph. + + Args: + entity: Entity to extract context for + graph: RDF graph containing the entity + + Returns: + Tuple of (types, properties, labels, is_predicate). + *is_predicate* is ``True`` when the entity appears in the + predicate position of at least one triple. + """ + types = [] + properties = set() + labels = [] + is_predicate = False + + # Extract information from triples + for s, p, o in graph: + # When entity is subject + if s == entity: + properties.add(p) + + # Collect types + if p == RDF.type and isinstance(o, URIRef): + types.append(o) + + # Collect labels + if p == RDFS.label and isinstance(o, Literal): + labels.append(str(o)) + + # When entity is object + elif o == entity: + properties.add(p) + + # When entity is used as predicate + if p == entity: + is_predicate = True + + return types, list(properties), labels, is_predicate + + def create_representation( + self, entity: URIRef, graph: RDFGraph + ) -> EntityRepresentation: + """Create a normalized representation r(e) for an entity. + + This combines the normalized form with semantic neighbors to create + a rich representation suitable for embedding. The entity role + (class / property / instance) is detected from the already-extracted + context so no additional graph scan is needed downstream. + + Args: + entity: Entity URI + graph: RDF graph containing the entity + + Returns: + EntityRepresentation containing r(e) and metadata + """ + from ontocast.tool.agg.uri_builder import detect_role_from_context + + # Get normalized form + normal_form = self.normalize_uri(entity) + + # Extract semantic context + types, properties, labels, is_predicate = self.extract_entity_context( + entity, graph + ) + + # Detect role from the already-extracted context (no extra graph scan) + role = detect_role_from_context(types, is_predicate) + + # Build representation string r(e) + parts = [normal_form] + + # Add labels if available (most informative) + if labels: + parts.extend( + self.normalize_string(label) for label in labels[:3] + ) # Max 3 labels + + # Add type information (very important semantic signal) + if types: + type_names = [self.normalize_uri(t) for t in types[:3]] # Max 3 types + parts.extend(f"type {tn}" for tn in type_names) + + # Add property information (additional semantic signal) + if properties: + # Filter out very common properties + filtered_props = [ + p for p in properties if p not in {RDF.type, RDFS.label, RDFS.comment} + ] + prop_names = [ + self.normalize_uri(p) for p in filtered_props[:5] + ] # Max 5 properties + parts.extend(f"has {pn}" for pn in prop_names) + + # Combine into representation + representation = " ".join(parts) + + # Check if ontology entity + is_ontology = self.is_ontology_entity(entity) + + return EntityRepresentation( + entity=entity, + normal_form=normal_form, + types=types, + properties=properties, + labels=labels, + representation=representation, + is_ontology_entity=is_ontology, + role=role, + ) + + def create_representations_batch( + self, entities: list[URIRef], graphs: dict[URIRef, RDFGraph] + ) -> dict[URIRef, EntityRepresentation]: + """Create representations for multiple entities. + + Args: + entities: List of entity URIs + graphs: Mapping from entity to its source graph + + Returns: + Dictionary mapping entity URIs to their representations + """ + representations = {} + + for entity in entities: + graph = graphs.get(entity) + if graph is not None: + representations[entity] = self.create_representation(entity, graph) + + return representations diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/agg/promoter.py b/ontology_platform/vendored/ontocast/ontocast/tool/agg/promoter.py new file mode 100644 index 0000000..35f6b36 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/agg/promoter.py @@ -0,0 +1,224 @@ +"""URI promotion from chunk-level to document-level. + +This module handles the promotion of chunk-level entity URIs to document-level URIs, +while preserving ontology entity URIs unchanged. +""" + +import logging +import re + +from rdflib import URIRef + +from ontocast.onto.constants import DEFAULT_IRI + +from .normalizer import EntityRepresentation + +logger = logging.getLogger(__name__) + + +class URIPromoter: + """Promotes chunk-level URIs to document-level URIs. + + This class is responsible for converting entities from chunk namespaces + to the document namespace, while keeping ontology entities unchanged. + """ + + def __init__( + self, + doc_namespace: str, + chunk_namespaces: set[str], + facts_iri: str = DEFAULT_IRI, + ): + """Initialize the URI promoter. + + Args: + doc_namespace: Document namespace for promoted URIs + chunk_namespaces: Set of chunk namespace URIs + facts_iri: Base IRI for fact entities. Entities under this + namespace are facts; all other entities are ontology + entities and preserved as-is. + """ + self.doc_namespace = self._normalize_namespace(doc_namespace) + self.chunk_namespaces = chunk_namespaces + self.facts_iri = facts_iri.rstrip("/") + "/" + self._used_uris: set[str] = set() + + def _normalize_namespace(self, namespace: str) -> str: + """Ensure namespace ends with appropriate separator.""" + return namespace if namespace.endswith(("/", "#")) else namespace + "/" + + def _clean_local_name(self, name: str) -> str: + """Clean a name for use as URI local part. + + Args: + name: Name to clean + + Returns: + Cleaned name suitable for URI + """ + # Replace invalid URI characters with underscores + cleaned = re.sub(r"[^\w\-.]", "_", name) + # Remove consecutive underscores + cleaned = re.sub(r"_+", "_", cleaned) + # Remove leading/trailing underscores + cleaned = cleaned.strip("_") + return cleaned or "entity" + + def _ensure_unique_uri(self, uri: str) -> str: + """Ensure URI is unique by appending counter if needed. + + Args: + uri: Proposed URI + + Returns: + Unique URI + """ + if uri not in self._used_uris: + self._used_uris.add(uri) + return uri + + # URI already used, append counter + base_uri = uri + counter = 1 + + while uri in self._used_uris: + # Extract local name and add counter + if "#" in base_uri: + namespace, local = base_uri.rsplit("#", 1) + uri = f"{namespace}#{local}_{counter}" + else: + namespace = base_uri.rstrip("/").rsplit("/", 1)[0] + local = ( + base_uri.rstrip("/").rsplit("/", 1)[1] + if "/" in base_uri.rstrip("/") + else base_uri + ) + uri = f"{namespace}/{local}_{counter}" + counter += 1 + + self._used_uris.add(uri) + return uri + + def should_promote(self, entity: URIRef) -> bool: + """Check if an entity should be promoted to document namespace. + + Args: + entity: Entity URI to check + + Returns: + True if entity should be promoted (is from chunk namespace) + """ + entity_str = str(entity) + + # Don't promote ontology entities (anything not under facts_iri) + if not entity_str.startswith(self.facts_iri): + return False + + # Promote chunk entities + if any(entity_str.startswith(ns) for ns in self.chunk_namespaces): + return True + + # Unknown namespace - be conservative, don't promote + return False + + def promote_entity( + self, entity: URIRef, representation: EntityRepresentation + ) -> URIRef: + """Promote a chunk entity to document namespace. + + Args: + entity: Original entity URI + representation: Entity representation with metadata + + Returns: + Promoted entity URI + """ + if not self.should_promote(entity): + # Keep ontology entities unchanged + return entity + + # Create new URI in document namespace + # Use normalized form as local name + local_name = self._clean_local_name(representation.normal_form) + + # Construct promoted URI + promoted_uri_str = f"{self.doc_namespace}{local_name}" + + # Ensure uniqueness + unique_uri_str = self._ensure_unique_uri(promoted_uri_str) + + return URIRef(unique_uri_str) + + def create_promotion_mapping( + self, + entities: list[URIRef], + representations: dict[URIRef, EntityRepresentation], + ) -> dict[URIRef, URIRef]: + """Create mapping from original to promoted URIs. + + Args: + entities: List of entity URIs to promote + representations: Dictionary mapping entities to their representations + + Returns: + Dictionary mapping original URIs to promoted URIs (e -> promoted(e)) + """ + mapping = {} + promoted_count = 0 + preserved_count = 0 + + for entity in entities: + representation = representations.get(entity) + if representation is None: + logger.warning(f"No representation found for entity {entity}") + mapping[entity] = entity + continue + + promoted = self.promote_entity(entity, representation) + mapping[entity] = promoted + + if promoted != entity: + promoted_count += 1 + else: + preserved_count += 1 + + logger.info( + f"Created promotion mapping: " + f"{promoted_count} promoted, " + f"{preserved_count} preserved (ontology entities)" + ) + + return mapping + + def compose_mappings( + self, + clustering_mapping: dict[URIRef, URIRef], + promotion_mapping: dict[URIRef, URIRef], + ) -> dict[URIRef, URIRef]: + """Compose clustering and promotion mappings. + + First, entities are mapped to their cluster representatives (clustering_mapping). + Then, representatives are promoted to document namespace (promotion_mapping). + + The composed mapping is: e -> promoted(representative(e)) + + Args: + clustering_mapping: Map from entity to cluster representative (e -> e_rep) + promotion_mapping: Map from representative to promoted URI (e_rep -> e') + + Returns: + Composed mapping (e -> e') + """ + composed = {} + + for original, representative in clustering_mapping.items(): + # Look up the promoted version of the representative + promoted = promotion_mapping.get(representative, representative) + composed[original] = promoted + + logger.info( + f"Composed mapping: {len(composed)} entities mapped to " + f"{len(set(composed.values()))} final URIs" + ) + + return composed diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/agg/rewriter.py b/ontology_platform/vendored/ontocast/ontocast/tool/agg/rewriter.py new file mode 100644 index 0000000..e2d399f --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/agg/rewriter.py @@ -0,0 +1,528 @@ +"""Graph rewriting for entity disambiguation. + +This module handles the robust application of entity mappings to RDF graphs, +replacing all occurrences of entities according to the mapping. + +Provenance is tracked using `RDF 1.2 reification +`_ together with the +`PROV-O `_ vocabulary. For every asserted +fact triple a **reifier** blank node is created:: + + _:r rdf:reifies <<( s p o )>> . + _:r prov:wasDerivedFrom . + +When the same triple originates from multiple chunks the reifier +accumulates several ``prov:wasDerivedFrom`` arcs. Chunk metadata +(``index``, ``hid``) is recorded as separate triples on the chunk URI. + +The merged graph is backed by the *oxigraph* store so that RDF 1.2 +triple-term syntax (``<<( s p o )>>``) is serialised correctly via +``pyoxigraph``. +""" + +import logging +from collections import defaultdict + +import pyoxigraph as ox +from oxrdflib._converter import to_ox +from rdflib import Literal, Node, URIRef +from rdflib.namespace import OWL, RDF, XSD + +from ontocast.onto.constants import PROV, RDF_REIFIES, SCHEMA +from ontocast.onto.content_unit import ContentUnit +from ontocast.onto.rdfgraph import RDFGraph + +logger = logging.getLogger(__name__) + +# Local alias for readability +_PROV = PROV +_SCHEMA = SCHEMA + + +class GraphRewriter: + """Rewrites RDF graphs by applying entity mappings. + + This class handles the robust replacement of entity URIs in RDF graphs + according to a mapping, while preserving graph structure and metadata. + """ + + def __init__( + self, + add_sameas_links: bool = True, + blocked_sameas_namespaces: tuple[str, ...] = (), + ): + """Initialize the graph rewriter. + + Args: + add_sameas_links: Whether to add owl:sameAs links for merged entities + blocked_sameas_namespaces: Namespace prefixes that should never appear + as subject or object in emitted owl:sameAs links. + """ + self.add_sameas_links = add_sameas_links + self.blocked_sameas_namespaces = blocked_sameas_namespaces + + @staticmethod + def _in_namespace(entity: URIRef, namespace: str) -> bool: + entity_str = str(entity) + if entity_str.startswith(namespace): + return True + slash_variant = namespace.rstrip("/") + "/" + hash_variant = namespace.rstrip("#") + "#" + return entity_str.startswith(slash_variant) or entity_str.startswith( + hash_variant + ) + + def should_emit_sameas(self, original: URIRef, canonical: URIRef) -> bool: + """Return whether a sameAs link is valid for emission.""" + if original == canonical: + return False + for namespace in self.blocked_sameas_namespaces: + if self._in_namespace(original, namespace) or self._in_namespace( + canonical, namespace + ): + return False + return True + + def _emit_sameas_links( + self, + target_graph: RDFGraph, + merged_entities: dict[URIRef, set[URIRef]], + ) -> None: + if not self.add_sameas_links: + return + for canonical, originals in merged_entities.items(): + for original in originals: + if self.should_emit_sameas(original, canonical): + target_graph.add((canonical, OWL.sameAs, original)) + + def apply_mapping_to_triple( + self, + subject: Node, + predicate: Node, + obj: Node, + mapping: dict[URIRef, URIRef], + ) -> tuple[Node, Node, Node]: + """Apply entity mapping to a single triple. + + Args: + subject: Triple subject + predicate: Triple predicate + obj: Triple object + mapping: Entity mapping + + Returns: + Mapped triple (subject, predicate, object) + """ + # Map subject if needed + new_subject = ( + mapping.get(subject, subject) if isinstance(subject, URIRef) else subject + ) + + # Map predicate if needed + new_predicate = ( + mapping.get(predicate, predicate) + if isinstance(predicate, URIRef) + else predicate + ) + + # Map object if needed + new_obj = mapping.get(obj, obj) if isinstance(obj, URIRef) else obj + + return new_subject, new_predicate, new_obj + + def rewrite_graph(self, graph: RDFGraph, mapping: dict[URIRef, URIRef]) -> RDFGraph: + """Rewrite a graph by applying entity mapping. + + Args: + graph: Original RDF graph + mapping: Entity mapping (e -> e') + + Returns: + New RDF graph with entities replaced according to mapping + """ + rewritten = RDFGraph() + + # Copy namespace bindings + for prefix, namespace in graph.namespaces(): + rewritten.bind(prefix, namespace) + + # Track which entities were merged for owl:sameAs links + merged_entities: dict[URIRef, set[URIRef]] = defaultdict(set) + for original, mapped in mapping.items(): + if original != mapped: + merged_entities[mapped].add(original) + + # Rewrite all triples + processed_triples = set() + + for s, p, o in graph: + # Apply mapping + new_s, new_p, new_o = self.apply_mapping_to_triple(s, p, o, mapping) + + # Create triple signature to avoid duplicates + triple_sig = (new_s, new_p, new_o) + + # Skip if we've already added this triple + if triple_sig in processed_triples: + continue + + # Add rewritten triple + rewritten.add(triple_sig) + processed_triples.add(triple_sig) + + # Add owl:sameAs links for merged entities + self._emit_sameas_links(rewritten, merged_entities) + + logger.info( + f"Rewrote graph: {len(graph)} -> {len(rewritten)} triples " + f"({len(merged_entities)} entities merged)" + ) + + return rewritten + + @staticmethod + def _merge_sameas_links( + merged_entities: dict[URIRef, set[URIRef]], + extra_sameas_links: dict[URIRef, set[URIRef]] | None, + ) -> dict[URIRef, set[URIRef]]: + """Merge mapping-derived sameAs links with explicitly provided aliases.""" + if not extra_sameas_links: + return merged_entities + for canonical, originals in extra_sameas_links.items(): + merged_entities[canonical].update( + original for original in originals if original != canonical + ) + return merged_entities + + def merge_graphs( + self, + graphs: list[RDFGraph], + mapping: dict[URIRef, URIRef], + base_namespace: str, + extra_sameas_links: dict[URIRef, set[URIRef]] | None = None, + suppress_sameas_origins: set[URIRef] | None = None, + ) -> RDFGraph: + """Merge multiple graphs into one, applying entity mapping. + + Args: + graphs: List of RDF graphs to merge. + mapping: Entity mapping (e -> e'). + base_namespace: Base namespace for the merged graph (bound as ``facts:``). + + Returns: + Single merged and rewritten graph. + """ + merged = RDFGraph() + + # Bind base namespace + merged.bind("facts", base_namespace) + + # Collect all namespaces from all graphs + all_namespaces = {} + for graph in graphs: + for prefix, namespace in graph.namespaces(): + if prefix not in all_namespaces: + all_namespaces[prefix] = namespace + elif all_namespaces[prefix] != namespace: + # Handle prefix conflicts + new_prefix = f"{prefix}_{len(all_namespaces)}" + all_namespaces[new_prefix] = namespace + + # Bind all namespaces + for prefix, namespace in all_namespaces.items(): + merged.bind(prefix, namespace) + + # Track processed triples to avoid duplicates + processed_triples = set() + + # Track merged entities for owl:sameAs + merged_entities: dict[URIRef, set[URIRef]] = defaultdict(set) + suppressed_origins = suppress_sameas_origins or set() + for original, mapped in mapping.items(): + if original != mapped: + if original in suppressed_origins: + continue + merged_entities[mapped].add(original) + + # Merge all graphs + for graph in graphs: + for s, p, o in graph: + # Apply mapping + new_s, new_p, new_o = self.apply_mapping_to_triple(s, p, o, mapping) + + triple_sig = (new_s, new_p, new_o) + + if triple_sig not in processed_triples: + merged.add(triple_sig) + processed_triples.add(triple_sig) + + merged_entities = self._merge_sameas_links(merged_entities, extra_sameas_links) + + # Add owl:sameAs links + self._emit_sameas_links(merged, merged_entities) + + total_original_triples = sum(len(g) for g in graphs) + logger.info( + f"Merged {len(graphs)} graphs: " + f"{total_original_triples} -> {len(merged)} triples " + f"({len(merged_entities)} entities merged)" + ) + + return merged + + # ------------------------------------------------------------------ + # provenance helpers + # ------------------------------------------------------------------ + + @staticmethod + def _to_ox_term( + node: Node, + ) -> ox.NamedNode | ox.BlankNode | ox.Literal: + """Convert an rdflib term to a pyoxigraph term via oxrdflib. + + The ``oxrdflib._converter.to_ox`` function has a broad return + type, but for RDF *terms* (``URIRef``, ``Literal``, ``BNode``) + it always produces the corresponding pyoxigraph type. + """ + result = to_ox(node) + assert isinstance(result, (ox.NamedNode, ox.BlankNode, ox.Literal)) + return result + + def _add_unit_metadata( + self, + graph: RDFGraph, + unit: ContentUnit, + ) -> URIRef: + """Add source-unit metadata triples and return the source unit URI. + + Emitted triples:: + + a prov:Entity, schema:Text ; + schema:position ; + schema:identifier ; + prov:generatedAtTime . + """ + + unit_uri = URIRef(unit.iri_absolute) + + graph.add((unit_uri, RDF.type, _PROV.Entity)) + graph.add((unit_uri, RDF.type, _SCHEMA.text)) + graph.add( + ( + unit_uri, + _PROV.generatedAtTime, + Literal(f"{unit.generated_at_iso}", datatype=XSD.dateTime), + ) + ) + graph.add( + ( + unit_uri, + _SCHEMA.position, + Literal(unit.index, datatype=XSD.integer), + ) + ) + graph.add((unit_uri, _SCHEMA.identifier, Literal(unit.hid))) + return unit_uri + + def _add_reified_provenance( + self, + graph: RDFGraph, + s: Node, + p: Node, + o: Node, + chunk_uri: URIRef, + reifier: ox.BlankNode | None = None, + ) -> ox.BlankNode: + """Attach provenance to a triple using RDF 1.2 reification. + + Creates (or reuses) a reifier blank node and emits:: + + _:r rdf:reifies <<( s p o )>> . + _:r prov:wasDerivedFrom . + + The ``rdf:reifies`` quad is only added when a *new* reifier is + created. The ``prov:wasDerivedFrom`` quad is always added so that + a shared triple accumulates one arc per source chunk. + + Args: + graph: Oxigraph-backed :class:`RDFGraph`. + s: Triple subject (rdflib term). + p: Triple predicate (rdflib term). + o: Triple object (rdflib term). + chunk_uri: URI of the source :class:`ContentUnit`. + reifier: Existing reifier to reuse. When *None* a fresh + blank node is created. + + Returns: + The reifier blank node (for later reuse). + """ + # Access the underlying pyoxigraph Store and graph context so + # that triples added here are visible through the rdflib API. + ox_store: ox.Store = graph.store._inner # type: ignore[attr-defined] + graph_ctx_raw = to_ox(graph.identifier) + assert isinstance(graph_ctx_raw, (ox.NamedNode, ox.BlankNode, ox.DefaultGraph)) + graph_ctx: ox.NamedNode | ox.BlankNode | ox.DefaultGraph = graph_ctx_raw + + # Convert rdflib terms → pyoxigraph terms + s_ox = self._to_ox_term(s) + p_ox = self._to_ox_term(p) + o_ox = self._to_ox_term(o) + + # Narrow types for ox.Triple (subject: NamedNode|BlankNode|Triple, + # predicate: NamedNode, object: any ox term). + assert isinstance(s_ox, (ox.NamedNode, ox.BlankNode)) + assert isinstance(p_ox, ox.NamedNode) + + # RDF 1.2 triple term + triple_term = ox.Triple(s_ox, p_ox, o_ox) + + if reifier is None: + reifier = ox.BlankNode() + # rdf:reifies is emitted only once per reifier + ox_store.add( + ox.Quad( + reifier, + ox.NamedNode(str(RDF_REIFIES)), + triple_term, + graph_ctx, + ) + ) + + # prov:wasDerivedFrom — one arc per source chunk + ox_store.add( + ox.Quad( + reifier, + ox.NamedNode(str(_PROV.wasDerivedFrom)), + ox.NamedNode(str(chunk_uri)), + graph_ctx, + ) + ) + + return reifier + + def merge_graphs_with_provenance( + self, + units: list[ContentUnit], + mapping: dict[URIRef, URIRef], + extra_sameas_links: dict[URIRef, set[URIRef]] | None = None, + suppress_sameas_origins: set[URIRef] | None = None, + ) -> RDFGraph: + """Merge multiple chunk graphs with per-triple provenance tracking. + + This method extends :meth:`merge_graphs` by: + + 1. Recording **chunk metadata** (``index``, ``hid``) as separate + triples using ``prov:Entity`` / ``schema:position`` / + ``schema:identifier``. + 2. Creating an **RDF 1.2 reifier** node for every asserted fact + triple using ``rdf:reifies`` with a triple term + ``<<( s p o )>>``, and linking it back to its source chunk via + ``prov:wasDerivedFrom``. If the same triple is produced by + several chunks the reifier accumulates multiple + ``prov:wasDerivedFrom`` arcs. + + The merged graph is backed by the *oxigraph* store so that + RDF 1.2 triple-term serialisation is available natively. + + Args: + units: Content units whose graphs are to be merged. + mapping: Entity mapping ``e → e'``. + + Returns: + Merged RDF graph with RDF 1.2 provenance annotations. + """ + merged = RDFGraph(store="oxigraph") + + # Bind well-known namespaces + merged.bind("prov", str(_PROV)) + merged.bind("schema", str(_SCHEMA)) + + # Collect all unique doc_iri namespaces and bind them + doc_iris: set[str] = set() + for unit in units: + if unit.doc_iri: + doc_iris.add(unit.doc_iri) + for idx, doc_iri in enumerate(sorted(doc_iris)): + prefix = f"doc{idx}" if len(doc_iris) > 1 else "doc" + merged.bind(prefix, doc_iri.rstrip("/") + "/") + + # Collect namespaces from all source graphs + all_namespaces: dict[str, str] = {} + for unit in units: + if unit.graph is None: + continue + for prefix, namespace in unit.graph.namespaces(): + if prefix not in all_namespaces and namespace != unit.iri: + all_namespaces[prefix] = namespace + + for prefix, namespace in all_namespaces.items(): + merged.bind(prefix, namespace) + + # Track processed fact triples + processed_triples: set[tuple[Node, Node, Node]] = set() + + # Track reifier blank nodes keyed by triple signature so that a + # shared triple accumulates multiple prov:wasDerivedFrom arcs on + # the *same* reifier. + reifier_map: dict[tuple[Node, Node, Node], ox.BlankNode] = {} + + # Merged-entity tracking for owl:sameAs + merged_entities: dict[URIRef, set[URIRef]] = defaultdict(set) + suppressed_origins = suppress_sameas_origins or set() + for original, mapped in mapping.items(): + if original != mapped: + if original in suppressed_origins: + continue + merged_entities[mapped].add(original) + + for unit in units: + if unit.graph is None: + continue + + # 1. Chunk metadata + chunk_uri = self._add_unit_metadata(merged, unit) + + # 2. Merge triples with provenance + for s, p, o in unit.graph: + new_s, new_p, new_o = self.apply_mapping_to_triple( + s, + p, + o, + mapping, + ) + triple_sig = (new_s, new_p, new_o) + + # Assert fact (deduplicated) + if triple_sig not in processed_triples: + merged.add(triple_sig) + processed_triples.add(triple_sig) + + # Attach RDF 1.2 reified provenance. + # Reuse the existing reifier when the triple was already + # seen from a previous chunk so that prov:wasDerivedFrom + # arcs accumulate on the same blank node. + existing_reifier = reifier_map.get(triple_sig) + reifier = self._add_reified_provenance( + merged, + new_s, + new_p, + new_o, + chunk_uri, + reifier=existing_reifier, + ) + if triple_sig not in reifier_map: + reifier_map[triple_sig] = reifier + + merged_entities = self._merge_sameas_links(merged_entities, extra_sameas_links) + + # owl:sameAs links + self._emit_sameas_links(merged, merged_entities) + + total_original = sum(len(u.graph) for u in units if u.graph is not None) + logger.info( + f"Merged {len(units)} unit graphs with provenance: " + f"{total_original} -> {len(merged)} triples " + f"({len(merged_entities)} entities merged)" + ) + + return merged diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/agg/uri_builder.py b/ontology_platform/vendored/ontocast/ontocast/tool/agg/uri_builder.py new file mode 100644 index 0000000..929c8ec --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/agg/uri_builder.py @@ -0,0 +1,394 @@ +"""URI construction with naming convention normalization. + +Builds final URIs for entity representatives following +RDF/Semantic Web naming conventions (see README.md): +- Classes (entities / types): PascalCase (e.g., JudicialDecision) +- Properties (predicates): lowerCamelCase (e.g., hasDecision) +- Instances with natural names: PascalCase (e.g., FrenchCourtOfCassation) +- Instances with structured/external IDs: preserve structure (e.g., Case_2023_456) + +Underscores are avoided in ontology terms (classes, properties). +Underscores are acceptable for instances derived from external IDs. +""" + +import logging +import re +from enum import StrEnum + +from rdflib import OWL, RDF, RDFS, URIRef + +from ontocast.onto.constants import DEFAULT_IRI +from ontocast.onto.rdfgraph import RDFGraph + +from .normalizer import EntityRepresentation + +logger = logging.getLogger(__name__) + +# Types that mark an entity as a class +_CLASS_TYPES = frozenset({RDFS.Class, OWL.Class}) + +# Types that mark an entity as a property +_PROPERTY_TYPES = frozenset( + {RDF.Property, OWL.ObjectProperty, OWL.DatatypeProperty, OWL.AnnotationProperty} +) + + +class EntityRole(StrEnum): + """Role of an entity in an RDF graph.""" + + CLASS = "class" + PROPERTY = "property" + INSTANCE = "instance" + + +def detect_role(entity: URIRef, graph: RDFGraph) -> EntityRole: + """Detect the role of an entity: class, property, or instance. + + Args: + entity: The entity URI. + graph: The RDF graph containing the entity. + + Returns: + The detected :class:`EntityRole`. + """ + entity_types: set[URIRef] = set() + is_predicate = False + + for s, p, o in graph: + if s == entity and p == RDF.type and isinstance(o, URIRef): + entity_types.add(o) + if p == entity: + is_predicate = True + + if entity_types & _CLASS_TYPES: + return EntityRole.CLASS + if entity_types & _PROPERTY_TYPES or is_predicate: + return EntityRole.PROPERTY + return EntityRole.INSTANCE + + +def detect_role_from_context( + types: list[URIRef], + is_predicate: bool = False, +) -> EntityRole: + """Detect entity role from pre-extracted context (no graph scan needed). + + This is the preferred entry point when the caller has already extracted + types and predicate usage via + :meth:`EntityNormalizer.extract_entity_context`, avoiding a redundant + full-graph iteration. + + Args: + types: ``rdf:type`` values of the entity. + is_predicate: Whether the entity appears in the predicate position + of at least one triple. + + Returns: + The detected :class:`EntityRole`. + """ + type_set = frozenset(types) + + if type_set & _CLASS_TYPES: + return EntityRole.CLASS + if type_set & _PROPERTY_TYPES or is_predicate: + return EntityRole.PROPERTY + return EntityRole.INSTANCE + + +def to_pascal_case(normalized: str) -> str: + """Convert a space-separated lowercase string to PascalCase. + + Args: + normalized: Space-separated lowercase string. + + Returns: + PascalCase string. + + Examples: + >>> to_pascal_case('judicial decision') + 'JudicialDecision' + >>> to_pascal_case('french court of cassation') + 'FrenchCourtOfCassation' + """ + words = normalized.split() + return "".join(w.capitalize() for w in words if w) + + +def to_lower_camel_case(normalized: str) -> str: + """Convert a space-separated lowercase string to lowerCamelCase. + + Args: + normalized: Space-separated lowercase string. + + Returns: + lowerCamelCase string. + + Examples: + >>> to_lower_camel_case('has decision') + 'hasDecision' + >>> to_lower_camel_case('date published') + 'datePublished' + """ + words = normalized.split() + if not words: + return "" + return words[0] + "".join(w.capitalize() for w in words[1:]) + + +def has_structured_id(entity: URIRef) -> bool: + """Detect if an entity represents a structured/external identifier. + + Structured IDs contain digits together with underscores, e.g. + ``Case_2023_456`` or ``Decision_2021_09_15``. + + Args: + entity: Original entity URI. + + Returns: + True if the entity appears to have a structured ID. + """ + local = str(entity).rsplit("/", 1)[-1].rsplit("#", 1)[-1] + return bool(re.search(r"\d", local) and "_" in local) + + +def format_structured_id(entity: URIRef) -> str: + """Format a structured identifier preserving underscores and digits. + + The leading word segment is capitalised so that the result starts + with an uppercase letter (e.g. ``Case_2023_456``). + + Args: + entity: Original entity URI. + + Returns: + Cleaned identifier string. + """ + local = str(entity).rsplit("/", 1)[-1].rsplit("#", 1)[-1] + cleaned = re.sub(r"[^\w]", "_", local) + cleaned = re.sub(r"_+", "_", cleaned).strip("_") + if not cleaned: + return "Entity" + # Capitalise first segment for readability + parts = cleaned.split("_", 1) + parts[0] = parts[0].capitalize() + return "_".join(parts) + + +def normalize_local_name( + representation: EntityRepresentation, + role: EntityRole | str, +) -> str: + """Produce a properly-cased local name following RDF conventions. + + Args: + representation: Entity representation with metadata. + role: Entity role (an :class:`EntityRole` value). + + Returns: + Properly cased local name. + """ + if role == EntityRole.PROPERTY: + return to_lower_camel_case(representation.normal_form) + + if role == EntityRole.INSTANCE and has_structured_id(representation.entity): + return format_structured_id(representation.entity) + + # Classes and instances with natural names → PascalCase + return to_pascal_case(representation.normal_form) + + +class URIBuilder: + """Build normalized URIs for all entities following RDF naming conventions. + + - **Fact entities** (under *base_iri*) get new URIs under *base_iri*. + - **Ontology entities** (everything else) are preserved as-is. + """ + + def __init__( + self, + base_iri: str = DEFAULT_IRI, + ): + """Initialise the builder. + + Args: + base_iri: Base IRI for fact entities (default ``DEFAULT_IRI``). + Entities under this namespace are facts; everything else is + treated as an ontology entity. + """ + self.base_iri = base_iri.rstrip("/") + "/" + self._used_uris: set[URIRef] = set() + + # ------------------------------------------------------------------ + # helpers + # ------------------------------------------------------------------ + + def is_ontology_entity(self, entity: URIRef) -> bool: + """Return True if *entity* does **not** belong to the facts namespace.""" + return not str(entity).startswith(self.base_iri) + + @staticmethod + def _extract_namespace(entity: URIRef) -> str: + """Extract the namespace part of a URI (everything before the local name). + + For ``http://example.org/ns#Foo`` returns ``http://example.org/ns#``. + For ``http://example.org/ns/Foo`` returns ``http://example.org/ns/``. + """ + uri_str = str(entity) + if "#" in uri_str: + return uri_str.rsplit("#", 1)[0] + "#" + trimmed = uri_str.rstrip("/") + if "/" in trimmed: + return trimmed.rsplit("/", 1)[0] + "/" + return uri_str + + def _ensure_unique_uri(self, base: str, local_name: str) -> URIRef: + """Return a unique URI under *base* for *local_name*.""" + candidate = URIRef(f"{base}{local_name}") + if candidate not in self._used_uris: + self._used_uris.add(candidate) + return candidate + + counter = 1 + while True: + candidate = URIRef(f"{base}{local_name}_{counter}") + if candidate not in self._used_uris: + self._used_uris.add(candidate) + return candidate + counter += 1 + + # ------------------------------------------------------------------ + # public API + # ------------------------------------------------------------------ + + def build_uri( + self, + entity: URIRef, + representation: EntityRepresentation, + role: EntityRole | str, + target_iri: URIRef | str | None = None, + is_ontology_entity: bool | None = None, + ) -> URIRef: + """Build a normalised URI for a single entity. + + Fact entities are normalised and placed under *target_iri* (falling + back to *base_iri*). Ontology entities are preserved as-is. + + Args: + entity: Original entity URI. + representation: Entity representation with metadata. + role: Entity role (an :class:`EntityRole` value). + target_iri: Optional document IRI to use as namespace for fact + entities instead of the default *base_iri*. When chunks carry + different ``doc_iri`` values the caller passes the appropriate + one here so that each fact is placed under its document + namespace. + is_ontology_entity: Explicit ontology/fact classification. When + provided this takes precedence over namespace-based inference. + + Returns: + Normalised URI. + """ + is_ontology = ( + self.is_ontology_entity(entity) + if is_ontology_entity is None + else is_ontology_entity + ) + + if is_ontology: + return entity + + local_name = normalize_local_name(representation, role) + base = (str(target_iri).rstrip("/") + "/") if target_iri else self.base_iri + return self._ensure_unique_uri(base=base, local_name=local_name) + + def create_entity_uri_mapping( + self, + identity_mapping: dict[URIRef, URIRef], + representations: dict[URIRef, EntityRepresentation], + entity_doc_iris: dict[URIRef, URIRef], + entity_is_ontology: dict[URIRef, bool], + ) -> dict[URIRef, URIRef]: + """Create final URI mapping from identity mapping + namespace policy. + + This method decouples canonical identity choice from URI surface choice: + identity mapping decides *what* is the same entity, while this method + decides *how* each source entity should be rendered as a final URI. + Fact entities are always rendered in their source ``doc_iri`` namespace. + Ontology entities are preserved as their canonical URI. + + Args: + identity_mapping: Mapping ``entity -> canonical_entity``. + representations: All entity representations. + entity_doc_iris: Mapping from source entity to source ``doc_iri``. + entity_is_ontology: Classification map where ``True`` means the + canonical entity should stay in ontology space. + + Returns: + Mapping ``entity -> final_uri``. + """ + self._used_uris.clear() + mapping: dict[URIRef, URIRef] = {} + canonical_cache: dict[tuple[URIRef, str], URIRef] = {} + + for entity, canonical in identity_mapping.items(): + rep = representations.get(canonical) + if rep is None: + mapping[entity] = entity + continue + + role = rep.role if rep.role is not None else EntityRole.INSTANCE + is_ontology = entity_is_ontology.get( + canonical, self.is_ontology_entity(canonical) + ) + if is_ontology: + mapping[entity] = canonical + continue + + doc_iri = entity_doc_iris.get(entity) + base = (str(doc_iri).rstrip("/") + "/") if doc_iri else self.base_iri + cache_key = (canonical, base) + if cache_key in canonical_cache: + mapping[entity] = canonical_cache[cache_key] + continue + + canonical_uri = self.build_uri( + canonical, + rep, + role, + target_iri=doc_iri, + is_ontology_entity=False, + ) + canonical_cache[cache_key] = canonical_uri + mapping[entity] = canonical_uri + + normalised = sum(1 for e, u in mapping.items() if e != u) + logger.info( + f"Built URI mapping: {len(mapping)} entities, {normalised} normalised" + ) + return mapping + + @staticmethod + def compose_mappings( + clustering_mapping: dict[URIRef, URIRef], + uri_mapping: dict[URIRef, URIRef], + ) -> dict[URIRef, URIRef]: + """Compose clustering and URI mappings. + + ``e → representative(e) → normalised_uri(representative(e))`` + + Args: + clustering_mapping: ``e → e_rep``. + uri_mapping: ``e_rep → final_uri``. + + Returns: + Composed mapping ``e → final_uri``. + """ + composed = { + original: uri_mapping.get(representative, representative) + for original, representative in clustering_mapping.items() + } + logger.info( + f"Composed mapping: {len(composed)} entities → " + f"{len(set(composed.values()))} final URIs" + ) + return composed diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/aggregate.py b/ontology_platform/vendored/ontocast/ontocast/tool/aggregate.py new file mode 100644 index 0000000..31238f0 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/aggregate.py @@ -0,0 +1,18 @@ +"""Graph aggregation for OntoCast. + +This module re-exports the embedding-based aggregator as the main aggregation +implementation. Use EmbeddingBasedAggregator for aggregating and disambiguating +RDF graphs from multiple content units. +""" + +from ontocast.tool.agg.aggregate import ( + EmbeddingBasedAggregator, + aggregate_chunk_graphs, + aggregate_content_unit_graphs, +) + +__all__ = [ + "EmbeddingBasedAggregator", + "aggregate_content_unit_graphs", + "aggregate_chunk_graphs", +] diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/atomic.py b/ontology_platform/vendored/ontocast/ontocast/tool/atomic.py new file mode 100644 index 0000000..506592f --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/atomic.py @@ -0,0 +1,152 @@ +"""Minimal tool contracts for atomic render/critic loops.""" + +from typing import Protocol + +from pydantic import BaseModel + +from ontocast.config import WebSearchConfig +from ontocast.onto.enum import WorkflowNode +from ontocast.tool.llm import LLMTool + + +class SearchHit(BaseModel): + """Single web-search hit used as optional grounding context.""" + + title: str + url: str + snippet: str + + +class AtomicLLMProvider(Protocol): + """Provides budget-aware LLM instances for atomic loop calls.""" + + async def get_llm_tool(self, budget_tracker) -> LLMTool: + """Return an LLM tool tied to the given budget tracker.""" + ... + + +class AtomicSearchProvider(Protocol): + """Provides optional web-search retrieval for ontology grounding.""" + + async def search(self, query: str, max_results: int) -> list[SearchHit]: + """Return web hits relevant to the query.""" + ... + + +class AtomicToolBox: + """Small tool surface used by atomic render/critic paths.""" + + def __init__( + self, + llm_provider: AtomicLLMProvider, + search_provider: AtomicSearchProvider | None = None, + web_search_config: WebSearchConfig | None = None, + web_search_enabled: bool = False, + web_search_top_k: int = 3, + web_search_max_snippet_chars: int = 400, + web_search_max_total_chars: int = 1800, + web_search_for_ontology_render: bool = True, + web_search_for_ontology_critic: bool = True, + web_search_for_facts_render: bool = False, + web_search_for_facts_critic: bool = False, + web_search_planner_enabled: bool = True, + web_search_planner_max_queries: int = 3, + web_search_planner_min_query_chars: int = 12, + web_search_planner_min_confidence: float = 0.35, + web_search_reuse_evidence_across_attempt: bool = True, + web_search_allowed_domains: tuple[str, ...] = (), + web_search_blocked_domains: tuple[str, ...] = (), + web_search_min_snippet_chars: int = 40, + ): + self.llm_provider = llm_provider + self.search_provider = search_provider + self.web_search_config = web_search_config + + if web_search_config is not None: + self.web_search_enabled = web_search_config.enabled + self.web_search_top_k = web_search_config.top_k + self.web_search_max_snippet_chars = web_search_config.max_snippet_chars + self.web_search_max_total_chars = web_search_config.max_total_chars + self.web_search_for_ontology_render = ( + web_search_config.ontology_render_enabled + ) + self.web_search_for_ontology_critic = ( + web_search_config.ontology_critic_enabled + ) + self.web_search_for_facts_render = web_search_config.facts_render_enabled + self.web_search_for_facts_critic = web_search_config.facts_critic_enabled + self.web_search_planner_enabled = web_search_config.planner_enabled + self.web_search_planner_max_queries = web_search_config.planner_max_queries + self.web_search_planner_min_query_chars = ( + web_search_config.planner_min_query_chars + ) + self.web_search_planner_min_confidence = ( + web_search_config.planner_min_confidence + ) + self.web_search_reuse_evidence_across_attempt = ( + web_search_config.reuse_evidence_across_attempt + ) + self.web_search_allowed_domains = { + value.strip().lower() + for value in web_search_config.allowed_domains + if value.strip() + } + self.web_search_blocked_domains = { + value.strip().lower() + for value in web_search_config.blocked_domains + if value.strip() + } + self.web_search_min_snippet_chars = web_search_config.min_snippet_chars + else: + self.web_search_enabled = web_search_enabled + self.web_search_top_k = web_search_top_k + self.web_search_max_snippet_chars = web_search_max_snippet_chars + self.web_search_max_total_chars = web_search_max_total_chars + self.web_search_for_ontology_render = web_search_for_ontology_render + self.web_search_for_ontology_critic = web_search_for_ontology_critic + self.web_search_for_facts_render = web_search_for_facts_render + self.web_search_for_facts_critic = web_search_for_facts_critic + self.web_search_planner_enabled = web_search_planner_enabled + self.web_search_planner_max_queries = web_search_planner_max_queries + self.web_search_planner_min_query_chars = web_search_planner_min_query_chars + self.web_search_planner_min_confidence = web_search_planner_min_confidence + self.web_search_reuse_evidence_across_attempt = ( + web_search_reuse_evidence_across_attempt + ) + self.web_search_allowed_domains = { + value.strip().lower() + for value in web_search_allowed_domains + if value.strip() + } + self.web_search_blocked_domains = { + value.strip().lower() + for value in web_search_blocked_domains + if value.strip() + } + self.web_search_min_snippet_chars = web_search_min_snippet_chars + + async def get_llm_tool(self, budget_tracker) -> LLMTool: + """Return a budget-aware LLM tool instance.""" + return await self.llm_provider.get_llm_tool(budget_tracker) + + async def search( + self, query: str, max_results: int | None = None + ) -> list[SearchHit]: + """Run optional web search and return normalized hits.""" + if not self.web_search_enabled or self.search_provider is None: + return [] + + result_limit = max_results if max_results is not None else self.web_search_top_k + return await self.search_provider.search(query=query, max_results=result_limit) + + def web_grounding_enabled_for_node(self, node: WorkflowNode) -> bool: + """Return whether web grounding is enabled for a workflow node.""" + if not self.web_search_enabled: + return False + mapping = { + WorkflowNode.TEXT_TO_ONTOLOGY: self.web_search_for_ontology_render, + WorkflowNode.CRITICISE_ONTOLOGY: self.web_search_for_ontology_critic, + WorkflowNode.TEXT_TO_FACTS: self.web_search_for_facts_render, + WorkflowNode.CRITICISE_FACTS: self.web_search_for_facts_critic, + } + return mapping.get(node, False) diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/cache.py b/ontology_platform/vendored/ontocast/ontocast/tool/cache.py new file mode 100644 index 0000000..7a7778a --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/cache.py @@ -0,0 +1,349 @@ +"""Generic caching functionality for OntoCast tools. + +This module provides a generic caching mechanism that can be used by various +tools to cache their results based on input content and configuration parameters. +""" + +import json +import logging +import os +from pathlib import Path +from typing import TYPE_CHECKING + +from ontocast.util import render_text_hash + +if TYPE_CHECKING: + from ontocast.config import Config + +logger = logging.getLogger(__name__) + + +def _get_default_cache_dir() -> Path: + """Get the default cache directory based on the environment. + + Returns: + Path: The appropriate cache directory path. + """ + # Check if we're in a test environment + if "pytest" in os.environ.get("_", ""): + # In tests, use a test-specific cache directory + return Path.cwd() / ".test_cache" + + # Check for common cache environment variables + cache_home = os.environ.get("XDG_CACHE_HOME") + if cache_home: + return Path(cache_home) / "ontocast" + + # Use platform-appropriate cache directory + if os.name == "nt": # Windows + cache_dir = Path.home() / "AppData" / "Local" / "ontocast" + else: # Unix-like systems + cache_dir = Path.home() / ".cache" / "ontocast" + + return cache_dir + + +class Cacher: + """Shared caching class for OntoCast tools. + + This class provides a unified interface for caching results from various + tools based on input content and configuration parameters. It manages + multiple subdirectories for different tools from a single instance. + + Attributes: + cache_dir: Base directory for caching. + """ + + def __init__( + self, + cache_dir: str | Path | None = None, + config: "Config | None" = None, + ): + """Initialize the shared cacher. + + Args: + cache_dir: Base directory for caching. If None, uses config or platform-appropriate default. + config: Optional config object to get cache_dir from. + """ + if cache_dir is None and config is not None: + # Try to get cache_dir from config + if hasattr(config, "tool_config") and hasattr( + config.tool_config, "path_config" + ): + cache_dir = config.tool_config.path_config.cache_dir + + if cache_dir is None: + cache_dir = _get_default_cache_dir() + + self.cache_dir = Path(cache_dir).expanduser() + self.cache_dir.mkdir(parents=True, exist_ok=True) + logger.debug(f"Shared cache directory set to: {self.cache_dir}") + + def _get_tool_cache_dir(self, subdirectory: str) -> Path: + """Get the cache directory for a specific tool subdirectory. + + Args: + subdirectory: The tool subdirectory name. + + Returns: + Path: The full path to the tool's cache directory. + """ + tool_cache_dir = self.cache_dir / subdirectory + tool_cache_dir.mkdir(parents=True, exist_ok=True) + return tool_cache_dir + + def _generate_cache_key( + self, + content: str | bytes, + config: dict[str, str | int | float | bool] | None = None, + **kwargs: str | int | float | bool, + ) -> str: + """Generate a cache key based on content and configuration. + + Args: + content: The input content (text, bytes, etc.). + config: Optional configuration dictionary. + **kwargs: Additional parameters that affect the result. + + Returns: + str: A hash string to use as cache key. + """ + # Convert content to string for hashing + if isinstance(content, bytes): + content_str = content.decode("utf-8", errors="ignore") + else: + content_str = str(content) + + # Create a dictionary with all relevant parameters + cache_data = { + "content": content_str, + "config": config or {}, + "kwargs": kwargs, + } + + # Convert to JSON string and hash it + cache_string = json.dumps(cache_data, sort_keys=True, default=str) + return render_text_hash(cache_string, digits=None) + + def _get_cache_file_path(self, cache_key: str, subdirectory: str) -> Path: + """Get the cache file path for a given cache key and subdirectory. + + Args: + cache_key: The cache key. + subdirectory: The tool subdirectory name. + + Returns: + Path: The path to the cache file. + """ + tool_cache_dir = self._get_tool_cache_dir(subdirectory) + return tool_cache_dir / f"{cache_key}.json" + + def get( + self, + content: str | bytes, + subdirectory: str, + config: dict[str, str | int | float | bool] | None = None, + **kwargs: str | int | float | bool, + ) -> str | dict | list | None: + """Get cached result for given content and configuration. + + Args: + content: The input content. + subdirectory: The tool subdirectory name. + config: Optional configuration dictionary. + **kwargs: Additional parameters that affect the result. + + Returns: + Optional[Any]: The cached result or None if not found. + """ + cache_key = self._generate_cache_key(content, config, **kwargs) + cache_file = self._get_cache_file_path(cache_key, subdirectory) + + if not cache_file.exists(): + return None + + try: + with open(cache_file, "r", encoding="utf-8") as f: + cached_data = json.load(f) + logger.debug(f"Cache hit for key: {cache_key[:16]}...") + return cached_data.get("result") + except (json.JSONDecodeError, IOError) as e: + logger.warning(f"Failed to read cache file {cache_file}: {e}") + return None + + def set( + self, + content: str | bytes, + result: str | dict | list, + subdirectory: str, + config: dict[str, str | int | float | bool] | None = None, + **kwargs: str | int | float | bool, + ) -> None: + """Cache a result for given content and configuration. + + Args: + content: The input content. + result: The result to cache. + subdirectory: The tool subdirectory name. + config: Optional configuration dictionary. + **kwargs: Additional parameters that affect the result. + """ + cache_key = self._generate_cache_key(content, config, **kwargs) + cache_file = self._get_cache_file_path(cache_key, subdirectory) + + # Prepare data for caching + cache_data = { + "result": result, + "content": str(content)[:100] + "..." + if len(str(content)) > 100 + else str(content), + "config": config or {}, + "kwargs": kwargs, + } + + try: + with open(cache_file, "w", encoding="utf-8") as f: + json.dump(cache_data, f, indent=2, default=str) + logger.debug(f"Cached result to {cache_file}") + except IOError as e: + logger.warning(f"Failed to write cache file {cache_file}: {e}") + + def clear(self, subdirectory: str | None = None) -> None: + """Clear cached results. + + Args: + subdirectory: If provided, clear only this subdirectory. If None, clear all. + """ + if subdirectory is None: + # Clear all subdirectories + if self.cache_dir.exists(): + for cache_file in self.cache_dir.glob("**/*.json"): + cache_file.unlink() + logger.info(f"Cleared all cache directories: {self.cache_dir}") + else: + # Clear specific subdirectory + tool_cache_dir = self._get_tool_cache_dir(subdirectory) + if tool_cache_dir.exists(): + for cache_file in tool_cache_dir.glob("*.json"): + cache_file.unlink() + logger.info(f"Cleared cache directory: {tool_cache_dir}") + + def get_cache_stats( + self, subdirectory: str | None = None + ) -> dict[str, int | dict[str, int]]: + """Get cache statistics. + + Args: + subdirectory: If provided, get stats for this subdirectory only. If None, get stats for all. + + Returns: + Dict[str, Any]: Dictionary with cache statistics. + """ + if subdirectory is None: + # Get stats for all subdirectories + if not self.cache_dir.exists(): + return {"total_files": 0, "total_size_bytes": 0, "subdirectories": {}} + + cache_files = list(self.cache_dir.glob("**/*.json")) + total_size = sum(f.stat().st_size for f in cache_files) + + # Group by subdirectory + subdir_stats = {} + for cache_file in cache_files: + subdir = cache_file.parent.name + if subdir not in subdir_stats: + subdir_stats[subdir] = {"files": 0, "size_bytes": 0} + subdir_stats[subdir]["files"] += 1 + subdir_stats[subdir]["size_bytes"] += cache_file.stat().st_size + + return { + "total_files": len(cache_files), + "total_size_bytes": total_size, + "subdirectories": subdir_stats, + } + else: + # Get stats for specific subdirectory + tool_cache_dir = self._get_tool_cache_dir(subdirectory) + if not tool_cache_dir.exists(): + return {"total_files": 0, "total_size_bytes": 0} + + cache_files = list(tool_cache_dir.glob("*.json")) + total_size = sum(f.stat().st_size for f in cache_files) + + return { + "total_files": len(cache_files), + "total_size_bytes": total_size, + } + + +class ToolCacher: + """Tool-specific wrapper for the shared Cacher. + + This class provides a tool-specific interface to the shared Cacher, + automatically handling the subdirectory parameter. + """ + + def __init__(self, shared_cacher: Cacher, subdirectory: str): + """Initialize the tool cacher. + + Args: + shared_cacher: The shared Cacher instance. + subdirectory: The subdirectory name for this tool. + """ + self.shared_cacher = shared_cacher + self.subdirectory = subdirectory + + def get( + self, + content: str | bytes, + config: dict[str, str | int | float | bool] | None = None, + **kwargs: str | int | float | bool, + ) -> str | dict | list | None: + """Get cached result for given content and configuration. + + Args: + content: The input content. + config: Optional configuration dictionary. + **kwargs: Additional parameters that affect the result. + + Returns: + Optional[Any]: The cached result or None if not found. + """ + return self.shared_cacher.get( + content=content, subdirectory=self.subdirectory, config=config, **kwargs + ) + + def set( + self, + content: str | bytes, + result: str | dict | list, + config: dict[str, str | int | float | bool] | None = None, + **kwargs: str | int | float | bool, + ) -> None: + """Cache a result for given content and configuration. + + Args: + content: The input content. + result: The result to cache. + config: Optional configuration dictionary. + **kwargs: Additional parameters that affect the result. + """ + self.shared_cacher.set( + content=content, + result=result, + subdirectory=self.subdirectory, + config=config, + **kwargs, + ) + + def clear(self) -> None: + """Clear cached results for this tool.""" + self.shared_cacher.clear(subdirectory=self.subdirectory) + + def get_cache_stats(self) -> dict[str, int | dict[str, int]]: + """Get cache statistics for this tool. + + Returns: + Dict[str, int]: Dictionary with cache statistics. + """ + return self.shared_cacher.get_cache_stats(subdirectory=self.subdirectory) diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/chunk/__init__.py b/ontology_platform/vendored/ontocast/ontocast/tool/chunk/__init__.py new file mode 100644 index 0000000..1e18ad7 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/chunk/__init__.py @@ -0,0 +1,16 @@ +"""Document chunking tools for OntoCast. + +This package provides tools for splitting documents into manageable chunks +for processing. It includes semantic chunking capabilities and utilities +for chunk management. + +Available tools: +- ChunkerTool: Main chunking tool for document segmentation +- util: Utility functions for chunk processing and management +""" + +from ontocast.tool.chunk.chunker import ChunkerTool + +__all__ = [ + "ChunkerTool", +] diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/chunk/chunker.py b/ontology_platform/vendored/ontocast/ontocast/tool/chunk/chunker.py new file mode 100644 index 0000000..d694ad5 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/chunk/chunker.py @@ -0,0 +1,260 @@ +import importlib +import logging +import re +import threading +from typing import Any, Literal + +from pydantic import Field + +from ontocast.config import ChunkConfig +from ontocast.tool.cache import Cacher, ToolCacher +from ontocast.tool.chunk.util import SENTENCE_SPLIT_REGEX, SemanticChunker +from ontocast.tool.onto import Tool + +logger = logging.getLogger(__name__) + +# Optional imports for semantic chunking +torch_module: Any | None = None +embedding_model_cls: Any | None = None +try: + torch_module = importlib.import_module("torch") + langchain_huggingface_module = importlib.import_module("langchain_huggingface") + embedding_model_cls = getattr( + langchain_huggingface_module, "HuggingFaceEmbeddings", None + ) + SEMANTIC_CHUNKING_AVAILABLE = embedding_model_cls is not None +except ImportError: + SEMANTIC_CHUNKING_AVAILABLE = False + + +class ChunkerTool(Tool): + """Tool for semantic chunking of documents. + + Falls back to naive chunking if sentence-transformers is not available. + Includes caching to avoid re-chunking the same text with the same parameters. + """ + + model: str = Field( + default="sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + description="HuggingFace model name for embeddings", + ) + config: ChunkConfig = Field( + default_factory=ChunkConfig, description="Chunking configuration parameters" + ) + chunking_mode: Literal["semantic", "naive"] = Field( + default="semantic" if SEMANTIC_CHUNKING_AVAILABLE else "naive", + description="Chunking mode: semantic (requires sentence-transformers) or naive (fallback)", + ) + cache: Any = Field(default=None, exclude=True) + + def __init__( + self, + chunk_config: ChunkConfig | None = None, + cache: Cacher | None = None, + **kwargs, + ): + """Initialize the ChunkerTool. + + Args: + chunk_config: Chunking configuration. If None, uses default ChunkConfig. + cache: Optional shared Cacher instance. If None, creates a new one. + **kwargs: Additional keyword arguments passed to the parent class. + """ + super().__init__(**kwargs) + self._model: Any | None = None + self._model_lock = threading.Lock() # Lock for thread-safe model initialization + + # Initialize cache - use shared cacher or create new one + if cache is not None: + self.cache = ToolCacher(cache, "chunker") + else: + # Fallback for backward compatibility + shared_cache = Cacher() + self.cache = ToolCacher(shared_cache, "chunker") + + # Override config if provided + if chunk_config is not None: + self.config = chunk_config + + # Override chunking mode if semantic chunking is not available + if not SEMANTIC_CHUNKING_AVAILABLE and self.chunking_mode == "semantic": + self.chunking_mode = "naive" + logger.warning( + "Semantic chunking not available (sentence-transformers not installed). " + "Falling back to naive chunking." + ) + + def _init_model(self): + """Initialize the embedding model in a thread-safe manner. + + Uses double-checked locking pattern to ensure the model is only + initialized once, even when called concurrently from multiple threads. + """ + # Fast path: if model already initialized, return immediately + if self._model is not None: + return + + # Acquire lock for thread-safe initialization + with self._model_lock: + # Double-check: another thread might have initialized it while we waited + if self._model is None and SEMANTIC_CHUNKING_AVAILABLE: + if embedding_model_cls is not None: + try: + self._model = embedding_model_cls( + model_name=self.model, + model_kwargs={ + "device": "cuda" + if torch_module is not None + and torch_module.cuda.is_available() + else "cpu" + }, + encode_kwargs={"normalize_embeddings": False}, + ) + logger.debug(f"Initialized embedding model: {self.model}") + except Exception as e: + logger.error(f"Failed to initialize embedding model: {e}") + # Set to a sentinel value to prevent repeated failed attempts + self._model = None + + def _naive_chunk(self, doc: str) -> list[str]: + """Naive chunking fallback when semantic chunking is not available. + + Args: + doc: The document text to chunk. + + Returns: + List of text chunks. + """ + # Split by paragraphs first (double newlines) + paragraphs = re.split(r"\n\s*\n", doc.strip()) + + chunks = [] + current_chunk = "" + + for paragraph in paragraphs: + paragraph = paragraph.strip() + if not paragraph: + continue + + # If adding this paragraph would exceed max_size, start a new chunk + if ( + current_chunk + and len(current_chunk) + len(paragraph) + 2 > self.config.max_size + ): + if current_chunk: + chunks.append(current_chunk.strip()) + current_chunk = paragraph + else: + if current_chunk: + current_chunk += "\n\n" + paragraph + else: + current_chunk = paragraph + + # If a single paragraph is too large, split it by sentences + if len(current_chunk) > self.config.max_size: + # Save the previous chunk if it exists + if len(current_chunk) - len(paragraph) - 2 > 0: + prev_chunk = current_chunk[ + : len(current_chunk) - len(paragraph) - 2 + ].strip() + if prev_chunk: + chunks.append(prev_chunk) + + # Split the large paragraph by sentences + sentences = re.split(r"(?<=[.!?])\s+", paragraph) + temp_chunk = "" + + for sentence in sentences: + if len(temp_chunk) + len(sentence) + 1 > self.config.max_size: + if temp_chunk: + chunks.append(temp_chunk.strip()) + temp_chunk = sentence + else: + if temp_chunk: + temp_chunk += " " + sentence + else: + temp_chunk = sentence + + current_chunk = temp_chunk + + # Add the last chunk + if current_chunk: + chunks.append(current_chunk.strip()) + + # Filter out chunks that are too small + chunks = [chunk for chunk in chunks if len(chunk) >= self.config.min_size] + + logger.info(f"Naive chunking produced {len(chunks)} chunks") + return chunks + + def __call__(self, doc: str) -> list[str]: + """Chunk the document using either semantic or naive chunking. + + Args: + doc: The document text to chunk. + + Returns: + List of text chunks. + """ + # Prepare configuration for caching + config_dict = { + "model": self.model, + "chunking_mode": self.chunking_mode, + "max_size": self.config.max_size, + "min_size": self.config.min_size, + "breakpoint_threshold_type": self.config.breakpoint_threshold_type, + "breakpoint_threshold_amount": self.config.breakpoint_threshold_amount, + } + + # Check cache first + cached_result = self.cache.get(doc, config=config_dict) + if cached_result is not None: + logger.debug("Cache hit for document chunking") + return cached_result + + # Perform chunking + if self.chunking_mode == "naive": + result = self._naive_chunk(doc) + else: + # Semantic chunking (requires sentence-transformers) + if not SEMANTIC_CHUNKING_AVAILABLE: + logger.warning( + "Semantic chunking requested but not available. Falling back to naive chunking." + ) + result = self._naive_chunk(doc) + else: + self._init_model() + documents = [doc] + + if self._model is None: + logger.warning( + "Model not initialized. Falling back to naive chunking." + ) + result = self._naive_chunk(doc) + elif SemanticChunker is None: + logger.warning( + "SemanticChunker not available. Falling back to naive chunking." + ) + result = self._naive_chunk(doc) + else: + text_splitter = SemanticChunker( + embeddings=self._model, + chunk_config=self.config, + sentence_split_regex=SENTENCE_SPLIT_REGEX, + ) + + # SemanticChunker now handles max_size internally + result_docs = text_splitter.create_documents(documents) + result = [doc.page_content for doc in result_docs] + + # Log chunk lengths for debugging + lens = [len(chunk) for chunk in result] + logger.info( + f"Semantic chunking produced {len(result)} chunks with lengths: {lens}" + ) + + # Cache the result + self.cache.set(doc, result, config=config_dict) + logger.debug("Cached document chunking result") + + return result diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/chunk/util.py b/ontology_platform/vendored/ontocast/ontocast/tool/chunk/util.py new file mode 100644 index 0000000..92f9eb1 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/chunk/util.py @@ -0,0 +1,292 @@ +import copy +import re +from typing import Any, Iterable, List, Sequence + +import numpy as np +from hdbscan import HDBSCAN +from langchain_core.documents import BaseDocumentTransformer, Document +from langchain_core.embeddings import Embeddings +from sklearn.decomposition import PCA +from umap import UMAP + +from ontocast.config import ChunkConfig + +# Regex pattern for splitting text into sentences +# Matches: paragraph breaks (double newlines) OR sentence endings followed by capital letters +SENTENCE_SPLIT_REGEX = r"(?:\n\s*\n+)|(?<=[.!?])\s+(?=[A-Z][a-z])" + + +class SemanticChunker(BaseDocumentTransformer): + def __init__( + self, + embeddings: Embeddings, + chunk_config: ChunkConfig, + sentence_split_regex: str, + ): + """Initialize SemanticChunker. + + Args: + embeddings: Embeddings model for generating sentence embeddings. + chunk_config: Chunking configuration containing min_size, max_size, etc. + sentence_split_regex: Regular expression pattern for splitting text into sentences. + """ + self.embeddings = embeddings + self.chunk_config = chunk_config + self.min_size = chunk_config.min_size + self.max_size = chunk_config.max_size + self.sentence_split_regex = sentence_split_regex + + def _build_sentence_windows( + self, + sentences: List[str], + window_size: int = 5, + ) -> List[str]: + if len(sentences) <= window_size: + return [" ".join(sentences)] + + windows = [] + for i in range(len(sentences)): + start = max(0, i - window_size // 2) + end = min(len(sentences), start + window_size) + window = " ".join(sentences[start:end]) + windows.append(window) + + return windows + + def _get_embeddings(self, sentences: List[str]) -> np.ndarray: + """Embeds sentences directly without buffering. + + Since we cluster all sentences together, the clustering algorithm + naturally captures semantic relationships without needing context windows. + """ + return np.array(self.embeddings.embed_documents(sentences)) + + def _cluster_sentences( + self, vectors: np.ndarray, sentences: List[str] + ) -> np.ndarray: + """Pipeline: PCA -> UMAP -> HDBSCAN with parameters favoring more clusters. + + Uses HDBSCAN hyperparameters tuned to create more clusters, which helps + ensure chunks respect max_size constraints. Large clusters will be split + post-processing. + + Args: + vectors: Embedding vectors for sentences. + sentences: Original sentence texts for length validation. + + Returns: + Cluster labels for each sentence. + """ + # 1. PCA to reduce noise + pca_dims = min(vectors.shape[0] - 1, 50) + if pca_dims > 1: + vectors = PCA(n_components=pca_dims).fit_transform(vectors) + + # 2. UMAP to 5 dimensions + # n_neighbors=2 captures very local structure for chunking + reducer = UMAP(n_components=5, n_neighbors=2, min_dist=0.0, metric="cosine") + reduced_vectors = reducer.fit_transform(vectors) + + # 3. HDBSCAN with parameters favoring more clusters + # Calculate optimal min_cluster_size based on max_size constraint + # We want clusters small enough that they can be combined without exceeding max_size + if len(sentences) > 0: + avg_sentence_len = sum(len(s) for s in sentences) / len(sentences) + # Target: clusters should be small enough that 2-3 clusters can fit in max_size + # This encourages more, smaller clusters + target_cluster_size = max(2, int(self.max_size / (avg_sentence_len * 2.5))) + min_cluster_size = min(target_cluster_size, len(sentences) // 3, 10) + min_cluster_size = max(2, min_cluster_size) # At least 2, at most 10 + else: + min_cluster_size = 2 + + # Use cluster_selection_epsilon to encourage more splits + # Higher epsilon = more aggressive splitting = more clusters + # We use a small epsilon (0.1-0.3) to encourage splits while maintaining semantics + clusterer = HDBSCAN( + min_cluster_size=min_cluster_size, + min_samples=1, # Lower min_samples = more clusters + metric="euclidean", + cluster_selection_epsilon=0.1, # Encourage more splits + cluster_selection_method="eom", # Excess of Mass method + ) + labels = clusterer.fit_predict(reduced_vectors) + + return labels + + def split_text(self, text: str) -> List[str]: + # Atomic split into sentences - chunks must contain whole sentences + # Use capturing groups to preserve delimiters + # Wrap the regex in a capturing group so delimiters are included in split result + pattern_with_capture = f"({self.sentence_split_regex})" + parts = re.split(pattern_with_capture, text) + + # Reconstruct sentences with their following delimiters + # parts alternates: [text1, delimiter1, text2, delimiter2, ..., textN] + # Handle case where text starts with delimiter (parts[0] empty) + sentences = [] + delimiters = [] # Track delimiter after each sentence + + # Skip leading empty part if text starts with delimiter + start_idx = 1 if parts and not parts[0].strip() else 0 + + i = start_idx + while i < len(parts): + if i % 2 == start_idx % 2: # Text parts (same parity as start) + text_part = parts[i].strip() + if text_part: # Non-empty text + sentences.append(parts[i]) # Keep original (with whitespace) + # Get the delimiter that follows (if any) + if i + 1 < len(parts): + delimiters.append(parts[i + 1]) + else: + delimiters.append("") # No delimiter after last sentence + i += 1 + + # Filter out empty sentences + if not sentences: + return [text] if text.strip() else [] + + if len(sentences) <= 1: + # If single sentence, return it even if it exceeds max_size + # (we can't split sentences, so we must keep it whole) + return sentences + + windows = self._build_sentence_windows(sentences, window_size=5) + vectors = self._get_embeddings(windows) + labels = self._cluster_sentences(vectors, sentences) + + # Process sentences in original order, grouping consecutive sentences + # from the same cluster into chunks + chunks = [] + i = 0 + while i < len(sentences): + label = labels[i] + + # Collect consecutive sentences with the same label + cluster_sentences = [sentences[i]] + cluster_delimiters = [delimiters[i] if i < len(delimiters) else ""] + i += 1 + + while i < len(sentences) and labels[i] == label: + cluster_sentences.append(sentences[i]) + cluster_delimiters.append(delimiters[i] if i < len(delimiters) else "") + i += 1 + + # Process this cluster + if label == -1: + # Noise cluster: each sentence becomes its own chunk + for idx, sentence in enumerate(cluster_sentences): + chunk = sentence + if idx < len(cluster_delimiters) and cluster_delimiters[idx]: + chunk += cluster_delimiters[idx] + chunks.append(chunk) + else: + # Regular cluster: group sentences respecting max_size + cluster_len = sum(len(s) for s in cluster_sentences) + delimiter_len = sum(len(d) for d in cluster_delimiters) + total_cluster_len = cluster_len + delimiter_len + + if total_cluster_len <= self.max_size: + # Cluster fits in one chunk + chunk_parts = [] + for j, sentence in enumerate(cluster_sentences): + chunk_parts.append(sentence) + if j < len(cluster_delimiters) and cluster_delimiters[j]: + chunk_parts.append(cluster_delimiters[j]) + chunks.append("".join(chunk_parts)) + else: + # Split cluster into multiple chunks + current_chunk = [] + current_delims = [] + current_len = 0 + + for j, sentence in enumerate(cluster_sentences): + sentence_len = len(sentence) + delim = ( + cluster_delimiters[j] if j < len(cluster_delimiters) else "" + ) + delim_len = len(delim) + + if current_len + sentence_len + delim_len > self.max_size: + # Current chunk is full + if current_chunk: + chunk_parts = [] + for k, s in enumerate(current_chunk): + chunk_parts.append(s) + if k < len(current_delims) and current_delims[k]: + chunk_parts.append(current_delims[k]) + chunks.append("".join(chunk_parts)) + current_chunk = [sentence] + current_delims = [delim] + current_len = sentence_len + delim_len + else: + current_chunk.append(sentence) + current_delims.append(delim) + current_len += sentence_len + delim_len + + # Add remaining chunk + if current_chunk: + chunk_parts = [] + for k, s in enumerate(current_chunk): + chunk_parts.append(s) + if k < len(current_delims) and current_delims[k]: + chunk_parts.append(current_delims[k]) + chunks.append("".join(chunk_parts)) + + return self._merge_small_chunks(chunks) + + def _merge_small_chunks(self, chunks: List[str]) -> List[str]: + """Greedy merge chunks that fall below min_size. + + Ensures no chunk exceeds max_size after merging. + Note: Chunks must contain whole sentences, so we merge at sentence boundaries. + """ + merged = [] + if not chunks: + return [] + + current = chunks[0] + for next_chunk in chunks[1:]: + # Calculate merged length - join without extra separator since chunks + # already contain their delimiters + merged_len = len(current) + len(next_chunk) + + if len(current) < self.min_size and merged_len <= self.max_size: + # Merge chunks (both contain whole sentences, so result is valid) + current += next_chunk + else: + # Can't merge without exceeding max_size, so keep current chunk + # Note: current chunk might exceed max_size if it's a single long sentence, + # but we can't split sentences, so we keep it as-is + merged.append(current) + current = next_chunk + + # Handle last chunk + merged.append(current) + + return merged + + def transform_documents( + self, documents: Sequence[Document], **kwargs: Any + ) -> Sequence[Document]: + return self.split_documents(list(documents)) + + def create_documents( + self, texts: List[str], metadatas: List[dict] | None = None + ) -> List[Document]: + _metadatas = metadatas or [{}] * len(texts) + documents = [] + for i, text in enumerate(texts): + for chunk in self.split_text(text): + metadata = copy.deepcopy(_metadatas[i]) + documents.append(Document(page_content=chunk, metadata=metadata)) + return documents + + def split_documents(self, documents: Iterable[Document]) -> List[Document]: + texts = [] + metadatas = [] + for doc in documents: + texts.append(doc.page_content) + metadatas.append(doc.metadata) + return self.create_documents(texts, metadatas) diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/converter.py b/ontology_platform/vendored/ontocast/ontocast/tool/converter.py new file mode 100644 index 0000000..4c5ef76 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/converter.py @@ -0,0 +1,134 @@ +"""Document conversion tools for OntoCast. + +This module provides functionality for converting various document formats +into structured data that can be processed by the OntoCast system. +""" + +import importlib +import logging +import pathlib +import threading +from io import BytesIO +from typing import Any, Union + +from pydantic import Field + +from .cache import Cacher, ToolCacher +from .onto import Tool + +logger = logging.getLogger(__name__) + + +class ConverterTool(Tool): + """Tool for converting documents to structured data. + + This class provides functionality for converting various document formats + into structured data that can be processed by the OntoCast system. + It includes caching to avoid re-converting the same documents. + + Attributes: + supported_extensions: Set of supported file extensions. + cache: Cacher instance for caching conversion results. + """ + + supported_extensions: set[str] = Field( + default={".pdf", ".ppt", ".pptx"}, + description="Set of supported file extensions", + ) + cache: Any = Field(default=None, exclude=True) + + def __init__( + self, + cache: Cacher | None = None, + **kwargs, + ): + """Initialize the converter tool. + + Args: + cache: Optional shared Cacher instance. If None, creates a new one. + **kwargs: Additional keyword arguments passed to the parent class. + """ + super().__init__(**kwargs) + self._converter = None + self._converter_lock = threading.Lock() # Lock for thread-safe converter access + + # Initialize cache - use shared cacher or create new one + if cache is not None: + self.cache = ToolCacher(cache, "converter") + else: + # Fallback for backward compatibility + shared_cache = Cacher() + self.cache = ToolCacher(shared_cache, "converter") + + try: + document_converter_module = importlib.import_module( + "docling.document_converter" + ) + DocumentConverter = getattr(document_converter_module, "DocumentConverter") + self._converter = DocumentConverter() + except ImportError as e: + logger.error(f"Could not import DocumentConverter: {e}") + + def __call__(self, file_input: Union[bytes, str, pathlib.Path]) -> dict[str, Any]: + """Convert a document to structured data. + + Args: + file_input: The input file as either bytes, string, or pathlib.Path. + + Returns: + dict[str, Any]: The converted document data. + """ + # For plain text input, no caching needed + if isinstance(file_input, str): + return {"text": file_input} + + # Prepare content for caching + if isinstance(file_input, bytes): + content_for_cache = file_input + elif isinstance(file_input, pathlib.Path): + content_for_cache = file_input.read_bytes() + else: + # Fallback for other types + return {"text": str(file_input)} + + # Check cache first + cached_result = self.cache.get(content_for_cache) + if cached_result is not None: + logger.debug("Cache hit for document conversion") + return cached_result + + # Convert document (with thread-safe access to converter) + with self._converter_lock: + if isinstance(file_input, bytes): + if self._converter is None: + raise ImportError("DocumentConverter not available") + try: + base_models_module = importlib.import_module( + "docling.datamodel.base_models" + ) + DocumentStream = getattr(base_models_module, "DocumentStream") + ds = DocumentStream(name="doc", stream=BytesIO(file_input)) + except ImportError: + raise ImportError( + f"Could not import DocumentConverter: {file_input}" + ) + result = self._converter.convert(ds) + doc = result.document.export_to_markdown() + converted_result = {"text": doc} + elif isinstance(file_input, pathlib.Path): + if self._converter is None: + raise ImportError( + f"Could not import DocumentConverter: {file_input}" + ) + result = self._converter.convert(file_input) + doc = result.document.export_to_markdown() + converted_result = {"text": doc} + else: + # Fallback for other types + converted_result = {"text": str(file_input)} + + # Cache the result + self.cache.set(content_for_cache, converted_result) + logger.debug("Cached document conversion result") + + return converted_result diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/graph_diff.py b/ontology_platform/vendored/ontocast/ontocast/tool/graph_diff.py new file mode 100644 index 0000000..0ab6853 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/graph_diff.py @@ -0,0 +1,375 @@ +"""Graph diff generation and application system. + +This module provides functionality for generating and applying diffs between +graph versions, enabling incremental updates and efficient context passing. +""" + +import logging +from datetime import datetime +from enum import StrEnum +from typing import Any + +from pydantic import BaseModel, Field + +from ontocast.onto.rdfgraph import RDFGraph + +logger = logging.getLogger(__name__) + + +class DiffOperation(StrEnum): + """Enumeration of diff operations.""" + + ADD = "add" + REMOVE = "remove" + MODIFY = "modify" + UNCHANGED = "unchanged" + + +class TripleDiff(BaseModel): + """Represents a diff for a single triple.""" + + subject: str = Field(description="Subject of the triple") + predicate: str = Field(description="Predicate of the triple") + object: str = Field(description="Object of the triple") + operation: DiffOperation = Field(description="Operation performed on this triple") + old_value: str | None = Field( + default=None, description="Old value for modifications" + ) + new_value: str | None = Field( + default=None, description="New value for modifications" + ) + metadata: dict[str, Any] = Field( + default_factory=dict, description="Additional metadata" + ) + + +class GraphDiff(BaseModel): + """Represents differences between two graph versions.""" + + # Diff metadata + diff_id: str = Field(description="Unique identifier for this diff") + source_version_id: str = Field(description="ID of the source version") + target_version_id: str = Field(description="ID of the target version") + created_at: datetime = Field( + default_factory=datetime.now, description="When this diff was created" + ) + + # Diff content + triple_diffs: list[TripleDiff] = Field( + default_factory=list, description="List of triple differences" + ) + + # Summary statistics + added_triples: int = Field(default=0, description="Number of added triples") + removed_triples: int = Field(default=0, description="Number of removed triples") + modified_triples: int = Field(default=0, description="Number of modified triples") + unchanged_triples: int = Field(default=0, description="Number of unchanged triples") + + # Context information + context_metadata: dict[str, Any] = Field( + default_factory=dict, description="Context metadata for this diff" + ) + + def get_summary(self) -> str: + """Get a summary of this diff. + + Returns: + str: Human-readable summary of the diff + """ + return f""" +Graph Diff Summary: +- Diff ID: {self.diff_id} +- Source: {self.source_version_id} → Target: {self.target_version_id} +- Created: {self.created_at.isoformat()} +- Added: {self.added_triples} triples +- Removed: {self.removed_triples} triples +- Modified: {self.modified_triples} triples +- Unchanged: {self.unchanged_triples} triples +- Total changes: {self.added_triples + self.removed_triples + self.modified_triples} +""" + + def get_sparql_operations(self) -> list[str]: + """Get SPARQL operations for applying this diff. + + Returns: + list[str]: List of SPARQL queries to apply the diff + """ + operations = [] + + for triple_diff in self.triple_diffs: + if triple_diff.operation == DiffOperation.ADD: + # Generate INSERT operation + sparql = f"INSERT DATA {{ {triple_diff.subject} {triple_diff.predicate} {triple_diff.object} . }}" + operations.append(sparql) + + elif triple_diff.operation == DiffOperation.REMOVE: + # Generate DELETE operation + sparql = f"DELETE DATA {{ {triple_diff.subject} {triple_diff.predicate} {triple_diff.object} . }}" + operations.append(sparql) + + elif triple_diff.operation == DiffOperation.MODIFY: + # Generate DELETE + INSERT for modifications + if triple_diff.old_value: + delete_sparql = f"DELETE DATA {{ {triple_diff.subject} {triple_diff.predicate} {triple_diff.old_value} . }}" + operations.append(delete_sparql) + if triple_diff.new_value: + insert_sparql = f"INSERT DATA {{ {triple_diff.subject} {triple_diff.predicate} {triple_diff.new_value} . }}" + operations.append(insert_sparql) + + return operations + + def is_empty(self) -> bool: + """Check if this diff is empty (no changes). + + Returns: + bool: True if no changes, False otherwise + """ + return ( + self.added_triples == 0 + and self.removed_triples == 0 + and self.modified_triples == 0 + ) + + def get_changed_subjects(self) -> set[str]: + """Get all subjects that have changes. + + Returns: + set[str]: Set of subject URIs that have changes + """ + return { + triple_diff.subject + for triple_diff in self.triple_diffs + if triple_diff.operation != DiffOperation.UNCHANGED + } + + def get_changed_predicates(self) -> set[str]: + """Get all predicates that have changes. + + Returns: + set[str]: Set of predicate URIs that have changes + """ + return { + triple_diff.predicate + for triple_diff in self.triple_diffs + if triple_diff.operation != DiffOperation.UNCHANGED + } + + +class DiffTool: + """Tool for generating and applying graph diffs.""" + + def __init__(self): + """Initialize the diff tool.""" + self.logger = logging.getLogger(__name__) + + def generate_diff( + self, + source_graph: RDFGraph, + target_graph: RDFGraph, + source_version_id: str, + target_version_id: str, + context_metadata: dict[str, Any] | None = None, + ) -> GraphDiff: + """Generate a diff between two graphs. + + Args: + source_graph: The source graph to compare from + target_graph: The target graph to compare to + source_version_id: ID of the source version + target_version_id: ID of the target version + context_metadata: Optional context metadata + + Returns: + GraphDiff: The generated diff + """ + self.logger.info( + f"Generating diff from {source_version_id} to {target_version_id}" + ) + + # Get triples from both graphs + source_triples = self._get_triples_set(source_graph) + target_triples = self._get_triples_set(target_graph) + + # Find differences + triple_diffs = [] + added_triples = 0 + removed_triples = 0 + modified_triples = 0 + unchanged_triples = 0 + + # Find added triples + for triple in target_triples - source_triples: + triple_diff = TripleDiff( + subject=triple[0], + predicate=triple[1], + object=triple[2], + operation=DiffOperation.ADD, + ) + triple_diffs.append(triple_diff) + added_triples += 1 + + # Find removed triples + for triple in source_triples - target_triples: + triple_diff = TripleDiff( + subject=triple[0], + predicate=triple[1], + object=triple[2], + operation=DiffOperation.REMOVE, + ) + triple_diffs.append(triple_diff) + removed_triples += 1 + + # Find unchanged triples + for triple in source_triples & target_triples: + triple_diff = TripleDiff( + subject=triple[0], + predicate=triple[1], + object=triple[2], + operation=DiffOperation.UNCHANGED, + ) + triple_diffs.append(triple_diff) + unchanged_triples += 1 + + # Create diff + diff = GraphDiff( + diff_id=f"diff_{source_version_id}_{target_version_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}", + source_version_id=source_version_id, + target_version_id=target_version_id, + triple_diffs=triple_diffs, + added_triples=added_triples, + removed_triples=removed_triples, + modified_triples=modified_triples, + unchanged_triples=unchanged_triples, + context_metadata=context_metadata or {}, + ) + + self.logger.info( + f"Generated diff with {added_triples} additions, {removed_triples} removals, {modified_triples} modifications" + ) + return diff + + def apply_diff(self, graph: RDFGraph, diff: GraphDiff) -> RDFGraph: + """Apply a diff to a graph. + + Args: + graph: The graph to apply the diff to + diff: The diff to apply + + Returns: + RDFGraph: The updated graph + """ + self.logger.info(f"Applying diff {diff.diff_id} to graph") + + # Create a copy of the graph + updated_graph = RDFGraph() + updated_graph += graph + + # Apply each triple diff + for triple_diff in diff.triple_diffs: + if triple_diff.operation == DiffOperation.ADD: + # Add the triple + updated_graph.add_triple( + triple_diff.subject, + triple_diff.predicate, + triple_diff.object, + ) + + elif triple_diff.operation == DiffOperation.REMOVE: + # Remove the triple + updated_graph.remove_triple( + triple_diff.subject, + triple_diff.predicate, + triple_diff.object, + ) + + elif triple_diff.operation == DiffOperation.MODIFY: + # Remove old, add new + if triple_diff.old_value: + updated_graph.remove_triple( + triple_diff.subject, + triple_diff.predicate, + triple_diff.old_value, + ) + if triple_diff.new_value: + updated_graph.add_triple( + triple_diff.subject, + triple_diff.predicate, + triple_diff.new_value, + ) + + self.logger.info( + f"Applied diff to graph, new triple count: {len(updated_graph)}" + ) + return updated_graph + + def _get_triples_set(self, graph: RDFGraph) -> set[tuple[str, str, str]]: + """Get a set of triples from a graph. + + Args: + graph: The graph to extract triples from + + Returns: + set[tuple[str, str, str]]: Set of (subject, predicate, object) tuples + """ + triples = set() + for triple in graph: + triples.add((str(triple[0]), str(triple[1]), str(triple[2]))) + return triples + + def get_diff_summary(self, diff: GraphDiff) -> str: + """Get a human-readable summary of a diff. + + Args: + diff: The diff to summarize + + Returns: + str: Human-readable summary + """ + return f""" +Diff Summary: +- ID: {diff.diff_id} +- Source: {diff.source_version_id} → Target: {diff.target_version_id} +- Created: {diff.created_at.isoformat()} +- Changes: {diff.added_triples} added, {diff.removed_triples} removed, {diff.modified_triples} modified +- Total triples: {len(diff.triple_diffs)} +- Changed subjects: {len(diff.get_changed_subjects())} +- Changed predicates: {len(diff.get_changed_predicates())} +""" + + def merge_diffs(self, diffs: list[GraphDiff]) -> GraphDiff: + """Merge multiple diffs into a single diff. + + Args: + diffs: List of diffs to merge + + Returns: + GraphDiff: Merged diff + """ + if not diffs: + raise ValueError("Cannot merge empty list of diffs") + + if len(diffs) == 1: + return diffs[0] + + # Start with the first diff + merged_diff = diffs[0] + + # Merge each subsequent diff + for diff in diffs[1:]: + # Combine triple diffs + merged_diff.triple_diffs.extend(diff.triple_diffs) + + # Update statistics + merged_diff.added_triples += diff.added_triples + merged_diff.removed_triples += diff.removed_triples + merged_diff.modified_triples += diff.modified_triples + merged_diff.unchanged_triples += diff.unchanged_triples + + # Update target version + merged_diff.target_version_id = diff.target_version_id + + # Update diff ID + merged_diff.diff_id = f"merged_{merged_diff.source_version_id}_{merged_diff.target_version_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + + self.logger.info(f"Merged {len(diffs)} diffs into single diff") + return merged_diff diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/graph_version_manager.py b/ontology_platform/vendored/ontocast/ontocast/tool/graph_version_manager.py new file mode 100644 index 0000000..05bf762 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/graph_version_manager.py @@ -0,0 +1,443 @@ +"""Graph version manager for tracking ontology and facts graph changes. + +This module provides functionality for managing versions of RDF graphs, +enabling incremental updates and change tracking. +""" + +import logging +from collections import defaultdict +from datetime import datetime +from typing import Any + +from pydantic import BaseModel, Field + +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.sparql_models import SPARQLOperationModel + +logger = logging.getLogger(__name__) + + +class VersionDetails(BaseModel): + """Pydantic model for version details in statistics.""" + + version_count: int = Field(description="Number of versions") + latest_size: int = Field(description="Size of the latest version") + latest_timestamp: str | None = Field(description="Timestamp of the latest version") + + +class VersionStatistics(BaseModel): + """Pydantic model for version statistics.""" + + total_ontologies: int = Field(description="Total number of ontologies") + total_chunks: int = Field(description="Total number of chunks") + total_ontology_versions: int = Field( + description="Total number of ontology versions" + ) + total_facts_versions: int = Field(description="Total number of facts versions") + ontology_details: dict[str, VersionDetails] = Field( + description="Details for each ontology" + ) + chunk_details: dict[str, VersionDetails] = Field( + description="Details for each chunk" + ) + + +class GraphVersion(BaseModel): + """Represents a version of a graph.""" + + id: str = Field(description="Unique identifier for this graph version") + graph: RDFGraph = Field(description="The RDF graph for this version") + timestamp: datetime = Field(description="When this version was created") + operations: list[SPARQLOperationModel] = Field( + default_factory=list, + description="List of SPARQL operations that created this version", + ) + metadata: dict[str, Any] = Field( + default_factory=dict, description="Optional metadata for this version" + ) + parent_version_id: str | None = Field( + default=None, description="ID of the parent version this was derived from" + ) + + def get_size(self) -> int: + """Get the number of triples in this version.""" + return len(self.graph) + + def get_namespaces(self) -> dict[str, str]: + """Get the namespaces bound in this version.""" + return dict(self.graph.namespaces()) + + +class GraphDiff(BaseModel): + """Represents differences between two graph versions.""" + + added_triples: list[tuple] = Field( + default_factory=list, description="List of triples that were added" + ) + removed_triples: list[tuple] = Field( + default_factory=list, description="List of triples that were removed" + ) + modified_triples: list[tuple[tuple, tuple]] = Field( + default_factory=list, + description="List of (old, new) triple pairs that were modified", + ) + added_namespaces: dict[str, str] = Field( + default_factory=dict, description="Namespaces that were added" + ) + removed_namespaces: dict[str, str] = Field( + default_factory=dict, description="Namespaces that were removed" + ) + + def is_empty(self) -> bool: + """Check if the diff is empty (no changes).""" + return ( + len(self.added_triples) == 0 + and len(self.removed_triples) == 0 + and len(self.modified_triples) == 0 + and len(self.added_namespaces) == 0 + and len(self.removed_namespaces) == 0 + ) + + +class GraphVersionManager: + """Manages versions of ontology and facts graphs.""" + + def __init__(self): + """Initialize the graph version manager.""" + self.ontology_versions: dict[str, list[GraphVersion]] = defaultdict(list) + self.facts_versions: dict[str, list[GraphVersion]] = defaultdict(list) + self.version_metadata: dict[str, dict[str, Any]] = {} + + def create_ontology_version( + self, + ontology_id: str, + graph: RDFGraph, + operations: list[SPARQLOperationModel] | None = None, + metadata: dict[str, Any] | None = None, + ) -> GraphVersion: + """Create a new version of an ontology. + + Args: + ontology_id: Unique identifier for the ontology. + graph: The RDF graph for this version. + operations: SPARQL operations that created this version. + metadata: Additional metadata for this version. + + Returns: + GraphVersion: The created version. + """ + version_number = len(self.ontology_versions[ontology_id]) + 1 + version_id = f"{ontology_id}_v{version_number}" + + # Get parent version if it exists + parent_version_id = None + if self.ontology_versions[ontology_id]: + parent_version_id = self.ontology_versions[ontology_id][-1].id + + version = GraphVersion( + id=version_id, + graph=graph, + timestamp=datetime.now(), + operations=operations or [], + metadata=metadata or {}, + parent_version_id=parent_version_id, + ) + + self.ontology_versions[ontology_id].append(version) + logger.info(f"Created ontology version {version_id} with {len(graph)} triples") + + return version + + def create_facts_version( + self, + chunk_id: str, + graph: RDFGraph, + operations: list[SPARQLOperationModel] | None = None, + metadata: dict[str, Any] | None = None, + ) -> GraphVersion: + """Create a new version of facts for a chunk. + + Args: + chunk_id: Unique identifier for the chunk. + graph: The RDF graph for this version. + operations: SPARQL operations that created this version. + metadata: Additional metadata for this version. + + Returns: + GraphVersion: The created version. + """ + version_number = len(self.facts_versions[chunk_id]) + 1 + version_id = f"{chunk_id}_v{version_number}" + + # Get parent version if it exists + parent_version_id = None + if self.facts_versions[chunk_id]: + parent_version_id = self.facts_versions[chunk_id][-1].id + + version = GraphVersion( + id=version_id, + graph=graph, + timestamp=datetime.now(), + operations=operations or [], + metadata=metadata or {}, + parent_version_id=parent_version_id, + ) + + self.facts_versions[chunk_id].append(version) + logger.info(f"Created facts version {version_id} with {len(graph)} triples") + + return version + + def get_latest_ontology_version(self, ontology_id: str) -> GraphVersion | None: + """Get the latest version of an ontology. + + Args: + ontology_id: The ontology identifier. + + Returns: + GraphVersion: The latest version, or None if not found. + """ + versions = self.ontology_versions.get(ontology_id, []) + return versions[-1] if versions else None + + def get_latest_facts_version(self, chunk_id: str) -> GraphVersion | None: + """Get the latest version of facts for a chunk. + + Args: + chunk_id: The chunk identifier. + + Returns: + GraphVersion: The latest version, or None if not found. + """ + versions = self.facts_versions.get(chunk_id, []) + return versions[-1] if versions else None + + def get_ontology_version( + self, ontology_id: str, version_index: int + ) -> GraphVersion | None: + """Get a specific version of an ontology. + + Args: + ontology_id: The ontology identifier. + version_index: The version index (0-based). + + Returns: + GraphVersion: The requested version, or None if not found. + """ + versions = self.ontology_versions.get(ontology_id, []) + if 0 <= version_index < len(versions): + return versions[version_index] + return None + + def get_facts_version( + self, chunk_id: str, version_index: int + ) -> GraphVersion | None: + """Get a specific version of facts for a chunk. + + Args: + chunk_id: The chunk identifier. + version_index: The version index (0-based). + + Returns: + GraphVersion: The requested version, or None if not found. + """ + versions = self.facts_versions.get(chunk_id, []) + if 0 <= version_index < len(versions): + return versions[version_index] + return None + + def calculate_ontology_diff( + self, ontology_id: str, from_version: int, to_version: int + ) -> GraphDiff: + """Calculate differences between two ontology versions. + + Args: + ontology_id: The ontology identifier. + from_version: The source version index. + to_version: The target version index. + + Returns: + GraphDiff: The differences between versions. + """ + from_ver = self.get_ontology_version(ontology_id, from_version) + to_ver = self.get_ontology_version(ontology_id, to_version) + + if not from_ver or not to_ver: + raise ValueError(f"Invalid version indices for ontology {ontology_id}") + + return self._calculate_graph_diff(from_ver.graph, to_ver.graph) + + def calculate_facts_diff( + self, chunk_id: str, from_version: int, to_version: int + ) -> GraphDiff: + """Calculate differences between two facts versions. + + Args: + chunk_id: The chunk identifier. + from_version: The source version index. + to_version: The target version index. + + Returns: + GraphDiff: The differences between versions. + """ + from_ver = self.get_facts_version(chunk_id, from_version) + to_ver = self.get_facts_version(chunk_id, to_version) + + if not from_ver or not to_ver: + raise ValueError(f"Invalid version indices for chunk {chunk_id}") + + return self._calculate_graph_diff(from_ver.graph, to_ver.graph) + + def _calculate_graph_diff( + self, from_graph: RDFGraph, to_graph: RDFGraph + ) -> GraphDiff: + """Calculate differences between two graphs. + + Args: + from_graph: The source graph. + to_graph: The target graph. + + Returns: + GraphDiff: The differences between graphs. + """ + from_triples = set(from_graph) + to_triples = set(to_graph) + + added_triples = list(to_triples - from_triples) + removed_triples = list(from_triples - to_triples) + + # For modified triples, we need to identify triples that changed + # This is a simplified approach - in practice, you might want more sophisticated matching + modified_triples = [] + + # Get namespace differences + from_namespaces = {k: str(v) for k, v in from_graph.namespaces()} + to_namespaces = {k: str(v) for k, v in to_graph.namespaces()} + + added_namespaces = { + k: v for k, v in to_namespaces.items() if k not in from_namespaces + } + removed_namespaces = { + k: v for k, v in from_namespaces.items() if k not in to_namespaces + } + + return GraphDiff( + added_triples=added_triples, + removed_triples=removed_triples, + modified_triples=modified_triples, + added_namespaces=added_namespaces, + removed_namespaces=removed_namespaces, + ) + + def get_ontology_version_count(self, ontology_id: str) -> int: + """Get the number of versions for an ontology. + + Args: + ontology_id: The ontology identifier. + + Returns: + int: The number of versions. + """ + return len(self.ontology_versions.get(ontology_id, [])) + + def get_facts_version_count(self, chunk_id: str) -> int: + """Get the number of versions for facts in a chunk. + + Args: + chunk_id: The chunk identifier. + + Returns: + int: The number of versions. + """ + return len(self.facts_versions.get(chunk_id, [])) + + def get_all_ontology_ids(self) -> list[str]: + """Get all ontology identifiers. + + Returns: + list[str]: All ontology identifiers. + """ + return list(self.ontology_versions.keys()) + + def get_all_chunk_ids(self) -> list[str]: + """Get all chunk identifiers. + + Returns: + list[str]: All chunk identifiers. + """ + return list(self.facts_versions.keys()) + + def delete_ontology_versions(self, ontology_id: str, keep_latest: bool = True): + """Delete all versions of an ontology. + + Args: + ontology_id: The ontology identifier. + keep_latest: If True, keep only the latest version. + """ + if ontology_id in self.ontology_versions: + if keep_latest and len(self.ontology_versions[ontology_id]) > 1: + # Keep only the latest version + latest_version = self.ontology_versions[ontology_id][-1] + self.ontology_versions[ontology_id] = [latest_version] + logger.info(f"Deleted all but latest version of ontology {ontology_id}") + else: + del self.ontology_versions[ontology_id] + logger.info(f"Deleted all versions of ontology {ontology_id}") + + def delete_facts_versions(self, chunk_id: str, keep_latest: bool = True): + """Delete all versions of facts for a chunk. + + Args: + chunk_id: The chunk identifier. + keep_latest: If True, keep only the latest version. + """ + if chunk_id in self.facts_versions: + if keep_latest and len(self.facts_versions[chunk_id]) > 1: + # Keep only the latest version + latest_version = self.facts_versions[chunk_id][-1] + self.facts_versions[chunk_id] = [latest_version] + logger.info( + f"Deleted all but latest version of facts for chunk {chunk_id}" + ) + else: + del self.facts_versions[chunk_id] + logger.info(f"Deleted all versions of facts for chunk {chunk_id}") + + def get_version_statistics(self) -> VersionStatistics: + """Get statistics about all versions. + + Returns: + VersionStatistics: Version statistics. + """ + ontology_details = {} + for ontology_id, versions in self.ontology_versions.items(): + ontology_details[ontology_id] = VersionDetails( + version_count=len(versions), + latest_size=versions[-1].get_size() if versions else 0, + latest_timestamp=versions[-1].timestamp.isoformat() + if versions + else None, + ) + + chunk_details = {} + for chunk_id, versions in self.facts_versions.items(): + chunk_details[chunk_id] = VersionDetails( + version_count=len(versions), + latest_size=versions[-1].get_size() if versions else 0, + latest_timestamp=versions[-1].timestamp.isoformat() + if versions + else None, + ) + + return VersionStatistics( + total_ontologies=len(self.ontology_versions), + total_chunks=len(self.facts_versions), + total_ontology_versions=sum( + len(versions) for versions in self.ontology_versions.values() + ), + total_facts_versions=sum( + len(versions) for versions in self.facts_versions.values() + ), + ontology_details=ontology_details, + chunk_details=chunk_details, + ) diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/llm.py b/ontology_platform/vendored/ontocast/ontocast/tool/llm.py new file mode 100644 index 0000000..a5db242 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/llm.py @@ -0,0 +1,470 @@ +"""Language Model (LLM) integration tool for OntoCast. + +This module provides integration with various language models through LangChain, +supporting both OpenAI and Ollama providers. It enables text generation and +structured data extraction capabilities with optional caching support. + +Cache Usage: + The LLM tool supports caching of responses to avoid redundant API calls. + Caching uses a shared Cacher instance that manages cache directories for all tools. + The cache directory is managed by the shared Cacher class and follows these rules: + + ```python + from ontocast.tool.llm import LLMTool + from ontocast.config import LLMConfig + from ontocast.tool.cache import Cacher + + # Create shared cache instance + shared_cache = Cacher() + + # Create LLM tool with shared cache + llm_tool = await LLMTool.acreate( + config=LLMConfig(...), + cache=shared_cache + ) + ``` + + Default cache locations: + - Tests: .test_cache/llm/ in the current working directory + - Windows: %USERPROFILE%\\AppData\\Local\\ontocast\\llm\ + - Unix/Linux: ~/.cache/ontocast/llm/ (or $XDG_CACHE_HOME/ontocast/llm/) + + Cache files are stored as JSON files with filenames based on SHA256 hashes + of the prompt and LLM configuration. This ensures that identical prompts + with the same configuration will return cached responses. + + The shared Cacher automatically manages subdirectories for different tools, + ensuring organized cache storage while maintaining a single cache instance. +""" + +import asyncio +import logging +from functools import wraps +from typing import Any, Callable, Type, TypeVar + +from langchain_core.language_models import BaseChatModel +from langchain_core.messages.ai import AIMessage +from langchain_core.output_parsers import PydanticOutputParser +from langchain_ollama import ChatOllama +from langchain_openai import ChatOpenAI +from pydantic import BaseModel, Field, SecretStr + +from ontocast.config import LLMConfig, LLMProvider + +from .cache import Cacher, ToolCacher +from .onto import Tool + +T = TypeVar("T", bound=BaseModel) + +logger = logging.getLogger(__name__) + + +def track_llm_usage(func: Callable) -> Callable: + """Decorator to track LLM usage automatically.""" + + @wraps(func) + def wrapper(self, *args, **kwargs): + # Get prompt for character counting + prompt = args[0] if args else "" + prompt_str = ( + self._prompt_to_string(prompt) + if hasattr(self, "_prompt_to_string") + else str(prompt) + ) + + # Call the original function + result = func(self, *args, **kwargs) + + # Track usage if budget tracker is available in the tool + if hasattr(self, "budget_tracker") and self.budget_tracker is not None: + chars_sent = len(prompt_str) + chars_received = ( + len(result.content) + if hasattr(result, "content") and result.content + else 0 + ) + self.budget_tracker.add_usage(chars_sent, chars_received) + + return result + + @wraps(func) + async def async_wrapper(self, *args, **kwargs): + # Get prompt for character counting + prompt = args[0] if args else "" + prompt_str = ( + self._prompt_to_string(prompt) + if hasattr(self, "_prompt_to_string") + else str(prompt) + ) + + # Call the original function + result = await func(self, *args, **kwargs) + + # Track usage if budget tracker is available in the tool + if hasattr(self, "budget_tracker") and self.budget_tracker is not None: + chars_sent = len(prompt_str) + chars_received = ( + len(result.content) + if hasattr(result, "content") and result.content + else len(str(result)) + ) + self.budget_tracker.add_usage(chars_sent, chars_received) + + return result + + return async_wrapper if asyncio.iscoroutinefunction(func) else wrapper + + +class LLMTool(Tool): + """Tool for interacting with language models. + + This class provides a unified interface for working with different language model + providers (OpenAI, Ollama) through LangChain. It supports both synchronous and + asynchronous operations. + + Attributes: + config: LLMConfig object containing all LLM settings. + cache: Cacher instance for caching LLM responses. + """ + + config: LLMConfig = Field(default_factory=LLMConfig) + cache: Any = Field(default=None, exclude=True) + budget_tracker: Any = Field(default=None, exclude=True) + + def __init__( + self, + cache: Cacher | None = None, + budget_tracker: Any = None, + **kwargs, + ): + """Initialize the LLM tool. + + Args: + cache: Optional shared Cacher instance. If None, creates a new one. + budget_tracker: Optional budget tracker instance for usage statistics. + **kwargs: Additional keyword arguments passed to the parent class. + """ + super().__init__(**kwargs) + self._llm = None + self.budget_tracker = budget_tracker + + # Initialize cache - use shared cacher or create new one + if cache is not None: + self.cache = ToolCacher(cache, "llm") + else: + # Fallback for backward compatibility + shared_cache = Cacher() + self.cache = ToolCacher(shared_cache, "llm") + + @classmethod + def create( + cls, + config: LLMConfig, + cache: Cacher | None = None, + budget_tracker: Any = None, + **kwargs, + ): + """Create a new LLM tool instance synchronously. + + Args: + config: LLMConfig object containing LLM settings. + cache: Optional shared Cacher instance. + budget_tracker: Optional budget tracker instance for usage statistics. + **kwargs: Additional keyword arguments for initialization. + + Returns: + LLMTool: A new instance of the LLM tool. + """ + return asyncio.run( + cls.acreate( + config=config, cache=cache, budget_tracker=budget_tracker, **kwargs + ) + ) + + @classmethod + async def acreate( + cls, + config: LLMConfig, + cache: Cacher | None = None, + budget_tracker: Any = None, + **kwargs, + ): + """Create a new LLM tool instance asynchronously. + + Args: + config: LLMConfig object containing LLM settings. + cache: Optional shared Cacher instance. + budget_tracker: Optional budget tracker instance for usage statistics. + **kwargs: Additional keyword arguments for initialization. + + Returns: + LLMTool: A new instance of the LLM tool. + """ + # Create and initialize the instance with the config + self = cls(config=config, cache=cache, budget_tracker=budget_tracker, **kwargs) + await self.setup() + return self + + async def setup(self): + """Set up the language model based on the configured provider. + + Raises: + ValueError: If the provider is not supported. + """ + if self.config.provider == LLMProvider.OPENAI: + if self.config.model_name.startswith("gpt-5"): + self.config.temperature = 1.0 + logger.warning( + f"Setting temperature to {self.config.temperature} for gpt-5 class " + f"model {self.config.model_name}" + ) + self._llm = ChatOpenAI( + model=self.config.model_name, # type: ignore + temperature=self.config.temperature, + base_url=self.config.base_url, # type: ignore + api_key=( + SecretStr(self.config.api_key) if self.config.api_key else None + ), # type: ignore + ) + elif self.config.provider == LLMProvider.OLLAMA: + self._llm = ChatOllama( + model=self.config.model_name, + base_url=self.config.base_url, + temperature=self.config.temperature, + ) + else: + raise ValueError(f"Unsupported provider: {self.config.provider}") + + @track_llm_usage + async def __call__(self, *args: Any, **kwds: Any) -> Any: + """Call the language model directly (asynchronous). + + Args: + *args: Positional arguments passed to the LLM. + **kwds: Keyword arguments passed to the LLM. + + Returns: + Any: The LLM's response. + """ + # Extract prompt from args (first argument is typically the prompt) + prompt = args[0] if args else "" + + # Prepare configuration for caching + config_dict = { + "provider": self.config.provider, + "model_name": self.config.model_name, + "temperature": self.config.temperature, + "base_url": self.config.base_url, + } + + # Check cache first + cached_response = self.cache.get(prompt, config=config_dict, **kwds) + + if cached_response is not None: + prompt_str = self._prompt_to_string(prompt) + logger.debug(f"Cache hit for __call__: {prompt_str[:50]}...") + # Return a mock BaseMessage object with the cached content + content = cached_response["content"] + content_str = content if isinstance(content, str) else str(content) + return AIMessage(content=content_str) + + # Generate new response + prompt_str = self._prompt_to_string(prompt) + logger.debug( + f"Cache miss, calling LLM for __call__, prompt size {len(prompt_str[:50])}..." + ) + + response = await self.llm.ainvoke(*args, **kwds) + + # Cache the response + response_data = { + "content": response.content, + "prompt": self._prompt_to_string(prompt), + "kwargs": kwds, + } + self.cache.set(prompt, response_data, config=config_dict, **kwds) + + return response + + @track_llm_usage + async def acall(self, *args: Any, **kwds: Any) -> Any: + """Call the language model directly (asynchronous). + + Args: + *args: Positional arguments passed to the LLM. + **kwds: Keyword arguments passed to the LLM. + + Returns: + Any: The LLM's response. + """ + # Extract prompt from args (first argument is typically the prompt) + prompt = args[0] if args else "" + + # Prepare configuration for caching + config_dict = { + "provider": self.config.provider, + "model_name": self.config.model_name, + "temperature": self.config.temperature, + "base_url": self.config.base_url, + } + + # Check cache first + cached_response = self.cache.get(prompt, config=config_dict, **kwds) + + if cached_response is not None: + prompt_str = self._prompt_to_string(prompt) + logger.debug(f"Cache hit for acall: {prompt_str[:50]}...") + # Return a mock BaseMessage object with the cached content + content = cached_response["content"] + content_str = content if isinstance(content, str) else str(content) + return AIMessage(content=content_str) + + # Generate new response + prompt_str = self._prompt_to_string(prompt) + logger.debug(f"Cache miss, calling LLM for acall: {prompt_str[:50]}...") + + response = await self.llm.ainvoke(*args, **kwds) + + # Cache the response + response_data = { + "content": response.content, + "prompt": self._prompt_to_string(prompt), + "kwargs": kwds, + } + self.cache.set(prompt, response_data, config=config_dict, **kwds) + + return response + + @property + def llm(self) -> BaseChatModel: + """Get the underlying language model instance. + + Returns: + BaseChatModel: The configured language model. + + Raises: + RuntimeError: If the LLM has not been properly initialized. + """ + if self._llm is None: + raise RuntimeError( + "LLM resource not properly initialized. Call setup() first." + ) + return self._llm + + def _prompt_to_string(self, prompt) -> str: + """Convert various prompt types to string for caching. + + Args: + prompt: The prompt object (string, StringPromptValue, etc.) + + Returns: + str: String representation of the prompt. + """ + if isinstance(prompt, str): + return prompt + elif hasattr(prompt, "to_string"): + return prompt.to_string() + elif hasattr(prompt, "text"): + return prompt.text + elif hasattr(prompt, "content"): + return prompt.content + else: + return str(prompt) + + @track_llm_usage + async def complete(self, prompt: str, **kwargs) -> Any: + """Generate a completion for the given prompt. + + Args: + prompt: The input prompt for generation. + **kwargs: Additional keyword arguments for generation. + + Returns: + Any: The generated completion. + """ + # Prepare configuration for caching + config_dict = { + "provider": self.config.provider, + "model_name": self.config.model_name, + "temperature": self.config.temperature, + "base_url": self.config.base_url, + } + + # Check cache first + cached_response = self.cache.get(prompt, config=config_dict, **kwargs) + + if cached_response is not None: + logger.debug(f"Cache hit for prompt: {prompt[:50]}...") + content = cached_response["content"] + return content if isinstance(content, str) else str(content) + + # Generate new response + logger.debug(f"Cache miss, calling LLM for prompt: {prompt[:50]}...") + + response = await self.llm.ainvoke(prompt, **kwargs) + + # Cache the response + response_data = { + "content": response.content, + "prompt": self._prompt_to_string(prompt), + "kwargs": kwargs, + } + self.cache.set(prompt, response_data, config=config_dict, **kwargs) + + return response.content + + @track_llm_usage + async def extract(self, prompt: str, output_schema: Type[T], **kwargs) -> T: + """Extract structured data from the prompt according to a schema. + + Args: + prompt: The input prompt for extraction. + output_schema: The Pydantic model class defining the output structure. + **kwargs: Additional keyword arguments for extraction. + + Returns: + T: The extracted data conforming to the output schema. + """ + parser = PydanticOutputParser(pydantic_object=output_schema) + format_instructions = parser.get_format_instructions() + + full_prompt = f"{prompt}\n\n{format_instructions}" + + # Prepare configuration for caching + config_dict = { + "provider": self.config.provider, + "model_name": self.config.model_name, + "temperature": self.config.temperature, + "base_url": self.config.base_url, + "output_schema": output_schema.__name__, + } + + # Check cache first + cached_response = self.cache.get(full_prompt, config=config_dict, **kwargs) + + if cached_response is not None: + logger.debug(f"Cache hit for extraction: {prompt[:50]}...") + # Parse the cached content + content = cached_response["content"] + if isinstance(content, str): + return parser.parse(content) + else: + # Fallback: convert to string if it's not already + return parser.parse(str(content)) + + # Generate new response + logger.debug(f"Cache miss, calling LLM for extraction: {prompt[:50]}...") + + response = await self.llm.ainvoke(full_prompt, **kwargs) + + # Cache the response + response_data = { + "content": response.content, + "prompt": self._prompt_to_string(full_prompt), + "output_schema": output_schema.__name__, + "kwargs": kwargs, + } + self.cache.set(full_prompt, response_data, config=config_dict, **kwargs) + + content = response.content + return parser.parse(content if isinstance(content, str) else str(content)) diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/onto.py b/ontology_platform/vendored/ontocast/ontocast/tool/onto.py new file mode 100644 index 0000000..94828c2 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/onto.py @@ -0,0 +1,71 @@ +"""Base tool class for OntoCast tools. + +This module provides the base Tool class that serves as a foundation for all +tools in the OntoCast system. It provides common functionality and interface +for tool implementations. +""" + +from pydantic import BaseModel, Field +from rdflib import URIRef + +from ontocast.onto.model import BasePydanticModel + + +class Tool(BasePydanticModel): + """Base class for all OntoCast tools. + + This class serves as the foundation for all tools in the OntoCast system. + It provides common functionality and interface that all tools must implement. + Tools should inherit from this class and implement their specific functionality. + + Attributes: + Inherits all attributes from BasePydanticModel. + """ + + def __init__(self, **kwargs): + """Initialize the tool. + + Args: + **kwargs: Keyword arguments passed to the parent class. + """ + super().__init__(**kwargs) + + +class EntityMetadata(BaseModel): + """Metadata for an entity in the graph.""" + + model_config = {"arbitrary_types_allowed": True} + + local_name: str = Field(description="The local name of the entity") + label: str | None = Field( + default=None, description="Optional human-readable label for the entity" + ) + comment: str | None = Field( + default=None, description="Optional comment describing the entity" + ) + types: set[URIRef] = Field( + default_factory=set, description="Set of RDF types for this entity" + ) + + +class PredicateMetadata(BaseModel): + """Metadata for a predicate in the graph.""" + + model_config = {"arbitrary_types_allowed": True} + + local_name: str = Field(description="The local name of the predicate") + label: str | None = Field( + default=None, description="Optional human-readable label for the predicate" + ) + comment: str | None = Field( + default=None, description="Optional comment describing the predicate" + ) + domain: None | URIRef = Field( + default=None, description="Optional domain of the predicate" + ) + range: None | URIRef = Field( + default=None, description="Optional range of the predicate" + ) + is_explicit_property: bool = Field( + default=False, description="Whether this is an explicit property" + ) diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/ontology_manager.py b/ontology_platform/vendored/ontocast/ontocast/tool/ontology_manager.py new file mode 100644 index 0000000..2ccb226 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/ontology_manager.py @@ -0,0 +1,483 @@ +"""Ontology management tool for OntoCast. + +This module provides functionality for managing multiple ontologies, including +loading, updating, and retrieving ontologies by name or IRI. Tracks version +lineage using hash-based identifiers. +""" + +import logging + +from pydantic import Field + +from ..onto.null import NULL_ONTOLOGY +from ..onto.ontology import Ontology +from ..onto.rdfgraph import RDFGraph +from .onto import Tool + +logger = logging.getLogger(__name__) + + +class OntologyManager(Tool): + """Manager for handling multiple ontologies with version tracking. + + This class provides functionality for managing a collection of ontologies, + tracking version lineage using hash-based identifiers. For each IRI, + it maintains a tree/graph of all versions identified by their hashes. + + Attributes: + ontology_versions: Dictionary mapping IRI to list of all + ontology versions (identified by hash). Each IRI can have + multiple versions forming a lineage tree. + """ + + ontology_versions: dict[str, list[Ontology]] = Field(default_factory=dict) + + def __init__(self, **kwargs): + """Initialize the ontology manager. + + Args: + **kwargs: Additional keyword arguments passed to the parent class. + """ + super().__init__(**kwargs) + # Cache dictionary mapping IRI to hash of freshest terminal ontology. + # Updated incrementally when ontologies are added. + self._cached_ontologies: dict[str, str] = {} + + def __contains__(self, item): + """Check if an item (IRI or ontology_id) is in the ontology manager. + + Args: + item: The IRI or ontology_id to check. + + Returns: + bool: True if the item exists in any version of any ontology. + """ + # Check by IRI (primary key) + if item in self.ontology_versions: + return True + # Check by ontology_id (fallback for backward compatibility) + for versions in self.ontology_versions.values(): + for o in versions: + if o.ontology_id == item: + return True + return False + + def add_ontology(self, ontology: Ontology) -> None: + """Add an ontology to the version tree for its IRI. + + If an ontology with the same hash already exists, it is not added again. + The ontology is added to the version tree for its IRI. + Ensures that created_at is set if not already present. + + Args: + ontology: The ontology to add. + """ + if not ontology.iri or ontology.iri == NULL_ONTOLOGY.iri: + logger.warning( + f"Cannot add ontology without valid IRI (ontology_id: {ontology.ontology_id})" + ) + return + + if not ontology.hash: + logger.warning(f"Cannot add ontology without hash (IRI: {ontology.iri})") + return + + # Ensure created_at is set + if not ontology.created_at: + from datetime import datetime, timezone + + ontology.created_at = datetime.now(timezone.utc) + logger.debug( + f"Set created_at for ontology {ontology.iri} with hash {ontology.hash[:8]}..." + ) + + if ontology.iri not in self.ontology_versions: + self.ontology_versions[ontology.iri] = [] + + # Check if this hash already exists + existing_hashes = {o.hash for o in self.ontology_versions[ontology.iri]} + if ontology.hash not in existing_hashes: + self.ontology_versions[ontology.iri].append(ontology) + # Update cache for this specific IRI (store hash only) + freshest = self.get_freshest_terminal_ontology_by_iri(ontology.iri) + if freshest and freshest.hash: + self._cached_ontologies[ontology.iri] = freshest.hash + logger.debug( + f"Added ontology {ontology.iri} with hash {ontology.hash[:8]}..." + ) + else: + logger.debug( + f"Ontology {ontology.iri} with hash {ontology.hash[:8]}... already exists" + ) + + def get_terminal_ontologies_by_iri(self, iri: str | None = None) -> list[Ontology]: + """Get terminal (leaf) ontologies in the version graph. + + Terminal ontologies are those that are not parents of any other ontology + in the version tree. If iri is provided, returns terminals for + that ontology only; otherwise returns terminals for all ontologies. + + Args: + iri: Optional IRI to filter by. + + Returns: + list[Ontology]: List of terminal ontologies. + """ + if iri: + if iri not in self.ontology_versions: + return [] + ontologies = self.ontology_versions[iri] + else: + ontologies = [ + o for versions in self.ontology_versions.values() for o in versions + ] + + if not ontologies: + return [] + + # Build a set of all parent hashes + all_parent_hashes = set() + for o in ontologies: + all_parent_hashes.update(o.parent_hashes) + + # Terminal nodes are those whose hash is not in any parent_hashes + terminal_hashes = {o.hash for o in ontologies} - all_parent_hashes + + return [o for o in ontologies if o.hash in terminal_hashes] + + def get_terminal_ontologies(self, ontology_id: str | None = None) -> list[Ontology]: + """Get terminal (leaf) ontologies by ontology_id (backward compatibility wrapper). + + Args: + ontology_id: Optional ontology_id to filter by. + + Returns: + list[Ontology]: List of terminal ontologies. + """ + if ontology_id: + # Find IRI(s) matching this ontology_id + matching_iris = [ + iri + for iri, versions in self.ontology_versions.items() + if any(o.ontology_id == ontology_id for o in versions) + ] + if not matching_iris: + return [] + # Get terminals for all matching IRIs + all_terminals = [] + for iri in matching_iris: + all_terminals.extend(self.get_terminal_ontologies_by_iri(iri)) + return all_terminals + else: + return self.get_terminal_ontologies_by_iri(None) + + def get_freshest_terminal_ontology_by_iri( + self, iri: str | None = None + ) -> Ontology | None: + """Get the freshest terminal ontology based on created_at timestamp. + + Returns the terminal ontology with the most recent `created_at` timestamp. + If multiple terminal ontologies exist, returns the one that was most recently + created. If no created_at is set, falls back to the first terminal ontology. + + Args: + iri: Optional IRI to filter by. If None, searches across + all ontologies. + + Returns: + Ontology: The freshest terminal ontology, or None if no terminal + ontologies exist. + """ + terminals = self.get_terminal_ontologies_by_iri(iri) + + if not terminals: + return None + + # Filter out ontologies without created_at and sort by created_at + with_timestamp = [o for o in terminals if o.created_at is not None] + without_timestamp = [o for o in terminals if o.created_at is None] + + if with_timestamp: + # Sort by created_at descending (most recent first) + # Type assertion: we know created_at is not None due to filter above + from datetime import datetime + from typing import cast + + freshest = max( + with_timestamp, + key=lambda o: cast(datetime, o.created_at), + ) + return freshest + elif without_timestamp: + # Fallback to first terminal if no timestamps available + return without_timestamp[0] + + return None + + def get_freshest_terminal_ontology( + self, ontology_id: str | None = None + ) -> Ontology | None: + """Get the freshest terminal ontology by ontology_id (backward compatibility wrapper). + + Args: + ontology_id: Optional ontology_id to filter by. + + Returns: + Ontology: The freshest terminal ontology, or None if no terminal + ontologies exist. + """ + if ontology_id: + # Find IRI(s) matching this ontology_id + matching_iris = [ + iri + for iri, versions in self.ontology_versions.items() + if any(o.ontology_id == ontology_id for o in versions) + ] + if not matching_iris: + return None + # Get freshest for all matching IRIs and return the most recent + candidates = [] + for iri in matching_iris: + freshest = self.get_freshest_terminal_ontology_by_iri(iri) + if freshest: + candidates.append(freshest) + if not candidates: + return None + # Return the most recent among all candidates + from datetime import datetime + from typing import cast + + with_timestamp = [o for o in candidates if o.created_at is not None] + if with_timestamp: + return max(with_timestamp, key=lambda o: cast(datetime, o.created_at)) + return candidates[0] + else: + return self.get_freshest_terminal_ontology_by_iri(None) + + def get_ontology_versions_by_iri(self, iri: str) -> list[Ontology]: + """Get all versions of an ontology by IRI. + + Args: + iri: The IRI to retrieve versions for. + + Returns: + list[Ontology]: List of all versions of the ontology. + """ + return self.ontology_versions.get(iri, []) + + def get_ontology_versions(self, ontology_id: str) -> list[Ontology]: + """Get all versions of an ontology by ontology_id (backward compatibility wrapper). + + Args: + ontology_id: The ontology_id to retrieve versions for. + + Returns: + list[Ontology]: List of all versions of the ontology. + """ + # Find all IRIs matching this ontology_id + all_versions = [] + for iri, versions in self.ontology_versions.items(): + if any(o.ontology_id == ontology_id for o in versions): + all_versions.extend(versions) + return all_versions + + def get_lineage_graph_by_iri(self, iri: str): + """Get the lineage graph for a specific IRI. + + Args: + iri: The IRI to get the lineage graph for. + + Returns: + networkx.DiGraph: The lineage graph for the ontology, or None if not found. + """ + if iri not in self.ontology_versions: + return None + + return Ontology.build_lineage_graph(self.ontology_versions[iri]) + + def get_lineage_graph(self, ontology_id: str): + """Get the lineage graph for a specific ontology_id (backward compatibility wrapper). + + Args: + ontology_id: The ontology_id to get the lineage graph for. + + Returns: + networkx.DiGraph: The lineage graph for the ontology, or None if not found. + """ + # Find first IRI matching this ontology_id + for iri, versions in self.ontology_versions.items(): + if any(o.ontology_id == ontology_id for o in versions): + return Ontology.build_lineage_graph(versions) + return None + + def get_ontology( + self, + ontology_id: str | None = None, + ontology_iri: str | None = None, + hash: str | None = None, + ) -> Ontology: + """Get an ontology by its IRI, ontology_id, or hash. + + If hash is provided, returns the specific version. Otherwise, returns + a terminal (most recent) version if multiple versions exist. + IRI is preferred over ontology_id for lookup. + + Args: + ontology_id: The short name of the ontology to retrieve (optional, for backward compatibility). + ontology_iri: The IRI of the ontology to retrieve (preferred). + hash: The hash of a specific version to retrieve (optional). + + Returns: + Ontology: The matching ontology if found, NULL_ONTOLOGY otherwise. + """ + # If hash is provided, search by hash first + if hash: + for versions in self.ontology_versions.values(): + for o in versions: + if o.hash == hash: + return o + + # Try by IRI first (preferred method) + if ontology_iri is not None: + if ontology_iri in self.ontology_versions: + versions = self.ontology_versions[ontology_iri] + if hash: + # Find specific version by hash + for o in versions: + if o.hash == hash: + return o + else: + # Return terminal version (most recent) + terminals = self.get_terminal_ontologies_by_iri(ontology_iri) + if terminals: + return terminals[0] + # Fallback to first version if no terminals + if versions: + return versions[0] + + # Try by ontology_id if provided (backward compatibility) + if ontology_id is not None: + # Find IRI(s) matching this ontology_id + matching_iris = [ + iri + for iri, versions in self.ontology_versions.items() + if any(o.ontology_id == ontology_id for o in versions) + ] + if matching_iris: + # Use first matching IRI + iri = matching_iris[0] + versions = self.ontology_versions[iri] + if hash: + # Find specific version by hash + for o in versions: + if o.hash == hash: + return o + else: + # Return terminal version (most recent) + terminals = self.get_terminal_ontologies_by_iri(iri) + if terminals: + return terminals[0] + # Fallback to first version if no terminals + if versions: + return versions[0] + + # If IRI is also provided, check consistency + if ontology_iri and ontology_iri != iri: + logger.warning( + f"Ontology id '{ontology_id}' matches IRI '{iri}' but different IRI '{ontology_iri}' was provided" + ) + + # Not found + return NULL_ONTOLOGY + + def get_ontology_iris(self) -> list[str]: + """Get a list of all ontology IRIs. + + Returns: + list[str]: List of ontology IRIs. + """ + return list(self.ontology_versions.keys()) + + def get_ontology_names(self) -> list[str]: + """Get a list of all ontology short names (backward compatibility wrapper). + + Returns: + list[str]: List of unique ontology short names. + """ + names = set() + for versions in self.ontology_versions.values(): + for o in versions: + if o.ontology_id: + names.add(o.ontology_id) + return sorted(list(names)) + + @property + def has_ontologies(self) -> bool: + """Check if there are any ontologies available. + + Returns: + bool: True if there are any ontologies, False otherwise. + """ + return len(self._cached_ontologies) > 0 or len(self.ontology_versions) > 0 + + @property + def ontologies(self) -> list[Ontology]: + """Get freshest terminal ontology for each IRI. + + This property provides backward compatibility with code that expects + a list of ontologies. Returns the freshest (most recently created) + terminal version for each IRI. + + The result is cached per IRI (as hashes) and updated incrementally + when ontologies are added. + + Returns: + list[Ontology]: List of freshest terminal ontologies, one per IRI. + """ + result = [] + + # Ensure cache is up to date for all IRIs + for iri in self.ontology_versions.keys(): + if iri not in self._cached_ontologies: + freshest = self.get_freshest_terminal_ontology_by_iri(iri) + if freshest and freshest.hash: + self._cached_ontologies[iri] = freshest.hash + + # Remove entries for IRIs that no longer exist + cached_iris = set(self._cached_ontologies.keys()) + current_iris = set(self.ontology_versions.keys()) + for removed_iri in cached_iris - current_iris: + del self._cached_ontologies[removed_iri] + + # Look up actual ontology objects by hash + for iri, cached_hash in self._cached_ontologies.items(): + if iri in self.ontology_versions: + # Find ontology with matching hash + for ontology in self.ontology_versions[iri]: + if ontology.hash == cached_hash: + result.append(ontology) + break + + return result + + def update_ontology(self, ontology_id: str, ontology_addendum: RDFGraph): + """Update an existing ontology with additional triples. + + Note: This method is deprecated. Use add_ontology() with a new version + that has the current hash in parent_hashes instead. + + Args: + ontology_id: The short name of the ontology to update. + ontology_addendum: The RDF graph containing additional triples to add. + """ + logger.warning( + "update_ontology() is deprecated. Use add_ontology() with version tracking instead." + ) + terminals = self.get_terminal_ontologies(ontology_id) + if terminals: + terminals[0] += ontology_addendum + # Update cache for the IRI (though this method is deprecated) + iri = terminals[0].iri + freshest = self.get_freshest_terminal_ontology_by_iri(iri) + if freshest and freshest.hash: + self._cached_ontologies[iri] = freshest.hash diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/sparql.py b/ontology_platform/vendored/ontocast/ontocast/tool/sparql.py new file mode 100644 index 0000000..6f610de --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/sparql.py @@ -0,0 +1,315 @@ +"""SPARQL tool for incremental graph updates. + +This module provides functionality for executing SPARQL operations on RDF graphs, +enabling incremental updates instead of full graph replacement. +""" + +import logging + +from rdflib import BNode, Literal, URIRef +from rdflib.plugins.sparql import prepareQuery + +from ontocast.onto.enum import SPARQLOperationType +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.sparql_models import SPARQLOperationModel +from ontocast.tool.triple_manager.core import TripleStoreManager + +logger = logging.getLogger(__name__) + + +class SPARQLTool: + """Tool for executing SPARQL operations on RDF graphs.""" + + def __init__(self, triple_store_manager: TripleStoreManager | None = None): + """Initialize SPARQL tool. + + Args: + triple_store_manager: Optional triple store manager for persistent storage. + """ + self.triple_store_manager = triple_store_manager + self.operation_history = [] + + def execute_operations( + self, graph: RDFGraph, operations: list[SPARQLOperationModel] + ) -> RDFGraph: + """Execute a list of SPARQL operations on a graph. + + Args: + graph: The RDF graph to operate on. + operations: List of SPARQL operations to execute. + + Returns: + RDFGraph: Updated graph after applying operations. + """ + logger.info(f"Executing {len(operations)} SPARQL operations") + + for operation in operations: + try: + self._execute_single_operation(graph, operation) + self.operation_history.append(operation) + logger.debug( + f"Executed {operation.operation_type} operation: {operation.description}" + ) + except Exception as e: + logger.error( + f"Failed to execute {operation.operation_type} operation: {str(e)}" + ) + raise + + return graph + + def execute_operation(self, operation: SPARQLOperationModel) -> None: + """Execute a single SPARQL operation. + + Args: + operation: The SPARQL operation to execute. + """ + # For now, we'll use a simple approach - in a real implementation, + # you might want to track which graph this operation should be applied to + logger.info( + f"Executing {operation.operation_type} operation: {operation.description}" + ) + # This is a placeholder - in practice, you'd need to specify which graph to operate on + # or maintain a default graph in the tool + + def _execute_single_operation( + self, graph: RDFGraph, operation: SPARQLOperationModel + ): + """Execute a single SPARQL operation. + + Args: + graph: The RDF graph to operate on. + operation: The SPARQL operation to execute. + """ + if operation.operation_type == SPARQLOperationType.INSERT: + self._execute_insert(graph, operation) + elif operation.operation_type == SPARQLOperationType.DELETE: + self._execute_delete(graph, operation) + elif operation.operation_type == SPARQLOperationType.UPDATE: + self._execute_update(graph, operation) + else: + raise ValueError(f"Unknown operation type: {operation.operation_type}") + + def _execute_insert(self, graph: RDFGraph, operation: SPARQLOperationModel): + """Execute INSERT operation. + + Args: + graph: The RDF graph to operate on. + operation: The INSERT operation to execute. + """ + # Parse the INSERT query + query = prepareQuery(operation.query) + + # For INSERT DATA, we need to parse the triples and add them to the graph + if "INSERT DATA" in operation.query.upper(): + # Extract triples from INSERT DATA query + triples = self._parse_insert_data_triples(operation.query) + for triple in triples: + graph.add(triple) + else: + # For other INSERT queries, execute against the graph + graph.query(query) + # INSERT queries typically don't return results, but we execute them + + def _execute_delete(self, graph: RDFGraph, operation: SPARQLOperationModel): + """Execute DELETE operation. + + Args: + graph: The RDF graph to operate on. + operation: The DELETE operation to execute. + """ + # Parse the DELETE query + query = prepareQuery(operation.query) + + # For DELETE DATA, we need to parse the triples and remove them from the graph + if "DELETE DATA" in operation.query.upper(): + # Extract triples from DELETE DATA query + triples = self._parse_delete_data_triples(operation.query) + for triple in triples: + graph.remove(triple) + else: + # For other DELETE queries, execute against the graph + graph.query(query) + # DELETE queries typically don't return results, but we execute them + + def _execute_update(self, graph: RDFGraph, operation: SPARQLOperationModel): + """Execute UPDATE operation. + + Args: + graph: The RDF graph to operate on. + operation: The UPDATE operation to execute. + """ + # Parse the UPDATE query + query = prepareQuery(operation.query) + + # Execute the UPDATE query + graph.query(query) + # UPDATE queries typically don't return results, but we execute them + + def _parse_insert_data_triples(self, query: str) -> list[tuple]: + """Parse triples from INSERT DATA query. + + Args: + query: The INSERT DATA query string. + + Returns: + List of triples to insert. + """ + # This is a simplified parser - in practice, you'd want a more robust parser + triples = [] + + # Extract the content between INSERT DATA { ... } + start = query.upper().find("INSERT DATA {") + if start == -1: + return triples + + start += len("INSERT DATA {") + end = query.rfind("}") + + if end == -1: + return triples + + data_content = query[start:end].strip() + + # Split by lines and parse each triple + lines = [line.strip() for line in data_content.split("\n") if line.strip()] + + for line in lines: + if line.endswith("."): + line = line[:-1] # Remove trailing period + + # Parse the triple (simplified - assumes standard N3 format) + parts = line.split() + if len(parts) >= 3: + subject = self._parse_term(parts[0]) + predicate = self._parse_term(parts[1]) + object_part = self._parse_term(" ".join(parts[2:])) + + if subject and predicate and object_part: + triples.append((subject, predicate, object_part)) + + return triples + + def _parse_delete_data_triples(self, query: str) -> list[tuple]: + """Parse triples from DELETE DATA query. + + Args: + query: The DELETE DATA query string. + + Returns: + List of triples to delete. + """ + # Similar to INSERT DATA parsing + return self._parse_insert_data_triples( + query.replace("DELETE DATA", "INSERT DATA") + ) + + def _parse_term(self, term: str): + """Parse a SPARQL term (subject, predicate, or object). + + Args: + term: The term string to parse. + + Returns: + Parsed RDF term (URIRef, Literal, or BNode). + """ + term = term.strip() + + if term.startswith("<") and term.endswith(">"): + # URI + return URIRef(term[1:-1]) + elif term.startswith('"') and term.endswith('"'): + # Literal + return Literal(term[1:-1]) + elif term.startswith("_:"): + # Blank node + return BNode(term[2:]) + elif term.startswith('"') and '"^^' in term: + # Typed literal + value, datatype = term.split('"^^') + return Literal(value[1:], datatype=URIRef(datatype)) + else: + # Assume it's a URI without angle brackets + return URIRef(term) + + def validate_operation(self, operation: SPARQLOperationModel) -> bool: + """Validate a SPARQL operation. + + Args: + operation: The operation to validate. + + Returns: + bool: True if valid, False otherwise. + """ + try: + prepareQuery(operation.query) + return True + except Exception as e: + logger.error(f"Invalid SPARQL operation: {str(e)}") + return False + + def get_operation_history(self) -> list[SPARQLOperationModel]: + """Get the history of executed operations. + + Returns: + List of executed operations. + """ + return self.operation_history.copy() + + def clear_history(self): + """Clear the operation history.""" + self.operation_history.clear() + + def create_insert_operation( + self, query: str, description: str = "" + ) -> SPARQLOperationModel: + """Create an INSERT operation. + + Args: + query: The SPARQL INSERT query. + description: Optional description of the operation. + + Returns: + SPARQLOperationModel: The created operation. + """ + return SPARQLOperationModel( + operation_type=SPARQLOperationType.INSERT, + query=query, + description=description, + ) + + def create_delete_operation( + self, query: str, description: str = "" + ) -> SPARQLOperationModel: + """Create a DELETE operation. + + Args: + query: The SPARQL DELETE query. + description: Optional description of the operation. + + Returns: + SPARQLOperationModel: The created operation. + """ + return SPARQLOperationModel( + operation_type=SPARQLOperationType.DELETE, + query=query, + description=description, + ) + + def create_update_operation( + self, query: str, description: str = "" + ) -> SPARQLOperationModel: + """Create an UPDATE operation. + + Args: + query: The SPARQL UPDATE query. + description: Optional description of the operation. + + Returns: + SPARQLOperationModel: The created operation. + """ + return SPARQLOperationModel( + operation_type=SPARQLOperationType.UPDATE, + query=query, + description=description, + ) diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/structured_sparql.py b/ontology_platform/vendored/ontocast/ontocast/tool/structured_sparql.py new file mode 100644 index 0000000..e32cd7f --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/structured_sparql.py @@ -0,0 +1,370 @@ +"""Structured SPARQL parser and executor. + +This module provides tools for parsing and executing structured SPARQL queries +with separate ADD, UPDATE, and REMOVE sections. +""" + +import logging + +from ontocast.onto.enum import SPARQLOperationType +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.sparql_models import ( + SPARQLOperationModel, + StructuredSPARQLQueryModel, +) + +logger = logging.getLogger(__name__) + + +class StructuredSPARQLParser: + """Parser for structured SPARQL queries with ADD, UPDATE, REMOVE sections.""" + + def __init__(self): + self.logger = logging.getLogger(__name__) + + def parse_structured_query(self, query_text: str) -> StructuredSPARQLQueryModel: + """Parse a structured SPARQL query from text. + + Args: + query_text: The structured SPARQL query text + + Returns: + StructuredSPARQLQueryModel: Parsed structured query + """ + self.logger.info("Parsing structured SPARQL query") + + # Split into sections + sections = self._split_into_sections(query_text) + + # Parse each section and combine all operations + all_operations = [] + all_operations.extend( + self._parse_section_operations( + sections.get("ADD", ""), SPARQLOperationType.INSERT + ) + ) + all_operations.extend( + self._parse_section_operations( + sections.get("UPDATE", ""), SPARQLOperationType.UPDATE + ) + ) + all_operations.extend( + self._parse_section_operations( + sections.get("REMOVE", ""), SPARQLOperationType.DELETE + ) + ) + + # Extract namespaces + namespaces = self._extract_namespaces(query_text) + + structured_query = StructuredSPARQLQueryModel( + operations=all_operations, + namespaces=namespaces, + ) + + self.logger.info(f"Parsed structured query: {structured_query.get_summary()}") + return structured_query + + def _split_into_sections(self, query_text: str) -> dict[str, str]: + """Split query text into ADD, UPDATE, REMOVE sections. + + Args: + query_text: The query text to split + + Returns: + dict[str, str]: Dictionary mapping section names to content + """ + sections = {} + current_section = None + current_content = [] + + lines = query_text.split("\n") + + for line in lines: + line = line.strip() + + # Check for section headers + if line.upper().startswith("-- ADD SECTION:") or line.upper().startswith( + "ADD SECTION:" + ): + if current_section: + sections[current_section] = "\n".join(current_content) + current_section = "ADD" + current_content = [] + elif line.upper().startswith( + "-- UPDATE SECTION:" + ) or line.upper().startswith("UPDATE SECTION:"): + if current_section: + sections[current_section] = "\n".join(current_content) + current_section = "UPDATE" + current_content = [] + elif line.upper().startswith( + "-- REMOVE SECTION:" + ) or line.upper().startswith("REMOVE SECTION:"): + if current_section: + sections[current_section] = "\n".join(current_content) + current_section = "REMOVE" + current_content = [] + else: + current_content.append(line) + + # Add the last section + if current_section: + sections[current_section] = "\n".join(current_content) + + return sections + + def _parse_section_operations( + self, section_content: str, default_type: SPARQLOperationType + ) -> list[SPARQLOperationModel]: + """Parse operations from a section. + + Args: + section_content: The section content + default_type: Default operation type for the section + + Returns: + list[SPARQLOperationModel]: List of parsed operations + """ + operations = [] + + if not section_content.strip(): + return operations + + # Split by PREFIX blocks and INSERT/UPDATE/DELETE blocks + blocks = self._split_into_blocks(section_content) + + for block in blocks: + if block.strip(): + operation = self._create_operation_from_block(block, default_type) + if operation: + operations.append(operation) + + return operations + + def _split_into_blocks(self, content: str) -> list[str]: + """Split content into SPARQL operation blocks. + + Args: + content: The content to split + + Returns: + list[str]: List of operation blocks + """ + blocks = [] + current_block = [] + in_operation = False + + lines = content.split("\n") + + for line in lines: + line = line.strip() + + if not line: + continue + + # Check if this is the start of a new operation + if ( + line.upper().startswith("INSERT") + or line.upper().startswith("UPDATE") + or line.upper().startswith("DELETE") + ): + # Save previous block if exists + if current_block: + blocks.append("\n".join(current_block)) + + # Start new block + current_block = [line] + in_operation = True + elif in_operation: + current_block.append(line) + + # Check if operation is complete + if line == "}" and current_block: + blocks.append("\n".join(current_block)) + current_block = [] + in_operation = False + + # Add any remaining block + if current_block: + blocks.append("\n".join(current_block)) + + return blocks + + def _create_operation_from_block( + self, block: str, default_type: SPARQLOperationType + ) -> SPARQLOperationModel | None: + """Create a SPARQL operation from a block. + + Args: + block: The operation block + default_type: Default operation type + + Returns: + SPARQLOperationModel | None: Created operation or None if invalid + """ + if not block.strip(): + return None + + # Determine operation type + if block.upper().startswith("INSERT"): + operation_type = SPARQLOperationType.INSERT + elif block.upper().startswith("UPDATE"): + operation_type = SPARQLOperationType.UPDATE + elif block.upper().startswith("DELETE"): + operation_type = SPARQLOperationType.DELETE + else: + operation_type = default_type + + return SPARQLOperationModel(operation_type=operation_type, query=block.strip()) + + def _extract_namespaces(self, query_text: str) -> dict[str, str]: + """Extract namespace declarations from query text. + + Args: + query_text: The query text + + Returns: + dict[str, str]: Dictionary mapping prefixes to URIs + """ + namespaces = {} + + lines = query_text.split("\n") + for line in lines: + line = line.strip() + + if line.upper().startswith("PREFIX "): + # Parse PREFIX declaration + parts = line.split() + if len(parts) >= 3: + prefix = parts[1].rstrip(":") + uri = parts[2].strip("<>") + namespaces[prefix] = uri + + return namespaces + + +class StructuredSPARQLExecutor: + """Executor for structured SPARQL queries.""" + + def __init__(self, sparql_tool): + """Initialize the executor. + + Args: + sparql_tool: The SPARQL tool for executing operations + """ + self.sparql_tool = sparql_tool + self.logger = logging.getLogger(__name__) + + def execute_structured_query( + self, structured_query: StructuredSPARQLQueryModel, graph: RDFGraph + ) -> bool: + """Execute a structured SPARQL query. + + Args: + structured_query: The structured query to execute + graph: The RDF graph to execute against + + Returns: + bool: True if execution was successful + """ + self.logger.info( + f"Executing structured query: {structured_query.get_summary()}" + ) + + try: + # Execute operations in order: ADD, UPDATE, REMOVE + all_operations = structured_query.get_all_operations() + + for operation in all_operations: + # Execute SPARQLOperationModel directly + self.sparql_tool.execute_operation(operation) + self.logger.debug( + f"Executed {operation.operation_type.value} operation" + ) + + self.logger.info("Structured query execution completed successfully") + return True + + except Exception as e: + self.logger.error(f"Failed to execute structured query: {e}") + return False + + def execute_add_operations( + self, structured_query: StructuredSPARQLQueryModel, graph: RDFGraph + ) -> bool: + """Execute only ADD operations from a structured query. + + Args: + structured_query: The structured query + graph: The RDF graph to execute against + + Returns: + bool: True if execution was successful + """ + return self._execute_operations(structured_query.get_add_operations(), "ADD") + + def execute_update_operations( + self, structured_query: StructuredSPARQLQueryModel, graph: RDFGraph + ) -> bool: + """Execute only UPDATE operations from a structured query. + + Args: + structured_query: The structured query + graph: The RDF graph to execute against + + Returns: + bool: True if execution was successful + """ + return self._execute_operations( + structured_query.get_update_operations(), "UPDATE" + ) + + def execute_remove_operations( + self, structured_query: StructuredSPARQLQueryModel, graph: RDFGraph + ) -> bool: + """Execute only REMOVE operations from a structured query. + + Args: + structured_query: The structured query + graph: The RDF graph to execute against + + Returns: + bool: True if execution was successful + """ + return self._execute_operations( + structured_query.get_remove_operations(), "REMOVE" + ) + + def _execute_operations( + self, operations: list[SPARQLOperationModel], section_name: str + ) -> bool: + """Execute a list of operations. + + Args: + operations: List of operations to execute + section_name: Name of the section for logging + + Returns: + bool: True if execution was successful + """ + if not operations: + self.logger.info(f"No {section_name} operations to execute") + return True + + try: + for operation in operations: + # Execute SPARQLOperationModel directly + self.sparql_tool.execute_operation(operation) + self.logger.debug( + f"Executed {section_name} {operation.operation_type.value} operation" + ) + + self.logger.info( + f"Executed {len(operations)} {section_name} operations successfully" + ) + return True + + except Exception as e: + self.logger.error(f"Failed to execute {section_name} operations: {e}") + return False diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/__init__.py b/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/__init__.py new file mode 100644 index 0000000..72c6268 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/__init__.py @@ -0,0 +1,51 @@ +"""Triple store management package for OntoCast. + +This package provides a unified interface for managing RDF triple stores +across different backends. It includes abstract base classes and concrete +implementations for various triple store technologies. + +The package supports: +- Abstract interfaces for triple store operations +- Neo4j implementation using the n10s plugin +- Fuseki implementation using Apache Fuseki +- Filesystem implementation for local storage + +All implementations support: +- Fetching and storing ontologies +- Serializing and retrieving facts +- Authentication and connection management +- Error handling and logging + +Example: + >>> from ontocast.tool.triple_manager import Neo4jTripleStoreManager + >>> manager = Neo4jTripleStoreManager(uri="bolt://localhost:7687") + >>> ontologies = manager.fetch_ontologies() +""" + +from .core import ( + TripleStoreManager, +) +from .filesystem_manager import ( + FilesystemTripleStoreManager, +) +from .fuseki import ( + FusekiTripleStoreManager, +) +from .mock import ( + MockFusekiTripleStoreManager, + MockNeo4jTripleStoreManager, + MockTripleStoreManager, +) +from .neo4j import ( + Neo4jTripleStoreManager, +) + +__all__ = [ + "TripleStoreManager", + "Neo4jTripleStoreManager", + "FusekiTripleStoreManager", + "FilesystemTripleStoreManager", + "MockTripleStoreManager", + "MockFusekiTripleStoreManager", + "MockNeo4jTripleStoreManager", +] diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/core.py b/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/core.py new file mode 100644 index 0000000..47d3474 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/core.py @@ -0,0 +1,159 @@ +"""Triple store management tools for OntoCast. + +This module provides functionality for managing RDF triple stores, including +abstract interfaces and concrete implementations for different triple store backends. +""" + +import abc +import os + +from pydantic import Field +from rdflib import Graph + +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.tool import Tool + + +class TripleStoreManager(Tool): + """Base class for managing RDF triple stores. + + This class defines the interface for triple store management operations, + including fetching and storing ontologies and their graphs. All concrete + triple store implementations should inherit from this class. + + This is an abstract base class that must be implemented by specific + triple store backends (e.g., Neo4j, Fuseki, Filesystem). + """ + + def __init__(self, **kwargs): + """Initialize the triple store manager. + + Args: + **kwargs: Additional keyword arguments passed to the parent class. + """ + super().__init__(**kwargs) + + @abc.abstractmethod + def fetch_ontologies(self) -> list[Ontology]: + """Fetch all available ontologies from the triple store. + + This method should retrieve all ontologies stored in the triple store + and return them as Ontology objects with their associated RDF graphs. + + Returns: + list[Ontology]: List of available ontologies with their graphs. + """ + return [] + + @abc.abstractmethod + def serialize_graph(self, graph: Graph, **kwargs) -> bool | None: + """Store an RDF graph in the triple store. + + This method should store the given RDF graph in the triple store. + The implementation may choose how to organize the storage (e.g., as named graphs, + in specific collections, etc.). + + Args: + graph: The RDF graph to store. + **kwargs: Implementation-specific arguments (e.g., fname for filesystem, graph_uri for Fuseki). + + Returns: + bool | None: Implementation-specific return value (bool for Fuseki, summary for Neo4j, None for Filesystem). + """ + pass + + @abc.abstractmethod + def serialize(self, o: Ontology | RDFGraph, **kwargs) -> bool | None: # type: ignore[override] + """Store an RDF graph in the triple store. + + This method should store the given RDF graph in the triple store. + The implementation may choose how to organize the storage (e.g., as named graphs, + in specific collections, etc.). + + Args: + o: RDF graph or Ontology object to store. + **kwargs: Implementation-specific arguments (e.g., graph_uri for Fuseki). + + Returns: + bool | None: Implementation-specific return value (bool for Fuseki, summary for Neo4j, None for Filesystem). + """ + pass + + @abc.abstractmethod + async def clean(self, dataset: str | None = None) -> None: + """Clean/flush data from the triple store. + + This method removes data from the triple store. For Fuseki, the optional + dataset parameter allows cleaning a specific dataset, or all datasets if None. + For Neo4j and Filesystem, the dataset parameter is ignored. + + Args: + dataset: Optional dataset name to clean (Fuseki only). If None, cleans + all data. For other stores, this parameter is ignored. + + Warning: This operation is irreversible and will delete all data. + + Raises: + NotImplementedError: If the triple store doesn't support cleaning. + """ + raise NotImplementedError("clean() method must be implemented by subclasses") + + +class TripleStoreManagerWithAuth(TripleStoreManager): + """Base class for triple store managers that require authentication. + + This class provides common functionality for triple store managers that + need URI and authentication credentials. It handles environment variable + loading and credential parsing. + + Attributes: + uri: The connection URI for the triple store. + auth: Authentication tuple (username, password) for the triple store. + """ + + uri: str | None = Field(default=None, description="Triple store connection URI") + auth: tuple | None = Field( + default=None, description="Triple store authentication tuple (user, password)" + ) + + def __init__(self, uri=None, auth=None, env_uri=None, env_auth=None, **kwargs): + """Initialize the triple store manager with authentication. + + This method handles loading URI and authentication credentials from + either direct parameters or environment variables. It also parses + authentication strings in the format "user/password". + + Args: + uri: Direct URI for the triple store connection. + auth: Direct authentication tuple or string in "user/password" format. + env_uri: Environment variable name for the URI (e.g., "NEO4J_URI"). + env_auth: Environment variable name for authentication (e.g., "NEO4J_AUTH"). + **kwargs: Additional keyword arguments passed to the parent class. + + Raises: + ValueError: If authentication string is not in "user/password" format. + + Example: + >>> manager = TripleStoreManagerWithAuth( + ... env_uri="NEO4J_URI", + ... env_auth="NEO4J_AUTH" + ... ) + """ + # Use env vars if not provided + uri = uri or (os.getenv(env_uri) if env_uri else None) + auth_env = auth or (os.getenv(env_auth) if env_auth else None) + + if auth_env and not isinstance(auth_env, tuple): + if "/" in auth_env: + user, password = auth_env.split("/", 1) + auth = (user, password) + else: + raise ValueError( + f"{env_auth or 'TRIPLESTORE_AUTH'} must be in 'user/password' format" + ) + elif isinstance(auth_env, tuple): + auth = auth_env + # else: auth remains None + + super().__init__(uri=uri, auth=auth, **kwargs) diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/filesystem_manager.py b/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/filesystem_manager.py new file mode 100644 index 0000000..1db6627 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/filesystem_manager.py @@ -0,0 +1,151 @@ +"""Filesystem triple store management for OntoCast. + +This module provides a concrete implementation of triple store management +using the local filesystem for storage. It supports reading and writing +ontologies and facts as Turtle files. +""" + +import logging +import pathlib + +from rdflib import Graph + +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.tool.triple_manager.core import TripleStoreManager + +logger = logging.getLogger(__name__) + + +class FilesystemTripleStoreManager(TripleStoreManager): + """Filesystem-based implementation of triple store management. + + This class provides a concrete implementation of triple store management + using the local filesystem for storage. It reads and writes ontologies + and facts as Turtle (.ttl) files in specified directories. + + The manager supports: + - Loading ontologies from a dedicated ontology directory + - Storing ontologies with versioned filenames + - Storing facts with customizable filenames based on specifications + - Error handling for file operations + + Attributes: + working_directory: Path to the working directory for storing data. + ontology_path: Optional path to the ontology directory for loading ontologies. + """ + + working_directory: pathlib.Path | None + ontology_path: pathlib.Path | None + + def __init__(self, **kwargs): + """Initialize the filesystem triple store manager. + + This method sets up the filesystem manager with the specified + working and ontology directories. + + Args: + **kwargs: Additional keyword arguments passed to the parent class. + working_directory: Path to the working directory for storing data. + ontology_path: Path to the ontology directory for loading ontologies. + + Example: + >>> manager = FilesystemTripleStoreManager( + ... working_directory="/path/to/work", + ... ontology_path="/path/to/ontologies" + ... ) + """ + super().__init__(**kwargs) + + def fetch_ontologies(self) -> list[Ontology]: + """Fetch all available ontologies from the filesystem. + + This method scans the ontology directory for Turtle (.ttl) files + and loads each one as an Ontology object. Files are processed + in sorted order for consistent results. + + Returns: + list[Ontology]: List of all ontologies found in the ontology directory. + + Example: + >>> ontologies = manager.fetch_ontologies() + >>> for onto in ontologies: + ... print(f"Loaded ontology: {onto.ontology_id}") + """ + ontologies = [] + if self.ontology_path is not None: + sorted_files = sorted(self.ontology_path.glob("*.ttl")) + for fname in sorted_files: + try: + ontology = Ontology.from_file(fname) + ontologies.append(ontology) + logger.debug(f"Successfully loaded ontology from {fname}") + except Exception as e: + logger.error(f"Failed to load ontology {fname}: {str(e)}") + return ontologies + + def serialize_graph(self, graph: Graph, **kwargs) -> bool | None: + """Store an RDF graph in the filesystem. + + This method stores the given RDF graph as a Turtle file in the + working directory. The filename is generated based on the graph_uri + parameter or defaults to "current.ttl". + + Args: + graph: The RDF graph to store. + fname: str + + Example: + >>> graph = RDFGraph() + >>> manager.serialize_graph(graph) + # Creates: working_directory/current.ttl + + >>> manager.serialize_graph(graph, fname="facts_abc.ttl") + """ + if self.working_directory is None: + return + + fname: str = kwargs.pop("fname") + output_path = self.working_directory / fname + graph.serialize(format="turtle", destination=output_path) + logger.info(f"Graph saved to {output_path}") + + def serialize(self, o: Ontology | RDFGraph, graph_uri: str | None = None): # type: ignore[override] + if isinstance(o, Ontology): + graph = o.graph + fname = f"ontology_{o.ontology_id}_{o.version}.ttl" + elif isinstance(o, RDFGraph): + graph = o + if graph_uri: + s = graph_uri.split("/")[-2:] + s = "_".join([x for x in s if x]) + fname = f"facts_{s}.ttl" + else: + fname = "facts_default.ttl" + else: + raise TypeError(f"unsupported obj of type {type(o)} received") + + self.serialize_graph(graph=graph, fname=fname) + + async def clean(self, dataset: str | None = None) -> None: + """Clean/flush all data from the filesystem triple store. + + This method deletes all Turtle (.ttl) files from both the working + directory and the ontology directory. + + Args: + dataset: Optional dataset parameter (ignored for Filesystem, which doesn't + support datasets). Included for interface compatibility. + + Warning: This operation is irreversible and will delete all data. + + Raises: + Exception: If the cleanup operation fails. + """ + if dataset is not None: + logger.warning( + f"Dataset parameter '{dataset}' ignored for Filesystem (datasets not supported)" + ) + logger.warning( + "clean method not implemented for FilesystemTripleStoreManager" + ) diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/fuseki.py b/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/fuseki.py new file mode 100644 index 0000000..1edc2d4 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/fuseki.py @@ -0,0 +1,811 @@ +"""Fuseki triple store management for OntoCast. + +This module provides a concrete implementation of triple store management +using Apache Fuseki as the backend. It supports named graphs for ontologies +and facts, with proper authentication and dataset management. +""" + +import asyncio +import logging +import re +from collections import defaultdict +from urllib.parse import quote + +import httpx +from pydantic import Field +from rdflib import Graph +from rdflib.namespace import OWL, RDF + +from ontocast.onto.constants import DEFAULT_DATASET, DEFAULT_ONTOLOGIES_DATASET +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.tool.triple_manager.core import TripleStoreManagerWithAuth + +logger = logging.getLogger(__name__) + + +def deterministic_turtle_serialization(graph: Graph) -> str: + """Create a deterministic Turtle serialization of an RDF graph. + + This function ensures that the same graph content will always produce + the same Turtle output, regardless of the order triples were added or + how they're stored in Fuseki. This is crucial for caching to work + correctly. + + Args: + graph: The RDF graph to serialize. + + Returns: + str: Deterministically serialized Turtle string. + """ + # Capture and sort namespaces + prefix_lines = [ + f"@prefix {p}: <{ns}> ." + for p, ns in sorted(graph.namespace_manager.namespaces()) + ] + + # Sort triples by their string representation + triples_sorted = sorted(graph, key=lambda t: (str(t[0]), str(t[1]), str(t[2]))) + + # Serialize triples using n3 format to get proper Turtle syntax + triple_lines = [ + f"{s.n3(graph.namespace_manager)} {p.n3(graph.namespace_manager)} {o.n3(graph.namespace_manager)} ." + for s, p, o in triples_sorted + ] + + # Return sorted prefixes followed by sorted triples + return "\n".join(prefix_lines + [""] + triple_lines) + + +def _compare_versions(ver1: str, ver2: str) -> int: + """Compare two semantic version strings. + + Args: + ver1: First version string (e.g., "1.2.3") + ver2: Second version string (e.g., "1.3.0") + + Returns: + int: Negative if ver1 < ver2, 0 if equal, positive if ver1 > ver2 + """ + + def _parse_version(v: str) -> tuple: + # Simple version parser - splits by dots and converts to int + parts = v.split(".") + result = [] + for part in parts: + # Remove any non-numeric suffix + numeric_part = re.sub(r"[^0-9].*$", "", part) + result.append(int(numeric_part) if numeric_part else 0) + # Pad to 3 components + while len(result) < 3: + result.append(0) + return tuple(result) + + try: + v1_parts = _parse_version(ver1) + v2_parts = _parse_version(ver2) + if v1_parts < v2_parts: + return -1 + elif v1_parts > v2_parts: + return 1 + return 0 + except Exception: + # If parsing fails, use string comparison + return 1 if ver1 > ver2 else (-1 if ver1 < ver2 else 0) + + +class FusekiTripleStoreManager(TripleStoreManagerWithAuth): + """Fuseki-based triple store manager. + + This class provides a concrete implementation of triple store management + using Apache Fuseki. It stores ontologies as named graphs using their + URIs as graph names, and supports dataset creation and cleanup. + + The manager uses Fuseki's REST API for all operations, including: + - Dataset creation and management + - Named graph operations for ontologies + - SPARQL queries for ontology discovery + - Graph-level data operations + + Attributes: + dataset: The Fuseki dataset name to use for storage. + clean: Whether to clean the dataset on initialization. + """ + + dataset: str | None = Field(default=None, description="Fuseki dataset name") + ontologies_dataset: str = Field( + default=DEFAULT_ONTOLOGIES_DATASET, + description="Fuseki dataset name for ontologies", + ) + + def __init__( + self, + uri=None, + auth=None, + dataset=None, + ontologies_dataset=None, + **kwargs, + ): + """Initialize the Fuseki triple store manager. + + This method sets up the connection to Fuseki and creates the dataset + if it doesn't exist. The dataset is NOT cleaned on initialization. + + Args: + uri: Fuseki server URI (e.g., "http://localhost:3030"). + auth: Authentication tuple (username, password) or string in "user/password" format. + dataset: Dataset name to use for storage. + ontologies_dataset: Dataset name for ontologies (defaults to separate dataset). + **kwargs: Additional keyword arguments passed to the parent class. + + Raises: + ValueError: If dataset is not specified in URI or as argument. + + Example: + >>> manager = FusekiTripleStoreManager( + ... uri="http://localhost:3030", + ... dataset="test" + ... ) + >>> # To clean the dataset, use the clean() method explicitly: + >>> await manager.clean() + """ + super().__init__( + uri=uri, auth=auth, env_uri="FUSEKI_URI", env_auth="FUSEKI_AUTH", **kwargs + ) + if dataset is None: + self.dataset = DEFAULT_DATASET + else: + self.dataset = dataset + self.ontologies_dataset = ontologies_dataset or DEFAULT_ONTOLOGIES_DATASET + + # Initialize httpx client for async operations + self._client: httpx.AsyncClient | None = None + + # Initialize datasets synchronously (for backward compatibility) + # In async contexts, use async_init() instead + asyncio.run(self._async_init_with_cleanup()) + + async def _async_init_with_cleanup(self): + """Wrapper for async_init that ensures proper cleanup when using asyncio.run(). + + This method creates a temporary client and ensures it's properly closed + before returning, preventing "Event loop is closed" errors. + """ + async with httpx.AsyncClient( + auth=self._prepare_auth(), timeout=30.0 + ) as temp_client: + # Temporarily replace the client + original_client = self._client + self._client = temp_client + try: + await self._async_init() + finally: + # Restore original client + self._client = original_client + + async def _async_init(self): + """Async initialization of datasets.""" + await self.init_dataset(self.dataset) + if self.ontologies_dataset != self.dataset: + await self.init_dataset(self.ontologies_dataset) + + def _prepare_auth(self) -> httpx.BasicAuth | None: + """Prepare httpx BasicAuth from self.auth. + + Returns: + httpx.BasicAuth instance or None if no auth is configured. + """ + if self.auth: + if isinstance(self.auth, tuple): + return httpx.BasicAuth(*self.auth) + elif isinstance(self.auth, str) and "/" in self.auth: + parts = self.auth.split("/", 1) + if len(parts) == 2: + username, password = parts[0], parts[1] + return httpx.BasicAuth(username, password) + return None + + async def _get_client(self) -> httpx.AsyncClient: + """Get or create the httpx async client.""" + if self._client is None: + auth = self._prepare_auth() + self._client = httpx.AsyncClient(auth=auth, timeout=30.0) + return self._client + + async def close(self): + """Close the httpx client.""" + if self._client is not None: + await self._client.aclose() + self._client = None + + async def update_dataset(self, new_dataset: str) -> None: + """Update the dataset name for this manager. + + This method allows changing the dataset without recreating the entire + manager, which is useful for API requests that specify different datasets. + + Args: + new_dataset: The new dataset name to use. + """ + if not new_dataset: + raise ValueError("Dataset name cannot be empty") + + self.dataset = new_dataset + await self.init_dataset(self.dataset) + logger.info(f"Updated Fuseki dataset to: {self.dataset}") + + async def clean(self, dataset: str | None = None) -> None: + """Clean/flush data from Fuseki dataset(s). + + This method removes all named graphs and clears the default graph + from the specified dataset, or all datasets if no dataset is specified. + + Args: + dataset: Optional dataset name to clean. If None, cleans both the main + dataset and the ontologies dataset. If specified, cleans only that dataset. + + Warning: This operation is irreversible and will delete all data + from the specified dataset(s). + + The method handles errors gracefully and logs the results of + each cleanup operation. + + Example: + >>> # Clean all datasets + >>> await manager.clean() + >>> # Clean specific dataset + >>> await manager.clean(dataset="my_dataset") + """ + if dataset is None: + # Clean all datasets (main and ontologies) + # self.dataset is guaranteed to be a string (set to DEFAULT_DATASET if None in __init__) + assert self.dataset is not None, "Dataset should never be None" + await self._clean_dataset_by_name(self.dataset) + logger.info(f"Fuseki dataset '{self.dataset}' cleaned (all data deleted)") + + # Also clean the ontologies dataset if it's different + if self.ontologies_dataset != self.dataset: + await self._clean_dataset_by_name(self.ontologies_dataset) + logger.info( + f"Fuseki ontologies dataset '{self.ontologies_dataset}' cleaned (all data deleted)" + ) + else: + # Clean only the specified dataset + await self._clean_dataset_by_name(dataset) + logger.info(f"Fuseki dataset '{dataset}' cleaned (all data deleted)") + + async def _clean_dataset_by_name(self, dataset_name: str) -> None: + """Clean a specific dataset by name. + + This is a helper method that performs the actual cleaning of a single dataset. + It deletes all named graphs and clears the default graph. + + Uses a temporary client to avoid event loop cleanup issues when called + from different async contexts. + + Args: + dataset_name: Name of the dataset to clean. + + Raises: + Exception: If the cleanup operation fails. + """ + # Use a temporary client to avoid event loop cleanup issues + async with httpx.AsyncClient(auth=self._prepare_auth(), timeout=30.0) as client: + try: + dataset_url = f"{self.uri}/{dataset_name}" + sparql_update_url = f"{dataset_url}/update" + sparql_url = f"{dataset_url}/sparql" + + # Delete all named graphs + query = """ + SELECT DISTINCT ?g WHERE { + GRAPH ?g { ?s ?p ?o } + } + """ + response = await client.post( + sparql_url, + data={"query": query, "format": "application/sparql-results+json"}, + ) + + if response.status_code == 200: + results = response.json() + tasks = [] + for binding in results.get("results", {}).get("bindings", []): + graph_uri = binding["g"]["value"] + # Delete the named graph using SPARQL UPDATE + drop_query = f"DROP GRAPH <{graph_uri}>" + tasks.append( + client.post( + sparql_update_url, + data={"update": drop_query}, + ) + ) + + # Execute all deletions in parallel + delete_responses = await asyncio.gather( + *tasks, return_exceptions=True + ) + for i, delete_response in enumerate(delete_responses): + graph_uri = results["results"]["bindings"][i]["g"]["value"] + if isinstance(delete_response, Exception): + logger.warning( + f"Failed to delete graph {graph_uri}: {delete_response}" + ) + elif isinstance(delete_response, httpx.Response): + if delete_response.status_code in (200, 204): + logger.debug(f"Deleted named graph: {graph_uri}") + else: + logger.warning( + f"Failed to delete graph {graph_uri}: {delete_response.status_code}" + ) + + # Clear the default graph using SPARQL UPDATE + clear_query = "CLEAR DEFAULT" + clear_response = await client.post( + sparql_update_url, + data={"update": clear_query}, + ) + if clear_response.status_code in (200, 204): + logger.debug(f"Cleared default graph in dataset '{dataset_name}'") + else: + logger.warning( + f"Failed to clear default graph in dataset '{dataset_name}': {clear_response.status_code}" + ) + except Exception as e: + logger.error(f"Failed to clean dataset '{dataset_name}': {e}") + raise + + async def init_dataset(self, dataset_name): + """Initialize a Fuseki dataset. + + This method creates a new dataset in Fuseki if it doesn't already exist. + It uses Fuseki's admin API to create the dataset with TDB2 storage. + + Uses a temporary client to avoid event loop cleanup issues when called + from different async contexts. + + Args: + dataset_name: Name of the dataset to create. + + Note: + This method will not fail if the dataset already exists. + """ + # Use a temporary client to avoid event loop cleanup issues + async with httpx.AsyncClient(auth=self._prepare_auth(), timeout=30.0) as client: + fuseki_admin_url = f"{self.uri}/$/datasets" + + payload = {"dbName": dataset_name, "dbType": "tdb2"} + + headers = {"Content-Type": "application/x-www-form-urlencoded"} + + response = await client.post( + fuseki_admin_url, data=payload, headers=headers + ) + + if response.status_code == 200 or response.status_code == 201: + logger.info(f"Fuseki dataset '{dataset_name}' created successfully.") + elif response.status_code == 409: + logger.info( + f"Fuseki status code: {response.status_code}; {response.text.strip()}" + ) + else: + logger.error( + f"Failed to create dataset {dataset_name}. Status code: {response.status_code}" + ) + logger.error(f"Response: {response.text.strip()}") + + def _get_dataset_url(self): + """Get the full URL for the dataset. + + Returns: + str: The complete URL for the dataset endpoint. + """ + return f"{self.uri}/{self.dataset}" + + def _get_ontologies_dataset_url(self): + """Get the full URL for the ontologies dataset. + + Returns: + str: The complete URL for the ontologies dataset endpoint. + """ + return f"{self.uri}/{self.ontologies_dataset}" + + def fetch_ontologies(self) -> list[Ontology]: + """Synchronous wrapper for fetch_ontologies. + + For async usage, use afetch_ontologies() instead. + """ + # Use a temporary client for this operation to avoid event loop cleanup issues + return asyncio.run(self._fetch_ontologies_with_cleanup()) + + async def afetch_ontologies(self) -> list[Ontology]: + """Async version of fetch_ontologies. + + This is the preferred method when running in an async context. + """ + return await self._fetch_ontologies_async() + + async def _fetch_ontologies_with_cleanup(self) -> list[Ontology]: + """Wrapper that ensures proper cleanup when using asyncio.run(). + + This method creates a temporary client and ensures it's properly closed + before returning, preventing "Event loop is closed" errors. + """ + async with httpx.AsyncClient( + auth=self._prepare_auth(), timeout=30.0 + ) as temp_client: + # Temporarily replace the client + original_client = self._client + self._client = temp_client + try: + return await self._fetch_ontologies_async() + finally: + # Restore original client + self._client = original_client + + async def _fetch_ontologies_async(self) -> list[Ontology]: + """Fetch all ontologies from their corresponding named graphs. + + This method discovers all ontologies in the Fuseki ontologies dataset and + fetches each one from its corresponding named graph. For versioned ontologies, + it returns only the latest version for each unique ontology IRI. + + 1. Discovery: List all named graphs (which may be versioned URIs) + 2. Fetching: Retrieve each ontology from its named graph (in parallel) + 3. Deduplication: For versioned ontologies, keep only the latest version + + Returns: + list[Ontology]: List of the latest version of each ontology found. + + Example: + >>> ontologies = await manager.fetch_ontologies() + >>> for onto in ontologies: + ... print(f"Found ontology: {onto.iri} v{onto.version}") + """ + client = await self._get_client() + sparql_url = f"{self._get_ontologies_dataset_url()}/sparql" + + # Step 1: List all named graphs + list_query = """ + SELECT DISTINCT ?g WHERE { + GRAPH ?g { ?s ?p ?o } + } + """ + response = await client.post( + sparql_url, + data={"query": list_query, "format": "application/sparql-results+json"}, + ) + if response.status_code != 200: + logger.error(f"Failed to list graphs from Fuseki: {response.text}") + return [] + + results = response.json() + graph_uris = [] + for binding in results.get("results", {}).get("bindings", []): + graph_uri = binding["g"]["value"] + graph_uris.append(graph_uri) + + logger.debug(f"Found {len(graph_uris)} named graphs: {graph_uris}") + + # Step 2: Fetch each ontology from its corresponding named graph (in parallel) + async def fetch_single_ontology(graph_uri: str) -> Ontology | None: + """Fetch a single ontology from a graph URI.""" + try: + graph = RDFGraph() + # URL encode the graph URI to handle special characters like # + encoded_graph_uri = quote(str(graph_uri), safe="/:") + export_url = f"{self._get_ontologies_dataset_url()}/get?graph={encoded_graph_uri}" + export_resp = await client.get( + export_url, headers={"Accept": "text/turtle"} + ) + + if export_resp.status_code == 200: + graph.parse(data=export_resp.text, format="turtle") + + # Re-serialize deterministically to ensure consistent cache keys + # This sorts both namespaces and triples alphabetically + deterministic_turtle = deterministic_turtle_serialization(graph) + + # Re-parse from deterministic serialization to ensure we have RDFGraph + deterministic_graph = RDFGraph() + deterministic_graph.parse( + data=deterministic_turtle, format="turtle" + ) + + # Copy namespace bindings from original graph + for prefix, namespace in graph.namespaces(): + if prefix: + deterministic_graph.bind(prefix, namespace) + + graph = deterministic_graph + + # Find the ontology IRI in the graph + for onto_subj, _, obj in graph.triples( + (None, RDF.type, OWL.Ontology) + ): + onto_iri = str(onto_subj) + # Extract base IRI if graph_uri is versioned + # Handle both hash fragments (#19193944...) and semantic versions (#v1.2.3) + if "#" in graph_uri: + base_iri = graph_uri.split("#")[0] + # Use base IRI from graph_uri (named graph identifier) + # The graph content should have simplified IRI, but use graph_uri as source of truth + onto_iri = base_iri + + ontology = Ontology( + graph=graph, + iri=onto_iri, + ) + # Load properties from graph (will strip any hash fragments if present) + ontology.sync_properties_from_graph() + logger.debug( + f"Successfully loaded ontology: {onto_iri} version: {ontology.version}" + ) + return ontology + else: + logger.warning( + f"Failed to fetch graph {graph_uri}: {export_resp.status_code}" + ) + except Exception as e: + logger.warning(f"Error fetching ontology from {graph_uri}: {e}") + return None + + # Fetch all ontologies in parallel + all_ontologies_results = await asyncio.gather( + *[fetch_single_ontology(uri) for uri in graph_uris], return_exceptions=True + ) + + # Filter out None and exceptions + all_ontologies = [] + for result in all_ontologies_results: + if isinstance(result, Exception): + logger.warning(f"Exception fetching ontology: {result}") + elif result is not None: + all_ontologies.append(result) + + # Step 3: Deduplicate and keep latest terminal versions + ontology_dict = defaultdict(list) + + for onto in all_ontologies: + ontology_dict[onto.iri].append(onto) + + # Build set of all parent hashes to identify terminal ontologies + # A terminal ontology is one that is not a parent for any other ontology + all_parent_hashes = set() + + for onto in all_ontologies: + if onto.hash: + # Collect all parent hashes + for parent_hash in onto.parent_hashes: + all_parent_hashes.add(parent_hash) + + # For each unique IRI, select the latest terminal ontology + ontologies = [] + + for iri, versions in ontology_dict.items(): + if len(versions) == 1: + ontologies.append(versions[0]) + else: + # Multiple versions - find terminal ontologies (not parents) + terminal_versions = [ + v for v in versions if v.hash and v.hash not in all_parent_hashes + ] + + if not terminal_versions: + # No terminal ontologies found - all are parents + # Fall back to non-terminal versions + logger.warning( + f"No terminal ontologies found for {iri}, " + f"using all versions for selection" + ) + terminal_versions = versions + + # Select latest by created_at among terminal ontologies + try: + versions_with_created = [ + v for v in terminal_versions if v.created_at is not None + ] + + if versions_with_created: + # Sort by created_at (most recent first) + versions_with_created.sort( + key=lambda x: x.created_at, reverse=True + ) + selected = versions_with_created[0] + hash_str = ( + f"{selected.hash[:16]}..." if selected.hash else "no hash" + ) + logger.debug( + f"Selected terminal ontology for {iri} " + f"by created_at: {selected.created_at} " + f"(hash: {hash_str})" + ) + ontologies.append(selected) + else: + # No created_at available - fall back to version-based sorting + versions_with_ver = [v for v in terminal_versions if v.version] + if versions_with_ver: + versions_with_ver.sort( + key=lambda x: str(x.version), reverse=False + ) + selected = versions_with_ver[-1] + logger.debug( + f"Selected terminal ontology for {iri} " + f"by version: {selected.version} " + f"(no created_at available)" + ) + ontologies.append(selected) + else: + # No version info either - use first terminal ontology + selected = terminal_versions[0] + logger.debug( + f"Selected first terminal ontology for {iri} " + f"(no created_at or version available)" + ) + ontologies.append(selected) + except Exception as e: + logger.warning( + f"Could not select terminal ontology for {iri}: {e}, " + f"using first version" + ) + ontologies.append(terminal_versions[0]) + + logger.info( + f"Successfully loaded {len(ontologies)} unique ontologies from Fuseki " + ) + return ontologies + + def serialize_graph(self, graph: Graph, **kwargs) -> bool | None: + """Synchronous wrapper for serialize_graph. + + For async usage, use aserialize_graph() instead. + """ + return asyncio.run(self._serialize_graph_with_cleanup(graph, **kwargs)) + + async def aserialize_graph(self, graph: Graph, **kwargs) -> bool | None: + """Async version of serialize_graph. + + This is the preferred method when running in an async context. + """ + return await self._serialize_graph_async(graph, **kwargs) + + async def _serialize_graph_with_cleanup( + self, graph: Graph, **kwargs + ) -> bool | None: + """Wrapper that ensures proper cleanup when using asyncio.run(). + + This method creates a temporary client and ensures it's properly closed + before returning, preventing "Event loop is closed" errors. + """ + async with httpx.AsyncClient( + auth=self._prepare_auth(), timeout=30.0 + ) as temp_client: + # Temporarily replace the client + original_client = self._client + self._client = temp_client + try: + return await self._serialize_graph_async(graph, **kwargs) + finally: + # Restore original client + self._client = original_client + + async def _serialize_graph_async(self, graph: Graph, **kwargs) -> bool | None: + """Store an RDF graph as a named graph in a specific Fuseki dataset. + + This is a private helper method that handles the common logic for storing + graphs in Fuseki datasets. + + Args: + graph: The RDF graph to store. + **kwargs: Additional parameters including graph_uri, dataset_url, default_graph_uri, log_prefix. + + Returns: + bool: True if the graph was successfully stored, False otherwise. + """ + client = await self._get_client() + graph_uri = kwargs.get("graph_uri") + dataset_url = kwargs.get("dataset_url") + default_graph_uri = kwargs.get("default_graph_uri") + log_prefix = kwargs.get("log_prefix") + + turtle_data = graph.serialize(format="turtle") + if graph_uri is None: + graph_uri = default_graph_uri + + # URL encode the graph URI to handle special characters like # + encoded_graph_uri = quote(str(graph_uri), safe="/:") + url = f"{dataset_url}/data?graph={encoded_graph_uri}" + headers = {"Content-Type": "text/turtle;charset=utf-8"} + response = await client.put(url, headers=headers, content=turtle_data) + if response.status_code in (200, 201, 204): + logger.info( + f"{log_prefix} graph {graph_uri} uploaded to Fuseki as named graph." + ) + return True + else: + logger.error( + f"Failed to upload {log_prefix.lower() if log_prefix else 'unknown'} graph {graph_uri}. Status code: {response.status_code}" + ) + logger.error(f"Response: {response.text}") + return False + + def serialize(self, o: Ontology | RDFGraph, **kwargs) -> bool | None: + """Synchronous wrapper for serialize. + + For async usage, use aserialize() instead. + """ + return asyncio.run(self._serialize_with_cleanup(o, **kwargs)) + + async def aserialize(self, o: Ontology | RDFGraph, **kwargs) -> bool | None: + """Async version of serialize. + + This is the preferred method when running in an async context. + """ + return await self._serialize_async(o, **kwargs) + + async def _serialize_with_cleanup( + self, o: Ontology | RDFGraph, **kwargs + ) -> bool | None: + """Wrapper that ensures proper cleanup when using asyncio.run(). + + This method creates a temporary client and ensures it's properly closed + before returning, preventing "Event loop is closed" errors. + """ + async with httpx.AsyncClient( + auth=self._prepare_auth(), timeout=30.0 + ) as temp_client: + # Temporarily replace the client + original_client = self._client + self._client = temp_client + try: + return await self._serialize_async(o, **kwargs) + finally: + # Restore original client + self._client = original_client + + async def _serialize_async(self, o: Ontology | RDFGraph, **kwargs) -> bool | None: + """Store an RDF graph as a named graph in Fuseki. + + This method stores the given RDF graph as a named graph in Fuseki. + The graph name is taken from the graph_uri parameter or defaults to + "urn:data:default". + + Args: + o: RDF graph or Ontology object. + **kwargs: Additional parameters including graph_uri. + + Returns: + bool: True if the graph was successfully stored, False otherwise. + + Example: + >>> graph = RDFGraph() + >>> success = await manager.serialize(graph) + + >>> success = await manager.serialize(graph, graph_uri="http://example.org/chunk1") + """ + graph_uri = kwargs.get("graph_uri") + + if isinstance(o, Ontology): + graph = o.graph + # Use versioned IRI for storage to enable multiple versions to coexist + graph_uri = o.versioned_iri + default_graph_uri = "urn:ontology:default" + log_prefix = "Ontology" + # Use ontologies dataset for ontology storage + dataset_url = self._get_ontologies_dataset_url() + elif isinstance(o, RDFGraph): + graph = o + default_graph_uri = "urn:data:default" + log_prefix = "Graph" + # Use regular dataset for facts storage + dataset_url = self._get_dataset_url() + else: + raise TypeError(f"unsupported obj of type {type(o)} received") + + return await self._serialize_graph_async( + graph=graph, + graph_uri=graph_uri, + dataset_url=dataset_url, + default_graph_uri=default_graph_uri, + log_prefix=log_prefix, + ) diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/mock.py b/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/mock.py new file mode 100644 index 0000000..4a3728c --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/mock.py @@ -0,0 +1,502 @@ +"""Mock triple store implementations for testing. + +This module provides mock implementations of triple store managers that simulate +the behavior of real triple stores (Fuseki, Neo4j) without requiring external +services. These mocks are useful for testing and development. + +The mocks maintain in-memory storage and provide the same interface as the +real implementations, allowing tests to run without external dependencies. +""" + +import logging +from typing import Any, Dict, List + +from pydantic import Field +from rdflib import Graph, URIRef +from rdflib.namespace import OWL, RDF + +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.util import derive_ontology_id +from ontocast.tool.triple_manager.core import ( + TripleStoreManager, + TripleStoreManagerWithAuth, +) + +logger = logging.getLogger(__name__) + + +class MockTripleStoreManager(TripleStoreManager): + """Mock triple store manager for testing. + + This class provides an in-memory implementation of triple store operations + that simulates the behavior of real triple stores without requiring external + services. It stores ontologies and graphs in memory and provides the same + interface as concrete implementations. + + Attributes: + ontologies: In-memory storage for ontologies. + graphs: In-memory storage for RDF graphs. + """ + + model_config = {"arbitrary_types_allowed": True} + + ontologies: List[Ontology] = Field( + default_factory=list, description="In-memory storage for ontologies" + ) + graphs: Dict[str, Graph] = Field( + default_factory=dict, description="In-memory storage for RDF graphs" + ) + + def __init__(self, **kwargs): + """Initialize the mock triple store manager. + + Args: + **kwargs: Additional keyword arguments passed to the parent class. + """ + super().__init__(**kwargs) + + def fetch_ontologies(self) -> List[Ontology]: + """Fetch all available ontologies from the mock store. + + Returns: + List[Ontology]: List of available ontologies with their graphs. + """ + return self.ontologies.copy() + + def serialize_graph(self, graph: Graph, **kwargs) -> bool | None: + """Store an RDF graph in the mock store. + + Args: + graph: The RDF graph to store. + **kwargs: Optional keyword arguments including graph_uri. + + Returns: + bool: True if the graph was stored successfully. + """ + graph_uri = kwargs.get("graph_uri") + # Create a new Graph and copy all triples + new_graph = Graph() + for triple in graph: + new_graph.add(triple) + + if graph_uri: + self.graphs[graph_uri] = new_graph + else: + # Generate a default URI based on graph content + graph_uri = f"mock://graph/{len(self.graphs)}" + self.graphs[graph_uri] = new_graph + + # Try to extract ontology information from the graph + ontology_id = self._extract_ontology_id(graph) + if ontology_id: + ontology = Ontology( + ontology_id=ontology_id, + title=f"Mock Ontology {ontology_id}", + description="Mock ontology for testing", + version="1.0.0", + iri=graph_uri, + graph=self._create_rdf_graph_from_graph(graph), + ) + # Update existing ontology or add new one + existing = next( + (o for o in self.ontologies if o.ontology_id == ontology_id), None + ) + if existing: + existing.graph = self._create_rdf_graph_from_graph(graph) + existing.iri = graph_uri + else: + self.ontologies.append(ontology) + + return True + + def serialize(self, o: Ontology | RDFGraph, **kwargs) -> bool | None: + """Store an Ontology or RDFGraph in the mock store. + + Args: + o: Ontology or RDFGraph object to store. + **kwargs: Additional keyword arguments. + + Returns: + bool: True if the object was stored successfully. + """ + if isinstance(o, Ontology): + graph = o.graph + graph_uri = o.iri + elif isinstance(o, RDFGraph): + graph = o + graph_uri = kwargs.get("graph_uri") + else: + raise TypeError(f"unsupported obj of type {type(o)} received") + + return self.serialize_graph(graph, graph_uri=graph_uri) + + def _extract_ontology_id(self, graph: Graph) -> str | None: + """Extract ontology ID from graph content. + + Args: + graph: The RDF graph to analyze. + + Returns: + str | None: The extracted ontology ID, or None if not found. + """ + # Look for owl:Ontology declarations + for s, p, o in graph.triples((None, RDF.type, OWL.Ontology)): + if isinstance(s, URIRef): + return derive_ontology_id(str(s)) + return None + + def clear(self): + """Clear all stored data.""" + self.ontologies.clear() + self.graphs.clear() + + async def clean(self, dataset: str | None = None) -> None: + """Clean/flush data from the mock triple store. + + Args: + dataset: Optional dataset name (ignored for mock, kept for interface compatibility). + """ + self.clear() + + def _create_rdf_graph_from_graph(self, graph: Graph) -> RDFGraph: + """Create an RDFGraph from a regular Graph by copying all triples. + + Args: + graph: The source graph to copy from. + + Returns: + RDFGraph: A new RDFGraph with all triples copied. + """ + rdf_graph = RDFGraph() + for triple in graph: + rdf_graph.add(triple) + return rdf_graph + + +class MockFusekiTripleStoreManager(TripleStoreManagerWithAuth): + """Mock Fuseki triple store manager for testing. + + This class simulates the behavior of FusekiTripleStoreManager without + requiring an actual Fuseki server. It maintains in-memory storage and + provides the same interface as the real implementation. + + Attributes: + dataset: The mock dataset name. + ontologies_dataset: The mock ontologies dataset name. + ontologies: In-memory storage for ontologies. + graphs: In-memory storage for RDF graphs. + """ + + model_config = {"arbitrary_types_allowed": True} + + dataset: str | None = None + ontologies_dataset: str = "ontologies" + ontologies: List[Ontology] = Field( + default_factory=list, description="In-memory storage for ontologies" + ) + graphs: Dict[str, Graph] = Field( + default_factory=dict, description="In-memory storage for RDF graphs" + ) + + def __init__( + self, + uri=None, + auth=None, + dataset=None, + ontologies_dataset=None, + clean=False, + **kwargs, + ): + """Initialize the mock Fuseki triple store manager. + + Args: + uri: Mock URI (ignored but kept for interface compatibility). + auth: Mock authentication (ignored but kept for interface compatibility). + dataset: Mock dataset name. + ontologies_dataset: Mock ontologies dataset name. + clean: Whether to clean the store on initialization. + **kwargs: Additional keyword arguments. + """ + super().__init__(uri=uri, auth=auth, **kwargs) + self.dataset = dataset or "test" + self.ontologies_dataset = ontologies_dataset or "ontologies" + + if clean: + self.clear() + + def fetch_ontologies(self) -> List[Ontology]: + """Fetch all available ontologies from the mock store. + + Returns: + List[Ontology]: List of available ontologies with their graphs. + """ + return self.ontologies.copy() + + def serialize_graph(self, graph: Graph, **kwargs) -> bool | None: + """Store an RDF graph in the mock store. + + Args: + graph: The RDF graph to store. + **kwargs: Optional keyword arguments including graph_uri. + + Returns: + bool: True if the graph was stored successfully. + """ + graph_uri = kwargs.get("graph_uri") + # Create a new Graph and copy all triples + new_graph = Graph() + for triple in graph: + new_graph.add(triple) + + if graph_uri: + self.graphs[graph_uri] = new_graph + else: + # Generate a default URI based on graph content + graph_uri = f"mock://{self.dataset}/graph/{len(self.graphs)}" + self.graphs[graph_uri] = new_graph + + # Try to extract ontology information from the graph + ontology_id = self._extract_ontology_id(graph) + if ontology_id: + ontology = Ontology( + ontology_id=ontology_id, + title=f"Mock Ontology {ontology_id}", + description="Mock ontology for testing", + version="1.0.0", + iri=graph_uri, + graph=self._create_rdf_graph_from_graph(graph), + ) + # Update existing ontology or add new one + existing = next( + (o for o in self.ontologies if o.ontology_id == ontology_id), None + ) + if existing: + existing.graph = self._create_rdf_graph_from_graph(graph) + existing.iri = graph_uri + else: + self.ontologies.append(ontology) + + return True + + def serialize(self, o: Ontology | RDFGraph, **kwargs) -> bool | None: + """Store an Ontology or RDFGraph in the mock store. + + Args: + o: Ontology or RDFGraph object to store. + **kwargs: Additional keyword arguments. + + Returns: + bool: True if the object was stored successfully. + """ + if isinstance(o, Ontology): + graph = o.graph + graph_uri = o.iri + elif isinstance(o, RDFGraph): + graph = o + graph_uri = kwargs.get("graph_uri") + else: + raise TypeError(f"unsupported obj of type {type(o)} received") + + return self.serialize_graph(graph, graph_uri=graph_uri) + + def _extract_ontology_id(self, graph: Graph) -> str | None: + """Extract ontology ID from graph content. + + Args: + graph: The RDF graph to analyze. + + Returns: + str | None: The extracted ontology ID, or None if not found. + """ + # Look for owl:Ontology declarations + for s, p, o in graph.triples((None, RDF.type, OWL.Ontology)): + if isinstance(s, URIRef): + return derive_ontology_id(str(s)) + return None + + def clear(self): + """Clear all stored data.""" + self.ontologies.clear() + self.graphs.clear() + + async def clean(self, dataset: str | None = None) -> None: + """Clean/flush data from the mock Fuseki triple store. + + Args: + dataset: Optional dataset name (ignored for mock, kept for interface compatibility). + """ + self.clear() + + def _create_rdf_graph_from_graph(self, graph: Graph) -> RDFGraph: + """Create an RDFGraph from a regular Graph by copying all triples. + + Args: + graph: The source graph to copy from. + + Returns: + RDFGraph: A new RDFGraph with all triples copied. + """ + rdf_graph = RDFGraph() + for triple in graph: + rdf_graph.add(triple) + return rdf_graph + + +class MockNeo4jTripleStoreManager(TripleStoreManagerWithAuth): + """Mock Neo4j triple store manager for testing. + + This class simulates the behavior of Neo4jTripleStoreManager without + requiring an actual Neo4j server. It maintains in-memory storage and + provides the same interface as the real implementation. + + Attributes: + ontologies: In-memory storage for ontologies. + graphs: In-memory storage for RDF graphs. + """ + + model_config = {"arbitrary_types_allowed": True} + + ontologies: List[Ontology] = Field( + default_factory=list, description="In-memory storage for ontologies" + ) + graphs: Dict[str, Graph] = Field( + default_factory=dict, description="In-memory storage for RDF graphs" + ) + + def __init__(self, uri=None, auth=None, clean=False, **kwargs): + """Initialize the mock Neo4j triple store manager. + + Args: + uri: Mock URI (ignored but kept for interface compatibility). + auth: Mock authentication (ignored but kept for interface compatibility). + clean: Whether to clean the store on initialization. + **kwargs: Additional keyword arguments. + """ + super().__init__(uri=uri, auth=auth, **kwargs) + + if clean: + self.clear() + + def fetch_ontologies(self) -> List[Ontology]: + """Fetch all available ontologies from the mock store. + + Returns: + List[Ontology]: List of available ontologies with their graphs. + """ + return self.ontologies.copy() + + def serialize_graph(self, graph: Graph, **kwargs) -> Dict[str, Any] | None: # type: ignore[override] + """Store an RDF graph in the mock store. + + Args: + graph: The RDF graph to store. + **kwargs: Optional keyword arguments including graph_uri. + + Returns: + Dict[str, Any]: Mock summary of the operation. + """ + graph_uri = kwargs.get("graph_uri") + # Create a new Graph and copy all triples + new_graph = Graph() + for triple in graph: + new_graph.add(triple) + + if graph_uri: + self.graphs[graph_uri] = new_graph + else: + # Generate a default URI based on graph content + graph_uri = f"mock://neo4j/graph/{len(self.graphs)}" + self.graphs[graph_uri] = new_graph + + # Try to extract ontology information from the graph + ontology_id = self._extract_ontology_id(graph) + if ontology_id: + ontology = Ontology( + ontology_id=ontology_id, + title=f"Mock Ontology {ontology_id}", + description="Mock ontology for testing", + version="1.0.0", + iri=graph_uri, + graph=self._create_rdf_graph_from_graph(graph), + ) + # Update existing ontology or add new one + existing = next( + (o for o in self.ontologies if o.ontology_id == ontology_id), None + ) + if existing: + existing.graph = self._create_rdf_graph_from_graph(graph) + existing.iri = graph_uri + else: + self.ontologies.append(ontology) + + # Return mock summary similar to Neo4j + return { + "nodes_created": len(graph), + "relationships_created": 0, + "properties_set": len(graph), + "labels_added": 1, + } + + def serialize(self, o: Ontology | RDFGraph, **kwargs) -> Dict[str, Any] | None: # type: ignore[override] + """Store an Ontology or RDFGraph in the mock store. + + Args: + o: Ontology or RDFGraph object to store. + **kwargs: Additional keyword arguments. + + Returns: + Dict[str, Any]: Mock summary of the operation. + """ + if isinstance(o, Ontology): + graph = o.graph + graph_uri = o.iri + elif isinstance(o, RDFGraph): + graph = o + graph_uri = kwargs.get("graph_uri") + else: + raise TypeError(f"unsupported obj of type {type(o)} received") + + return self.serialize_graph(graph, graph_uri=graph_uri) + + def _extract_ontology_id(self, graph: Graph) -> str | None: + """Extract ontology ID from graph content. + + Args: + graph: The RDF graph to analyze. + + Returns: + str | None: The extracted ontology ID, or None if not found. + """ + # Look for owl:Ontology declarations + for s, p, o in graph.triples((None, RDF.type, OWL.Ontology)): + if isinstance(s, URIRef): + return derive_ontology_id(str(s)) + return None + + def clear(self): + """Clear all stored data.""" + self.ontologies.clear() + self.graphs.clear() + + async def clean(self, dataset: str | None = None) -> None: + """Clean/flush data from the mock Neo4j triple store. + + Args: + dataset: Optional dataset name (ignored for Neo4j mock, kept for interface compatibility). + """ + self.clear() + + def _create_rdf_graph_from_graph(self, graph: Graph) -> RDFGraph: + """Create an RDFGraph from a regular Graph by copying all triples. + + Args: + graph: The source graph to copy from. + + Returns: + RDFGraph: A new RDFGraph with all triples copied. + """ + rdf_graph = RDFGraph() + for triple in graph: + rdf_graph.add(triple) + return rdf_graph diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/neo4j.py b/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/neo4j.py new file mode 100644 index 0000000..89940de --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/triple_manager/neo4j.py @@ -0,0 +1,476 @@ +"""Neo4j triple store management for OntoCast. + +This module provides a concrete implementation of triple store management +using Neo4j with the n10s (neosemantics) plugin. It handles RDF data +faithfully by using both n10s property graph representation and raw RDF +triple storage for accurate reconstruction. +""" + +import logging +from typing import Any + +from neo4j import GraphDatabase +from rdflib import Graph +from rdflib.namespace import OWL, RDF + +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.util import derive_ontology_id +from ontocast.tool.triple_manager.core import TripleStoreManagerWithAuth + +logger = logging.getLogger(__name__) + + +class Neo4jTripleStoreManager(TripleStoreManagerWithAuth): + """Neo4j-based triple store manager using n10s (neosemantics) plugin. + + This implementation handles RDF data more faithfully by using both the n10s + property graph representation and raw RDF triple storage for accurate reconstruction. + It provides comprehensive ontology management with namespace-based organization. + + The manager uses Neo4j's n10s plugin for RDF operations, including: + - RDF import and export via n10s + - Ontology metadata storage and retrieval + - Namespace-based ontology organization + - Faithful RDF graph reconstruction + + Attributes: + _driver: Private Neo4j driver instance. + """ + + _driver: Any = None # private attribute, not a pydantic field + + def __init__(self, uri=None, auth=None, **kwargs): + """Initialize the Neo4j triple store manager. + + This method sets up the connection to Neo4j, initializes the n10s + plugin configuration, and creates necessary constraints and indexes. + The database is NOT cleaned on initialization. + + Args: + uri: Neo4j connection URI (e.g., "bolt://localhost:7687"). + auth: Authentication tuple (username, password) or string in "user/password" format. + **kwargs: Additional keyword arguments passed to the parent class. + + Raises: + ImportError: If the neo4j Python driver is not installed. + + Example: + >>> manager = Neo4jTripleStoreManager( + ... uri="bolt://localhost:7687", + ... auth="neo4j/password" + ... ) + >>> # To clean the database, use the clean() method explicitly: + >>> await manager.clean() + """ + super().__init__( + uri=uri, auth=auth, env_uri="NEO4J_URI", env_auth="NEO4J_AUTH", **kwargs + ) + if GraphDatabase is None: + raise ImportError("neo4j Python driver is not installed.") + if self.uri is None: + raise ValueError("Neo4j URI is required but not provided.") + self._driver = GraphDatabase.driver(self.uri, auth=self.auth) + + # Type assertion: we know _driver is not None after initialization + assert self._driver is not None + + with self._driver.session() as session: + # Initialize n10s configuration + self._init_n10s_config(session) + + # Create constraints and indexes + self._create_constraints_and_indexes(session) + + async def clean(self, dataset: str | None = None) -> None: + """Clean/flush all data from the Neo4j database. + + This method deletes all nodes and relationships from the Neo4j database, + effectively clearing all stored data. + + Args: + dataset: Optional dataset parameter (ignored for Neo4j, which doesn't + support datasets). Included for interface compatibility. + + Warning: This operation is irreversible and will delete all data. + + Raises: + Exception: If the cleanup operation fails. + """ + if dataset is not None: + logger.warning( + f"Dataset parameter '{dataset}' ignored for Neo4j (datasets not supported)" + ) + + if self._driver is None: + raise ValueError("Neo4j driver is not initialized") + + with self._driver.session() as session: + try: + session.run("MATCH (n) DETACH DELETE n") + logger.info("Neo4j database cleaned (all nodes deleted)") + except Exception as e: + logger.error(f"Neo4j cleanup failed: {e}") + raise + + def _init_n10s_config(self, session): + """Initialize n10s configuration with better RDF handling. + + This method configures the n10s plugin for optimal RDF handling. + It sets up the configuration to preserve vocabulary URIs, handle + multivalued properties, and maintain RDF types as nodes. + + Args: + session: Neo4j session for executing configuration commands. + """ + try: + # Check if already configured + result = session.run("CALL n10s.graphconfig.show()") + if result.single(): + logger.debug("n10s already configured") + except: + pass + + try: + session.run(""" + CALL n10s.graphconfig.init({ + handleVocabUris: "KEEP", + handleMultival: "OVERWRITE", + typesToLabels: false, + keepLangTag: false, + keepCustomDataTypes: true, + handleRDFTypes: "NODES" + }) + """) + logger.debug("n10s configuration initialized") + except Exception as e: + logger.warning(f"n10s configuration failed: {e}") + + def _create_constraints_and_indexes(self, session): + """Create necessary constraints and indexes for optimal performance. + + This method creates Neo4j constraints and indexes that are needed + for efficient ontology operations and data integrity. + + Args: + session: Neo4j session for executing constraint/index creation commands. + """ + constraints = [ + "CREATE CONSTRAINT n10s_unique_uri IF NOT EXISTS FOR (r:Resource) REQUIRE r.uri IS UNIQUE", + "CREATE CONSTRAINT ontology_iri_unique IF NOT EXISTS FOR (o:Ontology) REQUIRE o.uri IS UNIQUE", + "CREATE INDEX namespace_prefix IF NOT EXISTS FOR (ns:Namespace) ON (ns.prefix)", + ] + + for constraint in constraints: + try: + session.run(constraint) + logger.debug(f"Created constraint/index: {constraint.split()[-1]}") + except Exception as e: + logger.debug(f"Constraint/index creation (might already exist): {e}") + + def _extract_namespace_prefix(self, uri: str) -> tuple[str, str]: + """Extract namespace and local name from URI. + + This method parses a URI to extract the namespace and local name + using common separators (#, /, :). + + Args: + uri: The URI to parse. + + Returns: + tuple[str, str]: A tuple of (namespace, local_name). + + Example: + >>> manager._extract_namespace_prefix("http://example.org/onto#Class") + ("http://example.org/onto#", "Class") + """ + common_separators = ["#", "/", ":"] + for sep in common_separators: + if sep in uri: + parts = uri.rsplit(sep, 1) + if len(parts) == 2: + return parts[0] + sep, parts[1] + return uri, "" + + def _get_ontology_namespaces(self, session) -> dict: + """Get all known ontology namespaces from the database. + + This method queries the Neo4j database to retrieve all known + namespace prefixes and their corresponding URIs. + + Args: + session: Neo4j session for executing queries. + + Returns: + dict: Dictionary mapping namespace prefixes to URIs. + """ + result = session.run(""" + MATCH (ns:Namespace) + RETURN ns.prefix as prefix, ns.uri as uri + UNION + MATCH (o:Ontology) + RETURN null as prefix, o.uri as uri + """) + + namespaces = {} + for record in result: + uri = record.get("uri") + prefix = record.get("prefix") + if uri: + if prefix: + namespaces[prefix] = uri + else: + # Extract potential namespace from ontology URI + ns, _ = self._extract_namespace_prefix(uri) + if ns != uri: # Only if we actually found a namespace + namespaces[ns] = ns + + return namespaces + + def fetch_ontologies(self) -> list[Ontology]: + """Fetch ontologies from Neo4j with faithful RDF reconstruction. + + This method retrieves all ontologies from Neo4j and reconstructs + their RDF graphs faithfully. It uses a multi-step process: + + 1. Identifies distinct ontologies by their namespace URIs + 2. Fetches all entities belonging to each ontology + 3. Reconstructs the RDF graph faithfully using stored triples when available + 4. Falls back to n10s property graph conversion when needed + + Returns: + list[Ontology]: List of all ontologies found in the database. + + Example: + >>> ontologies = manager.fetch_ontologies() + >>> for onto in ontologies: + ... print(f"Found ontology: {onto.iri}") + """ + ontologies = [] + + # Type assertion: we know _driver is not None after initialization + assert self._driver is not None + with self._driver.session() as session: + try: + # First, try to get explicitly stored ontology metadata + ontology_iris = self._fetch_ontology_iris(session) + + if ontology_iris: + for ont_iri in ontology_iris: + ontology = self._reconstruct_ontology_from_metadata( + session, ont_iri + ) + if ontology: + ontologies.append(ontology) + + except Exception as e: + logger.error(f"Error in fetch_ontologies: {e}") + + logger.info(f"Successfully loaded {len(ontologies)} ontologies") + return ontologies + + def _fetch_ontology_iris(self, session) -> list[str]: + """Fetch explicit ontology metadata from Neo4j. + + This method queries Neo4j to find all entities that are explicitly + typed as owl:Ontology. + + Args: + session: Neo4j session for executing queries. + + Returns: + list[str]: List of ontology IRIs found in the database. + """ + result = session.run(f""" + MATCH (o)-[:`{str(RDF.type)}`]->(t:Resource {{ uri: "{str(OWL.Ontology)}" }}) + WHERE o.uri IS NOT NULL + RETURN + o.uri AS iri + """) + + iris = [] + for record in result: + iri = record.get("iri", None) + iris += [iri] + iris = [iri for iri in iris if iri is not None] + return iris + + def _reconstruct_ontology_from_metadata(self, session, iri) -> Ontology | None: + """Reconstruct an ontology from its metadata and related entities. + + This method takes an ontology IRI and reconstructs the complete + ontology by fetching all related entities from the namespace. + + Args: + session: Neo4j session for executing queries. + iri: The ontology IRI to reconstruct. + + Returns: + Ontology | None: The reconstructed ontology, or None if failed. + """ + namespace_uri, _ = self._extract_namespace_prefix(iri) + + logger.debug(f"Reconstructing ontology: {iri} with namespace: {namespace_uri}") + + # Fallback to n10s export for this namespace + graph = self._export_namespace_via_n10s(session, namespace_uri) + if graph and len(graph) > 0: + return self._create_ontology_object(iri, iri, graph) + + def _export_namespace_via_n10s( + self, session, namespace_uri: str + ) -> RDFGraph | None: + """Export entities belonging to a namespace using n10s. + + This method uses Neo4j's n10s plugin to export all entities + belonging to a specific namespace as RDF triples. + + Args: + session: Neo4j session for executing queries. + namespace_uri: The namespace URI to export. + + Returns: + RDFGraph | None: The exported RDF graph, or None if failed. + """ + try: + result = session.run( + f""" + CALL n10s.rdf.export.cypher( + 'MATCH (n)-[r]->(m) WHERE n.uri STARTS WITH "{namespace_uri}" RETURN n,r,m', + {{format: 'Turtle'}} + ) + YIELD subject, predicate, object, isLiteral, literalType, literalLang + RETURN subject, predicate, object, isLiteral, literalType, literalLang + """ + ) + + # Process into Turtle format + turtle_lines = [] + + for record in result: + subj = record["subject"] + pred = record["predicate"] + obj = record["object"] + is_literal = record["isLiteral"] + literal_type = record["literalType"] + literal_lang = record["literalLang"] + + # Format object + if is_literal: + # Escape special characters in literals + obj = obj.replace('"', r"\"") + obj_str = f'"{obj}"' + + # Add datatype or language tag if present + if literal_lang: + obj_str += f"@{literal_lang}" + elif literal_type: + obj_str += f"^^<{literal_type}>" + else: + obj_str = f"<{obj}>" + + # Format triple + turtle_lines.append(f"<{subj}> <{pred}> {obj_str} .") + + # Combine into single string + turtle_string = "\n".join(turtle_lines) + + if turtle_string.strip(): + graph = RDFGraph() + graph.parse(data=turtle_string, format="turtle") + logger.debug( + f"Exported {len(graph)} triples via n10s for namespace {namespace_uri}" + ) + return graph + return None + + except Exception as e: + logger.debug( + f"Failed to export via n10s for namespace {namespace_uri}: {e}" + ) + + return None + + def _create_ontology_object( + self, iri: str, metadata: dict, graph: RDFGraph + ) -> Ontology: + """Create an Ontology object from IRI, metadata, and graph. + + Args: + iri: The ontology IRI. + metadata: Metadata dictionary (currently unused, kept for compatibility). + graph: The RDF graph containing the ontology data. + + Returns: + Ontology: The created ontology object. + """ + ontology_id = derive_ontology_id(iri) + return Ontology(graph=graph, iri=iri, ontology_id=ontology_id) + + def serialize_graph(self, graph: Graph, **kwargs) -> bool | None: + """Serialize an RDF graph to Neo4j with both n10s and raw triple storage. + + This method stores the given RDF graph in Neo4j using the n10s plugin + for RDF import. The data is stored as RDF triples that can be faithfully + reconstructed later. + + Args: + graph: The RDF graph to store. + **kwargs: Additional parameters (not used by Neo4j implementation). + + Returns: + Any: The result summary from n10s import operation. + """ + # Convert to RDFGraph if needed + if not isinstance(graph, RDFGraph): + rdf_graph = RDFGraph() + for triple in graph: + rdf_graph.add(triple) + for prefix, namespace in graph.namespaces(): + rdf_graph.bind(prefix, namespace) + graph = rdf_graph + + turtle_data = graph.serialize(format="turtle") + + # Type assertion: we know _driver is not None after initialization + assert self._driver is not None + with self._driver.session() as session: + # Store via n10s for graph queries + result = session.run( + "CALL n10s.rdf.import.inline($ttl, 'Turtle')", ttl=turtle_data + ) + summary = result.single() + + return summary + + def serialize(self, o: Ontology | RDFGraph, **kwargs) -> bool | None: + """Serialize an Ontology or RDFGraph to Neo4j with both n10s and raw triple storage. + + This method stores the given Ontology or RDFGraph in Neo4j using the n10s plugin + for RDF import. The data is stored as RDF triples that can be faithfully + reconstructed later. + + Args: + o: Ontology or RDFGraph object to store. + **kwargs: Additional keyword arguments (not used by Neo4j implementation). + + Returns: + Any: The result summary from n10s import operation. + """ + if isinstance(o, Ontology): + graph = o.graph + elif isinstance(o, RDFGraph): + graph = o + else: + raise TypeError(f"unsupported obj of type {type(o)} received") + + return self.serialize_graph(graph) + + def close(self): + """Close the Neo4j driver connection. + + This method should be called when the manager is no longer needed + to properly close the database connection and free resources. + """ + if self._driver: + self._driver.close() diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/validate.py b/ontology_platform/vendored/ontocast/ontocast/tool/validate.py new file mode 100644 index 0000000..ec6e99a --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/validate.py @@ -0,0 +1,423 @@ +"""Validation tools for OntoCast. + +This module provides functionality for validating RDF graphs and chunks, +including connectivity validation and graph structure verification. +""" + +import logging +from collections import defaultdict, deque +from typing import cast + +from pydantic import BaseModel, ConfigDict, Field +from rdflib import RDF, RDFS, Graph, Literal, URIRef + +from ontocast.onto.constants import PROV, SCHEMA +from ontocast.onto.content_unit import ContentUnit +from ontocast.onto.rdfgraph import RDFGraph + +logger = logging.getLogger(__name__) + + +class PredicateStats(BaseModel): + """Type definition for predicate statistics.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + total: int = 0 + with_labels: int = 0 + with_domains: int = 0 + with_ranges: int = 0 + + +class PredicateValidationResult(BaseModel): + """Type definition for predicate validation results.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + has_required_properties: bool = True + domain_range_consistent: bool = True + missing_labels: list[str] = Field(default_factory=list) + domain_range_violations: list[str] = Field(default_factory=list) + predicate_stats: PredicateStats = Field(default_factory=PredicateStats) + + +class ConnectivityResult(BaseModel): + """Type definition for connectivity validation results.""" + + model_config = ConfigDict(arbitrary_types_allowed=True) + + is_fully_connected: bool = False + num_components: int = 0 + total_entities: int = 0 + components: list[set[URIRef]] = Field(default_factory=list) + isolated_entities: list[URIRef] = Field(default_factory=list) + largest_component_size: int = 0 + has_required_properties: bool = True + domain_range_consistent: bool = True + missing_labels: list[str] = Field(default_factory=list) + domain_range_violations: list[str] = Field(default_factory=list) + predicate_stats: PredicateStats = Field(default_factory=PredicateStats) + + +def validate_and_connect_content_unit( + unit: ContentUnit, + auto_connect: bool = True, +) -> ContentUnit: + """Validate and optionally connect a content unit graph. + + This function validates the connectivity of a content unit RDF graph and + optionally connects any disconnected components. + + Args: + unit: The content unit containing the RDF graph to validate. + auto_connect: Whether to automatically connect disconnected graphs. + + Returns: + ContentUnit: The content unit with a validated and optionally connected graph. + """ + + # Ensure an RDFGraph instance + if not isinstance(unit.graph, RDFGraph): + logger.warning("received an redflib.Graph rather than RDFGraph") + new_graph = RDFGraph() + # Cast to Graph to satisfy type checker + graph = cast(Graph, unit.graph) + for triple in graph: + new_graph.add(triple) + for prefix, namespace in graph.namespaces(): + new_graph.bind(prefix, namespace) + unit.graph = new_graph + + validator = RDFGraphConnectivityValidator(unit.graph) + + result = validator.validate_connectivity() + + logger.debug(f"\n=== Connectivity Analysis for Content Unit {unit.iri} ===") + logger.debug(f"Fully connected: {result.is_fully_connected}") + logger.debug(f"Number of components: {result.num_components}") + logger.debug(f"Total entities: {result.total_entities}") + logger.debug(f"Largest component size: {result.largest_component_size}") + + if result.isolated_entities: + logger.debug(f"Isolated entities: {[str(e) for e in result.isolated_entities]}") + + # Create a new RDFGraph instance instead of using deepcopy + final_graph = RDFGraph() + for triple in unit.graph: + final_graph.add(triple) + # Copy namespace bindings + for prefix, namespace in unit.graph.namespaces(): + final_graph.bind(prefix, namespace) + + if not result.is_fully_connected and auto_connect: + final_graph = validator.make_graph_connected(unit.iri) + + unit.graph = final_graph + return unit + + +def validate_and_connect_chunk( + chunk: ContentUnit, + auto_connect: bool = True, +) -> ContentUnit: + """Backward-compatible alias for validate_and_connect_content_unit().""" + return validate_and_connect_content_unit(unit=chunk, auto_connect=auto_connect) + + +class RDFGraphConnectivityValidator: + """Validator for RDF graph connectivity. + + This class provides functionality for validating and ensuring connectivity + in RDF graphs, including finding connected components and adding bridging + relationships. + + Attributes: + graph: The RDF graph to validate. + """ + + def __init__(self, graph: RDFGraph): + """Initialize the validator. + + Args: + graph: The RDF graph to validate. + """ + self.graph = graph + + def get_all_entities(self) -> set[URIRef]: + """Extract all unique entities from the graph. + + Returns: + set[URIRef]: Set of all unique entity URIs in the graph. + """ + entities = set() + + for subj, _, obj in self.graph: + if isinstance(subj, URIRef): + entities.add(subj) + if isinstance(obj, URIRef): + entities.add(obj) + + return entities + + def build_adjacency_graph(self) -> dict[URIRef, set[URIRef]]: + """Build an adjacency representation of the RDF graph. + + Returns: + dict[URIRef, set[URIRef]]: Dictionary mapping entities to their neighbors. + """ + adjacency = defaultdict(set) + + for subj, _, obj in self.graph: + if isinstance(subj, URIRef) and isinstance(obj, URIRef): + adjacency[subj].add(obj) + adjacency[obj].add(subj) # Treat as undirected for connectivity + + return adjacency + + def find_connected_components(self) -> list[set[URIRef]]: + """Find all connected components in the graph using BFS. + + Returns: + list[set[URIRef]]: List of sets, each containing entities in a component. + """ + entities = self.get_all_entities() + adjacency = self.build_adjacency_graph() + visited = set() + components = [] + + for entity in entities: + if entity not in visited: + component = set() + queue = deque([entity]) + + while queue: + current = queue.popleft() + if current not in visited: + visited.add(current) + component.add(current) + + # Add neighbors to queue + for neighbor in adjacency.get(current, set()): + if neighbor not in visited: + queue.append(neighbor) + + if component: + components.append(component) + + return components + + def validate_predicates(self) -> PredicateValidationResult: + """Validate predicate consistency and required properties. + + Returns: + PredicateValidationResult: Pydantic model containing validation results and statistics. + """ + result = PredicateValidationResult() + + # Track all predicates + predicates = set() + for _, pred, _ in self.graph: + if isinstance(pred, URIRef): + predicates.add(pred) + + result.predicate_stats.total = len(predicates) + + # Check each predicate + for pred in predicates: + has_label = False + has_domain = False + has_range = False + domain = None + range_ = None + + # Get predicate properties + for s, p, o in self.graph: + if s == pred: + if p == RDFS.label: + has_label = True + result.predicate_stats.with_labels += 1 + elif p == RDFS.domain: + has_domain = True + domain = o + result.predicate_stats.with_domains += 1 + elif p == RDFS.range: + has_range = True + range_ = o + result.predicate_stats.with_ranges += 1 + + # Check required properties + if not has_label: + result.has_required_properties = False + result.missing_labels.append(str(pred)) + + # Check domain/range consistency in usage + if has_domain or has_range: + for s, p, o in self.graph: + if p == pred: + if has_domain and isinstance(s, URIRef): + # Check if subject is of correct domain type + subject_type = None + for s2, p2, o2 in self.graph: + if s2 == s and p2 == RDF.type: + subject_type = o2 + break + + if subject_type and domain and subject_type != domain: + result.domain_range_consistent = False + result.domain_range_violations.append( + f"Subject {s} of type {subject_type} " + f"used with predicate {pred} " + f"that requires domain {domain}" + ) + + if has_range and isinstance(o, URIRef): + # Check if object is of correct range type + object_type = None + for s2, p2, o2 in self.graph: + if s2 == o and p2 == RDF.type: + object_type = o2 + break + + if object_type and range_ and object_type != range_: + result.domain_range_consistent = False + result.domain_range_violations.append( + f"Object {o} of type {object_type} " + f"used with predicate {pred} " + f"that requires range {range_}" + ) + + return result + + def validate_connectivity(self) -> ConnectivityResult: + """Validate graph connectivity and return detailed results. + + Returns: + ConnectivityResult: Pydantic model containing connectivity information and + validation results. + """ + components = self.find_connected_components() + entities = self.get_all_entities() + + result = ConnectivityResult( + is_fully_connected=len(components) <= 1, + num_components=len(components), + total_entities=len(entities), + components=components, + ) + + if components: + result.largest_component_size = max(len(comp) for comp in components) + + # Find isolated entities (components of size 1) + result.isolated_entities = [ + list(comp)[0] for comp in components if len(comp) == 1 + ] + + # Add predicate validation results + predicate_validation = self.validate_predicates() + result.has_required_properties = predicate_validation.has_required_properties + result.domain_range_consistent = predicate_validation.domain_range_consistent + result.missing_labels = predicate_validation.missing_labels + result.domain_range_violations = predicate_validation.domain_range_violations + result.predicate_stats = predicate_validation.predicate_stats + + return result + + def make_graph_connected(self, chunk_iri) -> RDFGraph: + """Make a disconnected graph connected by adding bridging relationships. + + Args: + chunk_iri: The IRI of the chunk to use for the hub entity. + + Returns: + RDFGraph: A new connected graph. + """ + components = self.find_connected_components() + + if len(components) <= 1: + logger.info("RDFGraph is already connected") + return self.graph + + # Create a new graph with all original triples + connected_graph = RDFGraph() + for triple in self.graph: + connected_graph.add(triple) + + # Copy namespace bindings + for prefix, namespace in self.graph.namespaces(): + connected_graph.bind(prefix, namespace) + + connected_graph = self._connect_via_chunk_hub( + connected_graph, components, chunk_iri + ) + + logger.info(f"Connected {len(components)} components") + return connected_graph + + def _connect_via_chunk_hub( + self, graph: RDFGraph, components: list[set[URIRef]], chunk_iri + ) -> RDFGraph: + """Connect components by creating a chunk hub entity. + + Args: + graph: The graph to modify. + components: List of connected components to connect. + chunk_iri: The IRI to use for the hub entity. + + Returns: + RDFGraph: The modified graph with connected components. + """ + # Create or use existing chunk URI + hub_uri = URIRef(chunk_iri) + hub_id = hub_uri.split("/")[-1] + + # Add hub entity metadata + graph.add((hub_uri, RDF.type, SCHEMA.TextDigitalDocument)) + graph.add((hub_uri, RDFS.label, Literal(f"Chunk {hub_id}"))) + + # Connect hub to one representative entity from each component + for i, component in enumerate(components): + # Choose representative entity (could be improved with better heuristics) + representative = self._choose_representative_entity(component, graph) + + if representative is not None: + # Add bidirectional connections + graph.add((hub_uri, SCHEMA.hasPart, representative)) + graph.add((representative, PROV.wasQuotedFrom, hub_uri)) + + return graph + + def _choose_representative_entity( + self, component: set[URIRef], graph: RDFGraph + ) -> URIRef | None: + """Choose the best representative entity from a component. + + Args: + component: Set of entities in the component. + graph: The RDF graph containing the entities. + + Returns: + URIRef | None: The chosen representative entity, or None if empty. + """ + if not component: + return None + + entity_degrees = {} + entities_with_labels = set() + + for entity in component: + # Count connections + degree = sum(1 for s, p, o in graph if s == entity or o == entity) + entity_degrees[entity] = degree + + # Check if entity has a label + for s, p, o in graph: + if s == entity and p in [RDFS.label, RDFS.comment]: + entities_with_labels.add(entity) + break + + # Prefer entities with labels and high degree + if entities_with_labels: + return max(entities_with_labels, key=lambda e: entity_degrees.get(e, 0)) + else: + return max(component, key=lambda e: entity_degrees.get(e, 0)) diff --git a/ontology_platform/vendored/ontocast/ontocast/tool/web_search.py b/ontology_platform/vendored/ontocast/ontocast/tool/web_search.py new file mode 100644 index 0000000..0594bf6 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/tool/web_search.py @@ -0,0 +1,60 @@ +"""Web-search providers used for optional ontology grounding.""" + +import asyncio +from typing import Any + +from ontocast.tool.atomic import SearchHit + + +class DuckDuckGoSearchProvider: + """DuckDuckGo-backed search provider.""" + + def __init__( + self, + timeout_seconds: int | float = 8, + region: str = "wt-wt", + safesearch: str = "moderate", + ): + self.timeout_seconds = max(1, int(timeout_seconds)) + self.region = region + self.safesearch = safesearch + + async def search(self, query: str, max_results: int) -> list[SearchHit]: + """Search DuckDuckGo and return normalized hits.""" + return await asyncio.to_thread( + self._search_sync, query=query, max_results=max_results + ) + + def _search_sync(self, query: str, max_results: int) -> list[SearchHit]: + # Import lazily so environments without this optional dependency can still + # run with web search disabled. + from duckduckgo_search import DDGS + + hits: list[SearchHit] = [] + with DDGS(timeout=self.timeout_seconds) as ddgs: + results = ddgs.text( + query, + region=self.region, + safesearch=self.safesearch, + max_results=max_results, + ) + for item in results: + normalized = self._normalize_item(item) + if normalized is not None: + hits.append(normalized) + return hits + + def _normalize_item(self, item: Any) -> SearchHit | None: + if not isinstance(item, dict): + return None + + title = str(item.get("title") or item.get("heading") or "").strip() + url = str(item.get("href") or item.get("url") or "").strip() + snippet = str(item.get("body") or item.get("snippet") or "").strip() + if not url or not snippet: + return None + + if not title: + title = url + + return SearchHit(title=title, url=url, snippet=snippet) diff --git a/ontology_platform/vendored/ontocast/ontocast/toolbox.py b/ontology_platform/vendored/ontocast/ontocast/toolbox.py new file mode 100644 index 0000000..ab9ab4e --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/toolbox.py @@ -0,0 +1,428 @@ +import logging + +from langchain_core.output_parsers import PydanticOutputParser +from langchain_core.prompts import PromptTemplate + +from ontocast.config import Config, WebSearchProvider +from ontocast.onto.constants import ONTOLOGY_NULL_IRI +from ontocast.onto.ontology import Ontology, OntologyProperties +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.state import AgentState +from ontocast.tool import ( + AtomicToolBox, + ChunkerTool, + ConverterTool, + FilesystemTripleStoreManager, + FusekiTripleStoreManager, + Neo4jTripleStoreManager, +) +from ontocast.tool.aggregate import EmbeddingBasedAggregator +from ontocast.tool.cache import Cacher +from ontocast.tool.graph_diff import DiffTool +from ontocast.tool.graph_version_manager import GraphVersionManager +from ontocast.tool.llm import LLMTool +from ontocast.tool.ontology_manager import OntologyManager +from ontocast.tool.sparql import SPARQLTool +from ontocast.tool.triple_manager.core import TripleStoreManager +from ontocast.tool.web_search import DuckDuckGoSearchProvider + +logger = logging.getLogger(__name__) + + +async def update_ontology_properties(o: Ontology, llm_tool: LLMTool): + """Update ontology properties using LLM analysis, only if missing. + + This function uses the LLM tool to analyze and update the properties + of a given ontology based on its graph content, but only if any key + property is missing or empty. + """ + # Only update if any key property is missing or empty + if (o.title is None) or (o.ontology_id is None) or (o.description is None): + props = await render_ontology_summary(o, llm_tool) + o.set_properties(**props.model_dump()) + + +async def update_ontology_manager(om: OntologyManager, llm_tool: LLMTool): + """Update properties for all ontologies in the manager. + + This function iterates through all ontologies in the manager and updates + their properties using the LLM tool. + + Args: + om: The ontology manager containing ontologies to update. + llm_tool: The LLM tool instance for analysis. + """ + for o in om.ontologies: + await update_ontology_properties(o, llm_tool) + + +class ToolBox: + """A container class for all tools used in the ontology processing workflow. + + This class initializes and manages various tools needed for document processing, + ontology management, and LLM interactions. + + Args: + config: Configuration object containing all necessary settings. + """ + + def __init__(self, config: Config): + # Store the config for later use + self.config = config + + # Get tool configuration + tool_config = config.get_tool_config() + + # Extract configuration values + working_directory = tool_config.path_config.working_directory + ontology_directory = tool_config.path_config.ontology_directory + + # Create shared cache instance with config + self.shared_cache = Cacher(config=config) + + # LLM configuration - pass the entire LLM config to the tool + self.llm_provider = tool_config.llm_config.provider + self.llm: LLMTool = LLMTool.create( + config=tool_config.llm_config, cache=self.shared_cache + ) + self.search_provider = None + if tool_config.web_search.enabled: + if tool_config.web_search.provider == WebSearchProvider.DUCKDUCKGO: + self.search_provider = DuckDuckGoSearchProvider( + timeout_seconds=tool_config.web_search.timeout_seconds, + region=tool_config.web_search.region, + safesearch=tool_config.web_search.safesearch, + ) + else: + raise ValueError( + f"Unsupported web-search provider: {tool_config.web_search.provider}" + ) + self.atomic_tools = AtomicToolBox( + llm_provider=self, + search_provider=self.search_provider, + web_search_config=tool_config.web_search, + ) + + # Initialize managers based on backend configuration + self.filesystem_manager: FilesystemTripleStoreManager | None = None + self.triple_store_manager: TripleStoreManager | None = None + + # Automatically determine which backends to use based on available configuration + use_fuseki = tool_config.fuseki.uri and tool_config.fuseki.auth + use_neo4j = ( + tool_config.neo4j.uri is not None and tool_config.neo4j.auth is not None + ) + use_filesystem_triple_store = working_directory is not None + use_filesystem_manager = working_directory is not None + + # Validate that we have at least one backend configured + if not any([use_fuseki, use_neo4j, use_filesystem_triple_store]): + raise ValueError( + "No backend configured. Please provide Fuseki/Neo4j credentials or working directory and ontology directory." + ) + + # Create main triple store manager (only one can be active) + # Note: Dataset/database is NOT cleaned on initialization + # Use the clean() method or /flush endpoint to explicitly clean the store + if use_fuseki and tool_config.fuseki.uri and tool_config.fuseki.auth: + self.triple_store_manager = FusekiTripleStoreManager( + uri=tool_config.fuseki.uri, + auth=tool_config.fuseki.auth, + dataset=tool_config.fuseki.dataset, + ontologies_dataset=tool_config.fuseki.ontologies_dataset, + ) + elif use_neo4j and tool_config.neo4j.uri and tool_config.neo4j.auth: + self.triple_store_manager = Neo4jTripleStoreManager( + uri=tool_config.neo4j.uri, auth=tool_config.neo4j.auth + ) + elif use_filesystem_triple_store: + if working_directory is None: + raise ValueError( + "Working directory directory must be provided for filesystem triple store" + ) + self.triple_store_manager = FilesystemTripleStoreManager( + working_directory=working_directory, + ontology_path=ontology_directory, + ) + + # Create filesystem manager (can be combined with other backends) + if use_filesystem_manager: + self.filesystem_manager = FilesystemTripleStoreManager( + working_directory=working_directory, + ontology_path=ontology_directory, + ) + + self.ontology_manager: OntologyManager = OntologyManager() + self.converter: ConverterTool = ConverterTool(cache=self.shared_cache) + self.chunker: ChunkerTool = ChunkerTool( + chunk_config=tool_config.chunk_config, cache=self.shared_cache + ) + self.aggregator: EmbeddingBasedAggregator = EmbeddingBasedAggregator( + embedding_model=tool_config.aggregation.embedding_model, + similarity_threshold=tool_config.aggregation.similarity_threshold, + ) + + # SPARQL, version management, and diff tools + self.sparql_tool: SPARQLTool = SPARQLTool( + triple_store_manager=self.triple_store_manager + ) + self.version_manager: GraphVersionManager = GraphVersionManager() + self.diff_tool: DiffTool = DiffTool() + + async def get_llm_tool(self, budget_tracker): + """Get an LLM tool instance with a specific budget tracker. + + Args: + budget_tracker: The budget tracker instance to use. + + Returns: + LLMTool: LLM tool with the specified budget tracker. + """ + # Create a new LLM tool with the budget tracker + return await LLMTool.acreate( + config=self.config.tool_config.llm_config, + cache=self.shared_cache, + budget_tracker=budget_tracker, + ) + + async def update_dataset(self, dataset: str) -> None: + """Update the dataset for the Fuseki triple store manager. + + This method allows changing the dataset without recreating the entire + ToolBox, which is efficient for API requests that specify different datasets. + + Args: + dataset: The new dataset name to use. + """ + if self.triple_store_manager is not None: + from ontocast.tool.triple_manager.fuseki import FusekiTripleStoreManager + + if isinstance(self.triple_store_manager, FusekiTripleStoreManager): + await self.triple_store_manager.update_dataset(dataset) + else: + logger.warning( + "Cannot update dataset: triple store manager is not Fuseki" + ) + + def get_atomic_tools(self) -> AtomicToolBox: + """Return the minimal toolbox used by atomic render/critic paths.""" + return self.atomic_tools + + def serialize(self, state: AgentState) -> None: + # Add current ontology to ontology manager for version tracking + if state.current_ontology and state.current_ontology.hash: + self.ontology_manager.add_ontology(state.current_ontology) + + if self.filesystem_manager is not None: + self.filesystem_manager.serialize(state.current_ontology) + if state.render_facts: + self.filesystem_manager.serialize( + state.aggregated_facts, + graph_uri=state.graph_uri, + ) + if ( + self.triple_store_manager is not None + and self.triple_store_manager != self.filesystem_manager + ): + # Store ontology in main dataset for reasoning + self.triple_store_manager.serialize(state.current_ontology) + if state.render_facts: + self.triple_store_manager.serialize( + state.aggregated_facts, + graph_uri=state.graph_uri, + ) + + async def initialize(self) -> None: + """Initialize the toolbox with ontologies and their properties. + + This method synchronizes ontologies between filesystem and triple store, + then fetches ontologies from the triple store and updates their properties + using the LLM tool. + """ + + # Synchronize ontologies and add them to ontology manager + synchronized_ontologies = await self._synchronize_ontologies() + for ontology in synchronized_ontologies: + self.ontology_manager.add_ontology(ontology) + await update_ontology_manager(om=self.ontology_manager, llm_tool=self.llm) + + async def _synchronize_ontologies(self) -> list[Ontology]: + """Synchronize ontologies between filesystem and triple store. + + This method checks both filesystem_manager and triple_store_manager for + ontologies and populates triple_store_manager with any ontologies from + filesystem_manager that are not present in triple_store_manager. + + Returns: + list: The final set of ontologies after synchronization + """ + import asyncio + + filesystem_ontologies = [] + if self.filesystem_manager is not None: + # Run sync method in thread pool to avoid blocking + filesystem_ontologies += await asyncio.to_thread( + self.filesystem_manager.fetch_ontologies + ) + logger.info(f"Found {len(filesystem_ontologies)} ontologies in filesystem") + + triple_store_ontologies = [] + if ( + self.triple_store_manager is not None + and self.triple_store_manager != self.filesystem_manager + ): + # Use async version if available, otherwise run sync version in thread pool + afetch_method = getattr( + self.triple_store_manager, "afetch_ontologies", None + ) + if afetch_method is not None: + triple_store_ontologies += await afetch_method() + else: + triple_store_ontologies += await asyncio.to_thread( + self.triple_store_manager.fetch_ontologies + ) + logger.info( + f"Found {len(triple_store_ontologies)} ontologies in triple store" + ) + + # Get IRIs from both sources + triple_store_iris = {o.iri for o in triple_store_ontologies} + + # Find ontologies in filesystem that need to be synced to triple store + for fs_onto in filesystem_ontologies: + if fs_onto.iri not in triple_store_iris: + logger.info( + f"Syncing ontology from filesystem to triple store: {fs_onto.iri} " + f"(version: {fs_onto.version})" + ) + # Store the filesystem ontology to triple store with its version + if self.triple_store_manager is not None: + # Use async version if available, otherwise run sync version in thread pool + aserialize_method = getattr( + self.triple_store_manager, "aserialize", None + ) + if aserialize_method is not None: + await aserialize_method(fs_onto) + else: + await asyncio.to_thread( + self.triple_store_manager.serialize, fs_onto + ) + # Add to triple_store_ontologies list + triple_store_ontologies.append(fs_onto) + + return triple_store_ontologies + + +async def render_ontology_summary(ontology: Ontology, llm_tool) -> OntologyProperties: + """Generate a summary of ontology properties using LLM analysis. + + This function uses the LLM tool to analyze an RDF graph and generate + a structured summary of its properties. Only unset fields are requested. + + Args: + ontology: The ontology to analyze (for checking which fields are set). + llm_tool: The LLM tool instance for analysis. + + Returns: + OntologyProperties: A structured summary containing only the missing properties. + """ + from pydantic import create_model + + # Sample the graph intelligently (first 100 sections) + # This provides context without overwhelming the LLM + sampled_graph = sample_ontology_graph(ontology.graph, max_triples=100) + # Serialize with consistent ordering to ensure determinism + ontology_str = sampled_graph.serialize() + + # Determine which fields are unset and need LLM inference + unset_fields = {} + fields_to_fetch = [] + + # Fields we want to potentially fetch from LLM (excluding internal fields like created_at) + fields_to_check = ["title", "description", "ontology_id", "version", "iri"] + + # For Ontology objects, only fetch fields that are unset + for field in fields_to_check: + value = getattr(ontology, field, None) + if value is None or (field == "iri" and value == ONTOLOGY_NULL_IRI): + fields_to_fetch.append(field) + # Get the field definition from the base model + base_field = OntologyProperties.model_fields[field] + unset_fields[field] = (base_field.annotation, base_field) + + if not unset_fields: + # All fields are already set, return empty props + return OntologyProperties() + + # Create a dynamic model with only unset fields + DynamicProps = create_model("DynamicOntologyProps", **unset_fields) + + # Define the output parser + parser = PydanticOutputParser(pydantic_object=DynamicProps) + + # Create the prompt template with format instructions + field_list_str = "\n- ".join(fields_to_fetch) + format_instructions = parser.get_format_instructions() + + # Build the template - use format_instructions as a separate variable to avoid brace conflicts + template = ( + "Below is a sample of an ontology in Turtle format:\n\n" + "```ttl\n{ontology_str}\n```\n\n" + "Extract ONLY the following properties that are missing:\n" + f"- {field_list_str}\n\n" + "{format_instructions}" + ) + + prompt = PromptTemplate( + template=template, + input_variables=["ontology_str"], + partial_variables={"format_instructions": format_instructions}, + ) + + response = await llm_tool(prompt.format_prompt(ontology_str=ontology_str)) + dynamic_props = parser.parse(response.content) + + # Convert dynamic props to OntologyProperties + result = OntologyProperties() + for field in unset_fields.keys(): + value = getattr(dynamic_props, field, None) + if value is not None: + setattr(result, field, value) + + return result + + +def sample_ontology_graph(graph: RDFGraph, max_triples: int = 100) -> RDFGraph: + """Sample an ontology graph to provide a representative subset. + + This function serializes the graph to Turtle format and takes the first + N blank-line separated sections. This is deterministic and simpler than + complex triple selection logic. + + Args: + graph: The full ontology graph + max_triples: Maximum number of sections to include in the sample + + Returns: + RDFGraph: A sampled version of the ontology with representative triples + """ + # Serialize to turtle + turtle_str = graph.serialize(format="turtle") + + # Split on blank lines (typical turtle format uses \n\n to separate blocks) + sections = turtle_str.split("\n\n") + + # Take first max_triples sections (or fewer if graph is smaller) + num_sections = min(len(sections), max_triples) + sampled_turtle = "\n\n".join(sections[:num_sections]) + + # Parse back into a graph + sampled = RDFGraph() + sampled.parse(data=sampled_turtle, format="turtle") + + # Copy namespace bindings from original graph + for prefix, namespace in graph.namespaces(): + if prefix: + sampled.bind(prefix, namespace) + + return sampled diff --git a/ontology_platform/vendored/ontocast/ontocast/util.py b/ontology_platform/vendored/ontocast/ontocast/util.py new file mode 100644 index 0000000..1c1e0f8 --- /dev/null +++ b/ontology_platform/vendored/ontocast/ontocast/util.py @@ -0,0 +1,46 @@ +import hashlib + +from rdflib import Graph +from rdflib.namespace import NamespaceManager + + +def iri2namespace(iri: str, ontology: bool = False) -> str: + """Convert an IRI to a namespace string. + + Args: + iri: The IRI to convert. + ontology: If True, append '#' for ontology namespace, otherwise '/'. + + Returns: + str: The converted namespace string. + """ + iri = iri.rstrip("#") + return f"{iri}#" if ontology else f"{iri}/" + + +def get_rdflib_namespace_mappings() -> dict: + g = Graph() + ns_manager = NamespaceManager(g) + return {str(uri): prefix for prefix, uri in ns_manager.namespaces()} + + +CONVENTIONAL_MAPPINGS = get_rdflib_namespace_mappings() + + +def render_text_hash(text: str, digits: int | None = 12) -> str: + """Generate a SHA-256 hash for the given text. + + This is the single hashing entry point for the entire codebase. + All modules that need to derive a hash from text should use this function + instead of calling ``hashlib`` directly. + + Args: + text: The text to hash. + digits: Number of hex digits to return (default: 12). + Pass ``None`` to return the full 64-character hex digest. + + Returns: + A hex string hash of the text. + """ + digest = hashlib.sha256(text.encode()).hexdigest() + return digest[:digits] if digits is not None else digest diff --git a/ontology_platform/vendored/ontocast/pyproject.toml b/ontology_platform/vendored/ontocast/pyproject.toml new file mode 100644 index 0000000..c576e60 --- /dev/null +++ b/ontology_platform/vendored/ontocast/pyproject.toml @@ -0,0 +1,115 @@ +[build-system] +build-backend = "hatchling.build" +requires = ["hatchling"] + +[dependency-groups] +dev = [ + "pre-commit>=4.2.0", + "pytest-dotenv>=0.5.2", + "pytest-order>=1.3.0", + "pytest>=8.3.5", + "requests>=2.32.3", + "ruff>=0.11.2", + "ty>=0.0.14" +] +docs = [ + "griffe<2.0.0", + "mkdocs-gen-files>=0.5.0", + "mkdocs-glightbox>=0.4.0", + "mkdocs-jupyter>=0.25.1", + "mkdocs-literate-nav>=0.6.2", + "mkdocs-material>=9.6.14", + "mkdocs>=1.6.1", + "mkdocstrings-python>=2.0.3", + "mkdocstrings[python]>=0.29.1" +] +plot = [ + "pygraphviz>=1.14" +] + +[project] +dependencies = [ + "asyncio>=3.4.3", + "click>=8.1.8", + "duckduckgo-search>=8.1.1", + "hdbscan>=0.8.41", + "httpx>=0.27.0", + "langchain-core>=0.3.60", + "langchain-experimental>=0.3.4", + "langchain-huggingface>=0.2.0", + "langchain-ollama>=0.3.3", + "langchain-openai>=0.3.17", + "langchain>=0.3.25", + "langgraph>=0.2.35", + "neo4j>=5.28.1", + "networkx>=3.0", + "owlready2>=0.47", + "oxrdflib>=0.5.0", + "pydantic>=2.11.9", + "pyld>=2.0.4", + "rapidfuzz>=3.13.0", + "rdflib>=7.1.4", + "rich>=14.0.0", + "robyn>=0.66.2", + "simsimd>=6.2.1", + "suthing>=0.4.1", + "umap-learn>=0.5.11" +] +description = "Agentic ontology and knowledge graph co-generation" +name = "ontocast" +readme = "README.md" +requires-python = ">=3.12,<4.0" +version = "0.3.0" + +[project.optional-dependencies] +doc-processing = [ + "docling>=2.57.0", + "easyocr>=1.7.2", + "sentence-transformers>=5.1.1" +] + +[project.scripts] +cmp-states = "ontocast.cli.cmp_states:main" +ontocast = "ontocast.cli.serve:run" +pdfs-to-markdown = "ontocast.cli.pdfs_to_markdown:main" +plot-graph = "ontocast.cli.plot_graph:main" +test-api = "ontocast.cli.test_api:main" + +[tool.hatch.build.targets.wheel] +packages = ["data", "ontocast"] + +[tool.pyright] +venv = ".venv" +venvPath = "." + +[tool.pytest.ini_options] +addopts = [ + "--disable-warnings", + "--strict-markers", + "--tb=short", + "-v" +] +markers = [ + "integration: marks tests as integration tests", + "slow: marks tests as slow (deselect with '-m \"not slow\"')", + "unit: marks tests as unit tests" +] +python_classes = ["Test*"] +python_files = ["test_*.py"] +python_functions = ["test_*"] +testpaths = ["test"] + +[tool.ruff] +line-length = 88 + +[tool.ruff.format] +line-ending = "auto" + +[tool.ruff.lint] +select = ["E", "F", "I001", "W"] + +[tool.ruff.lint.per-file-ignores] +"ontocast/prompt/*.py" = ["E501"] + +[tool.uv] +default-groups = ["docs"] diff --git a/ontology_platform/vendored/ontocast/pytest.ini b/ontology_platform/vendored/ontocast/pytest.ini new file mode 100644 index 0000000..df7fa02 --- /dev/null +++ b/ontology_platform/vendored/ontocast/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +env_files = .env \ No newline at end of file diff --git a/ontology_platform/vendored/ontocast/shell.nix b/ontology_platform/vendored/ontocast/shell.nix new file mode 100644 index 0000000..77e14d5 --- /dev/null +++ b/ontology_platform/vendored/ontocast/shell.nix @@ -0,0 +1,77 @@ +{ pkgs ? import {} }: + +let + python = pkgs.python312; +in +pkgs.mkShell { + name = "ontocast-dev-shell"; + + buildInputs = with pkgs; [ + python + uv + + git + cmake + stdenv.cc.cc + stdenv.cc.libcxx + + pkg-config + graphviz + cairo + + tesseract + poppler + opencv + libGL + mesa + libglvnd + glib + + # X11 runtime deps + xorg.libX11 + xorg.libXcursor + xorg.libXrandr + xorg.libXinerama + xorg.libXi + xorg.libxcb + + # BLAS/LAPACK for NumPy/hdbscan + openblas + lapack + + # Python build tools + python.pkgs.setuptools + python.pkgs.wheel + python.pkgs.cython + python.pkgs.numpy + python.pkgs.scipy + ]; + + shellHook = '' + # Ensure we're using Nix Python (3.12) + export UV_PYTHON="${python}/bin/python" + export UV_PROJECT_ENVIRONMENT="$PWD/.venv" + + # Make libstdc++ visible to Python C extensions + export LD_LIBRARY_PATH="${pkgs.stdenv.cc.cc.lib}/lib:${pkgs.xorg.libxcb.out}/lib:${pkgs.mesa}/lib:${pkgs.libglvnd}/lib:${pkgs.glib.out}/lib:$LD_LIBRARY_PATH" + export LIBRARY_PATH="${pkgs.stdenv.cc.cc.lib}/lib:$LIBRARY_PATH" + + # Create venv if missing + if [ ! -d "$UV_PROJECT_ENVIRONMENT" ]; then + echo "Creating uv virtualenv $UV_PROJECT_ENVIRONMENT..." + uv venv --python "$UV_PYTHON" "$UV_PROJECT_ENVIRONMENT" + fi + + # "Activate" without sourcing: just put the venv first in PATH + export VIRTUAL_ENV="$UV_PROJECT_ENVIRONMENT" + export PATH="$VIRTUAL_ENV/bin:$PATH" + + # Sync project deps + echo "Syncing dependencies with uv..." + uv sync --group dev --all-extras + + echo "🐍 ontocast_api uv dev environment ready" + echo "Python: $(python --version)" + echo "uv: $(uv --version)" + ''; +} diff --git a/ontology_platform/vendored/ontocast/test/__init__.py b/ontology_platform/vendored/ontocast/test/__init__.py new file mode 100644 index 0000000..a0c305a --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/__init__.py @@ -0,0 +1,12 @@ +"""Test suite for OntoCast. + +This package contains comprehensive tests for the OntoCast framework, +including unit tests, integration tests, and test utilities. + +Test categories: +- Triple store tests (Neo4j, Fuseki) +- Ontology and state management tests +- Fact extraction and rendering tests +- Validation and utility tests +- Integration tests for the complete workflow +""" diff --git a/참고/playwright-main/tests/assets/network-tab/script.js b/ontology_platform/vendored/ontocast/test/aggregation/__init__.py similarity index 100% rename from 참고/playwright-main/tests/assets/network-tab/script.js rename to ontology_platform/vendored/ontocast/test/aggregation/__init__.py diff --git a/ontology_platform/vendored/ontocast/test/aggregation/test_aggregate_pipeline.py b/ontology_platform/vendored/ontocast/test/aggregation/test_aggregate_pipeline.py new file mode 100644 index 0000000..219b7f0 --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/aggregation/test_aggregate_pipeline.py @@ -0,0 +1,934 @@ +from rdflib import OWL, RDF, RDFS, Literal, URIRef + +from ontocast.onto.constants import DEFAULT_IRI, PROV, RDF_REIFIES, SCHEMA +from ontocast.onto.content_unit import ContentUnit, OutputType +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.tool.agg.aggregate import EmbeddingBasedAggregator +from ontocast.util import render_text_hash + + +def make_fact_unit( + text: str, + index: int, + doc_iri: URIRef | str, + ttl: str, +) -> ContentUnit: + graph = RDFGraph() + graph.parse(data=ttl, format="turtle") + return ContentUnit( + text=text, + index=index, + doc_iri=URIRef(str(doc_iri)), + graph=graph, + type=OutputType.FACTS, + ) + + +def make_ontology_unit( + text: str, + index: int, + doc_iri: URIRef | str, + ttl: str, +) -> ContentUnit: + graph = RDFGraph() + graph.parse(data=ttl, format="turtle") + return ContentUnit( + text=text, + index=index, + doc_iri=URIRef(str(doc_iri)), + graph=graph, + type=OutputType.ONTOLOGIES, + ) + + +def test_aggregate_graphs_returns_empty_graph_for_no_units() -> None: + aggregator = EmbeddingBasedAggregator() + result = aggregator.aggregate_graphs([]) + assert len(result) == 0 + + +def test_fact_entities_use_doc_iri_namespace() -> None: + doc_iri = "https://my-org.io/reports/annual2025" + ttl = f""" + @prefix facts: <{DEFAULT_IRI}/> . + @prefix rdf: . + @prefix rdfs: . + facts:Revenue rdf:type facts:FinancialMetric . + facts:Revenue rdfs:label "Revenue" . + facts:Revenue facts:amount "42000000" . + """ + unit = make_fact_unit("Revenue was $42M.", 0, doc_iri, ttl) + + result = EmbeddingBasedAggregator().aggregate_graphs([unit]) + assert len(result) > 0 + + fact_subjects = { + str(subject) + for subject, predicate, _ in result + if isinstance(subject, URIRef) + and predicate != RDF.type + and not str(subject).startswith("http://www.w3.org") + and not str(subject).startswith("https://schema.org") + and "/stmt/" not in str(subject) + and "/chunk/" not in str(subject) + } + assert fact_subjects + assert any(subject.startswith(doc_iri) for subject in fact_subjects) + + +def test_aggregate_graphs_merges_overlapping_facts(monkeypatch) -> None: + doc_iri = "https://example.org/docs/report1" + ttl_chunk_0 = f""" + @prefix facts: <{DEFAULT_IRI}/> . + @prefix rdf: . + @prefix rdfs: . + facts:UnitedStates rdf:type facts:Country . + facts:UnitedStates rdfs:label "United States" . + facts:UnitedStates facts:capitalCity "Washington, D.C." . + facts:UnitedStates facts:currency "USD" . + """ + ttl_chunk_1 = f""" + @prefix facts: <{DEFAULT_IRI}/> . + @prefix rdf: . + @prefix rdfs: . + facts:united_states rdf:type facts:Country . + facts:united_states rdfs:label "United States" . + facts:united_states facts:population "331000000" . + """ + ttl_chunk_2 = f""" + @prefix facts: <{DEFAULT_IRI}/> . + @prefix rdf: . + @prefix rdfs: . + facts:UnitedStatesBank rdf:type facts:Company . + facts:UnitedStatesBank rdfs:label "United States Bank" . + facts:UnitedStatesBank facts:headquarters "Portland" . + """ + + text_0 = "The United States has capital Washington, D.C. and uses USD." + text_1 = "In another section, united_states is described with population data." + text_2 = "United States Bank is headquartered in Portland." + units = [ + make_fact_unit( + text_0, + 0, + doc_iri, + ttl_chunk_0, + ), + make_fact_unit( + text_1, + 1, + doc_iri, + ttl_chunk_1, + ), + make_fact_unit( + text_2, + 2, + doc_iri, + ttl_chunk_2, + ), + ] + aggregator = EmbeddingBasedAggregator() + + def cluster_by_normal_form(representations): + clusters_by_key: dict[str, list[URIRef]] = {} + for entity, representation in representations.items(): + clusters_by_key.setdefault(representation.normal_form, []).append(entity) + return list(clusters_by_key.values()), {} + + monkeypatch.setattr( + aggregator.clusterer, "cluster_entities", cluster_by_normal_form + ) + result = aggregator.aggregate_graphs(units) + result.bind("unused", "https://unused.example/") + turtle = result.serialize(format="turtle") + + assert "Washington, D.C." in turtle + assert "USD" in turtle + assert "331000000" in turtle + assert "Portland" in turtle + assert "@prefix doc:" in turtle + assert "@prefix unused:" not in turtle + assert len(list(result.triples((None, RDFS.label, None)))) >= 2 + + us_subjects = { + subject + for subject in result.subjects(RDFS.label, Literal("United States")) + if isinstance(subject, URIRef) + } + assert len(us_subjects) == 1 + us_entity = next(iter(us_subjects)) + + bank_subjects = { + subject + for subject in result.subjects(RDFS.label, Literal("United States Bank")) + if isinstance(subject, URIRef) + } + assert len(bank_subjects) == 1 + bank_entity = next(iter(bank_subjects)) + + assert us_entity != bank_entity + assert str(us_entity).startswith(doc_iri) + assert str(bank_entity).startswith(doc_iri) + + assert (us_entity, None, Literal("USD")) in result + assert (us_entity, None, Literal("331000000")) in result + assert (bank_entity, None, Literal("Portland")) in result + + original_camel = URIRef(f"{DEFAULT_IRI}/UnitedStates") + original_snake = URIRef(f"{DEFAULT_IRI}/united_states") + assert (us_entity, OWL.sameAs, original_camel) not in result + assert (us_entity, OWL.sameAs, original_snake) not in result + + statement_nodes = list(result.subjects(RDF_REIFIES, None)) + assert statement_nodes + assert all( + len(set(result.objects(stmt, PROV.wasDerivedFrom))) >= 1 + for stmt in statement_nodes + ) + + chunk_ids = {str(value) for value in result.objects(None, SCHEMA.identifier)} + expected_ids = { + render_text_hash(text_0), + render_text_hash(text_1), + render_text_hash(text_2), + } + assert expected_ids <= chunk_ids + + +def test_aggregate_graphs_preserves_ontology_uris_and_provenance(monkeypatch) -> None: + doc_iri = "https://example.org/docs/report1" + ttl_chunk_0 = """ + @prefix ex: . + @prefix rdfs: . + @prefix rdf: . + ex:Person rdf:type rdfs:Class . + ex:Person rdfs:label "Person" . + """ + ttl_chunk_1 = """ + @prefix ex: . + @prefix rdfs: . + @prefix rdf: . + ex:Persno rdf:type rdfs:Class . + ex:Persno rdfs:label "Person" . + """ + units = [ + make_ontology_unit("Defines Person class.", 0, doc_iri, ttl_chunk_0), + make_ontology_unit("Repeats class with typo URI.", 1, doc_iri, ttl_chunk_1), + ] + + aggregator = EmbeddingBasedAggregator() + + def force_typo_and_canonical_in_one_cluster(representations): + canonical = URIRef("http://example.org/onto#Person") + typo = URIRef("http://example.org/onto#Persno") + entities = set(representations.keys()) + if canonical in entities and typo in entities: + return [[canonical, typo]], {} + return [list(entities)], {} + + monkeypatch.setattr( + aggregator.clusterer, + "cluster_entities", + force_typo_and_canonical_in_one_cluster, + ) + + result = aggregator.aggregate_graphs(units) + + canonical = URIRef("http://example.org/onto#Person") + typo = URIRef("http://example.org/onto#Persno") + + assert (canonical, RDFS.label, Literal("Person")) in result + assert (typo, RDFS.label, Literal("Person")) in result + assert (canonical, OWL.sameAs, typo) in result or ( + typo, + OWL.sameAs, + canonical, + ) in result + assert str(canonical).startswith("http://example.org/onto#") + + statement_nodes = list(result.subjects(RDF_REIFIES, None)) + assert statement_nodes + assert all( + len(set(result.objects(stmt, PROV.wasDerivedFrom))) >= 1 + for stmt in statement_nodes + ) + + +def test_facts_doc_entity_does_not_replace_ontology_entity(monkeypatch) -> None: + doc_iri = "https://example.org/docs/case-42" + ontology_court = URIRef("https://growgraph.dev/fcaont#CourAppelRouen") + doc_court = URIRef(f"{doc_iri}/CourAppelRouen") + heard_at = URIRef("https://growgraph.dev/fcaont#heardAt") + court_type = URIRef("https://growgraph.dev/fcaont#Court") + + ttl = f""" + @prefix doc: <{doc_iri}/> . + @prefix fcaont: . + @prefix rdf: . + doc:Case1 fcaont:heardAt doc:CourAppelRouen . + doc:Case2 fcaont:heardAt fcaont:CourAppelRouen . + doc:CourAppelRouen rdf:type fcaont:Court . + fcaont:CourAppelRouen rdf:type fcaont:Court . + """ + unit = make_fact_unit("Rouen court references.", 0, doc_iri, ttl) + aggregator = EmbeddingBasedAggregator() + ontology_graph = RDFGraph() + ontology_graph.add((ontology_court, RDF.type, court_type)) + + def force_doc_and_ontology_court_together(representations): + entities = set(representations.keys()) + if doc_court in entities and ontology_court in entities: + remaining = [e for e in entities if e not in {doc_court, ontology_court}] + return [[doc_court, ontology_court], *[[e] for e in remaining]], {} + return [list(entities)], {} + + monkeypatch.setattr( + aggregator.clusterer, + "cluster_entities", + force_doc_and_ontology_court_together, + ) + + result = aggregator.aggregate_graphs([unit], ontology_graph=ontology_graph) + + assert (ontology_court, RDF.type, court_type) in result + assert (doc_court, RDF.type, court_type) in result + assert (ontology_court, OWL.sameAs, doc_court) not in result + assert (doc_court, OWL.sameAs, ontology_court) not in result + + heard_at_targets = set(result.objects(None, heard_at)) + assert ontology_court in heard_at_targets + assert doc_court in heard_at_targets + + +def test_ontology_entities_in_same_cluster_keep_original_iris(monkeypatch) -> None: + doc_iri = "https://example.org/docs/case-43" + court_fr = URIRef("https://growgraph.dev/fcaont#CourAppelRouen") + court_en = URIRef("https://growgraph.dev/fcaont#AppealCourt_Rouen") + heard_at = URIRef("https://growgraph.dev/fcaont#heardAt") + same_as = OWL.sameAs + rdfs_label = RDFS.label + + ttl = f""" + @prefix doc: <{doc_iri}/> . + @prefix fcaont: . + @prefix rdfs: . + doc:Case1 fcaont:heardAt fcaont:CourAppelRouen . + doc:Case2 fcaont:heardAt fcaont:AppealCourt_Rouen . + fcaont:CourAppelRouen rdfs:label "Cour d'appel de Rouen" . + fcaont:AppealCourt_Rouen rdfs:label "Rouen Court of Appeal" . + """ + unit = make_fact_unit("Rouen court variants.", 0, doc_iri, ttl) + aggregator = EmbeddingBasedAggregator() + ontology_graph = RDFGraph() + ontology_graph.add((court_fr, rdfs_label, Literal("Cour d'appel de Rouen"))) + ontology_graph.add((court_en, rdfs_label, Literal("Rouen Court of Appeal"))) + + def force_ontology_variants_together(representations): + entities = set(representations.keys()) + if court_fr in entities and court_en in entities: + remaining = [ + entity for entity in entities if entity not in {court_fr, court_en} + ] + return [[court_fr, court_en], *[[entity] for entity in remaining]], {} + return [list(entities)], {} + + monkeypatch.setattr( + aggregator.clusterer, + "cluster_entities", + force_ontology_variants_together, + ) + + result = aggregator.aggregate_graphs([unit], ontology_graph=ontology_graph) + + assert (court_fr, rdfs_label, Literal("Cour d'appel de Rouen")) in result + assert (court_en, rdfs_label, Literal("Rouen Court of Appeal")) in result + assert (court_fr, heard_at, None) not in result + assert (court_en, heard_at, None) not in result + + heard_at_targets = set(result.objects(None, heard_at)) + assert court_fr in heard_at_targets + assert court_en in heard_at_targets + + assert (court_fr, same_as, court_en) in result or ( + court_en, + same_as, + court_fr, + ) in result + + +def test_tentative_ontology_like_alias_maps_to_known_ontology(monkeypatch) -> None: + doc_iri = "https://example.org/docs/case-44" + known_court = URIRef("https://growgraph.dev/fcaont#AppealCourtRouen") + invented_court = URIRef("https://growgraph.dev/fcaont#AppealCourt_Rouen") + heard_at = URIRef("https://growgraph.dev/fcaont#heardAt") + court_type = URIRef("https://growgraph.dev/fcaont#Court") + + ttl = f""" + @prefix doc: <{doc_iri}/> . + @prefix fcaont: . + @prefix rdf: . + doc:Case1 fcaont:heardAt fcaont:AppealCourt_Rouen . + fcaont:AppealCourt_Rouen rdf:type fcaont:Court . + """ + unit = make_fact_unit("Invented ontology-like alias.", 0, doc_iri, ttl) + aggregator = EmbeddingBasedAggregator() + ontology_graph = RDFGraph() + ontology_graph.add((known_court, RDF.type, court_type)) + ontology_graph.add((known_court, RDFS.label, Literal("Rouen Court of Appeal"))) + + def force_known_and_invented_together(representations): + entities = set(representations.keys()) + if known_court in entities and invented_court in entities: + remaining = [ + entity + for entity in entities + if entity not in {known_court, invented_court} + ] + return [ + [known_court, invented_court], + *[[entity] for entity in remaining], + ], {} + return [list(entities)], {} + + monkeypatch.setattr( + aggregator.clusterer, + "cluster_entities", + force_known_and_invented_together, + ) + + result = aggregator.aggregate_graphs([unit], ontology_graph=ontology_graph) + + heard_at_targets = set(result.objects(None, heard_at)) + assert known_court in heard_at_targets + assert invented_court not in heard_at_targets + assert (known_court, OWL.sameAs, invented_court) not in result + + +def test_tentative_only_ontology_like_entities_are_preserved(monkeypatch) -> None: + doc_iri = "https://example.org/docs/case-45" + invented_court_1 = URIRef("https://growgraph.dev/fcaont#AppealCourt_Rouen") + invented_court_2 = URIRef("https://growgraph.dev/fcaont#CourtOfAppealRouen") + heard_at = URIRef("https://growgraph.dev/fcaont#heardAt") + + ttl = f""" + @prefix doc: <{doc_iri}/> . + @prefix fcaont: . + doc:Case1 fcaont:heardAt fcaont:AppealCourt_Rouen . + doc:Case2 fcaont:heardAt fcaont:CourtOfAppealRouen . + """ + unit = make_fact_unit("Tentative ontology-like terms only.", 0, doc_iri, ttl) + aggregator = EmbeddingBasedAggregator() + + def force_tentatives_together(representations): + entities = set(representations.keys()) + if invented_court_1 in entities and invented_court_2 in entities: + remaining = [ + entity + for entity in entities + if entity not in {invented_court_1, invented_court_2} + ] + return [ + [invented_court_1, invented_court_2], + *[[entity] for entity in remaining], + ], {} + return [list(entities)], {} + + monkeypatch.setattr( + aggregator.clusterer, + "cluster_entities", + force_tentatives_together, + ) + + result = aggregator.aggregate_graphs([unit]) + + heard_at_targets = set(result.objects(None, heard_at)) + assert invented_court_1 in heard_at_targets + assert invented_court_2 in heard_at_targets + + +def test_unused_ontology_entities_do_not_create_spurious_sameas() -> None: + doc_iri = "https://example.org/docs/case-46" + court_in_facts = URIRef("https://growgraph.dev/fcaont#CourAppelRouen") + heard_at = URIRef("https://growgraph.dev/fcaont#heardAt") + court_type = URIRef("https://growgraph.dev/fcaont#AppealCourt") + unused_a = URIRef("https://growgraph.dev/fcaont#CourAppelParis") + unused_b = URIRef("https://growgraph.dev/fcaont#CourAppelLyon") + + ttl = f""" + @prefix doc: <{doc_iri}/> . + @prefix fcaont: . + @prefix rdf: . + doc:Case1 fcaont:heardAt fcaont:CourAppelRouen . + fcaont:CourAppelRouen rdf:type fcaont:AppealCourt . + """ + unit = make_fact_unit("Case heard at Rouen court of appeal.", 0, doc_iri, ttl) + ontology_graph = RDFGraph() + ontology_graph.add((court_in_facts, RDF.type, court_type)) + ontology_graph.add((unused_a, RDF.type, court_type)) + ontology_graph.add((unused_b, RDF.type, court_type)) + + result = EmbeddingBasedAggregator().aggregate_graphs( + [unit], ontology_graph=ontology_graph + ) + + assert (unused_a, OWL.sameAs, unused_b) not in result + assert (unused_b, OWL.sameAs, unused_a) not in result + assert court_in_facts in set(result.objects(None, heard_at)) + + +def test_tentative_with_incompatible_type_does_not_merge_to_known_ontology( + monkeypatch, +) -> None: + doc_iri = "https://example.org/docs/case-47" + known_conviction = URIRef("https://growgraph.dev/fcaont#Conviction") + tentative_person = URIRef("https://growgraph.dev/fcaont#Conviction1") + associated_with = URIRef("https://growgraph.dev/fcaont#isAssociatedWith") + conviction_type = URIRef("https://growgraph.dev/fcaont#Conviction") + + ttl = f""" + @prefix doc: <{doc_iri}/> . + @prefix fcaont: . + @prefix rdf: . + @prefix schema: . + doc:Judgment1 fcaont:isAssociatedWith fcaont:Conviction1 . + fcaont:Conviction1 rdf:type schema:Person . + """ + unit = make_fact_unit("Person associated with judgment.", 0, doc_iri, ttl) + aggregator = EmbeddingBasedAggregator() + ontology_graph = RDFGraph() + ontology_graph.add((known_conviction, RDF.type, conviction_type)) + ontology_graph.add((known_conviction, RDFS.label, Literal("Conviction"))) + + def force_known_and_tentative_together(representations): + entities = set(representations.keys()) + if known_conviction in entities and tentative_person in entities: + remaining = [ + entity + for entity in entities + if entity not in {known_conviction, tentative_person} + ] + return [ + [known_conviction, tentative_person], + *[[entity] for entity in remaining], + ], {} + return [list(entities)], {} + + monkeypatch.setattr( + aggregator.clusterer, + "cluster_entities", + force_known_and_tentative_together, + ) + + result = aggregator.aggregate_graphs([unit], ontology_graph=ontology_graph) + + assert tentative_person in set(result.objects(None, associated_with)) + assert known_conviction not in set(result.objects(None, associated_with)) + assert (known_conviction, OWL.sameAs, tentative_person) not in result + + +def test_tentative_alias_merged_without_sameas_leak(monkeypatch) -> None: + doc_iri = "https://example.org/docs/case-47b" + known_conviction = URIRef("https://growgraph.dev/fcaont#Conviction") + tentative_alias = URIRef("https://growgraph.dev/fcaont#Conviction1") + associated_with = URIRef("https://growgraph.dev/fcaont#isAssociatedWith") + class_type = URIRef("http://www.w3.org/2000/01/rdf-schema#Class") + + ttl = f""" + @prefix doc: <{doc_iri}/> . + @prefix fcaont: . + @prefix rdf: . + doc:Judgment1 fcaont:isAssociatedWith fcaont:Conviction1 . + fcaont:Conviction1 rdf:type fcaont:Conviction . + """ + unit = make_fact_unit("Ontology-like alias mention.", 0, doc_iri, ttl) + aggregator = EmbeddingBasedAggregator() + ontology_graph = RDFGraph() + ontology_graph.add((known_conviction, RDF.type, class_type)) + + def force_known_and_tentative_together(representations): + entities = set(representations.keys()) + if known_conviction in entities and tentative_alias in entities: + remaining = [ + entity + for entity in entities + if entity not in {known_conviction, tentative_alias} + ] + return [ + [known_conviction, tentative_alias], + *[[entity] for entity in remaining], + ], {} + return [list(entities)], {} + + monkeypatch.setattr( + aggregator.clusterer, + "cluster_entities", + force_known_and_tentative_together, + ) + + result = aggregator.aggregate_graphs([unit], ontology_graph=ontology_graph) + + assert known_conviction in set(result.objects(None, associated_with)) + assert tentative_alias not in set(result.objects(None, associated_with)) + assert (known_conviction, OWL.sameAs, tentative_alias) not in result + + +def test_non_alias_ontology_terms_do_not_emit_sameas(monkeypatch) -> None: + doc_iri = "https://example.org/docs/case-48" + appeal = URIRef("https://growgraph.dev/fcaont#Appeal") + appeal_decision = URIRef("https://growgraph.dev/fcaont#AppealDecision") + type_class = URIRef("http://www.w3.org/2000/01/rdf-schema#Class") + + ttl = f""" + @prefix doc: <{doc_iri}/> . + @prefix fcaont: . + @prefix rdf: . + @prefix rdfs: . + fcaont:Appeal rdf:type rdfs:Class . + fcaont:AppealDecision rdf:type rdfs:Class . + """ + unit = make_fact_unit("Ontology class references.", 0, doc_iri, ttl) + aggregator = EmbeddingBasedAggregator() + ontology_graph = RDFGraph() + ontology_graph.add((appeal, RDF.type, type_class)) + ontology_graph.add((appeal_decision, RDF.type, type_class)) + + def force_together(representations): + entities = set(representations.keys()) + if appeal in entities and appeal_decision in entities: + remaining = [ + entity for entity in entities if entity not in {appeal, appeal_decision} + ] + return [[appeal, appeal_decision], *[[entity] for entity in remaining]], {} + return [list(entities)], {} + + monkeypatch.setattr(aggregator.clusterer, "cluster_entities", force_together) + result = aggregator.aggregate_graphs([unit], ontology_graph=ontology_graph) + + assert (appeal, OWL.sameAs, appeal_decision) not in result + assert (appeal_decision, OWL.sameAs, appeal) not in result + + +def test_entity_in_namespace_accepts_exact_prefix_namespace() -> None: + entity = URIRef("https://growgraph.dev/factsConviction1") + assert EmbeddingBasedAggregator._entity_in_namespace( + entity, "https://growgraph.dev/facts" + ) + + +def test_fact_entity_forced_with_known_ontology_uses_identity_guard( + monkeypatch, +) -> None: + doc_iri = "https://example.org/docs/case-49" + known_conviction = URIRef("https://growgraph.dev/fcaont#Conviction") + fact_conviction = URIRef("https://growgraph.dev/factsConviction1") + associated_with = URIRef("https://growgraph.dev/fcaont#isAssociatedWith") + class_type = URIRef("http://www.w3.org/2000/01/rdf-schema#Class") + + ttl = f""" + @prefix doc: <{doc_iri}/> . + @prefix cd: . + @prefix fcaont: . + @prefix rdf: . + @prefix schema: . + doc:Judgment1 fcaont:isAssociatedWith cd:Conviction1 . + cd:Conviction1 rdf:type schema:Person . + """ + unit = make_fact_unit("Forced mixed cluster.", 0, doc_iri, ttl) + aggregator = EmbeddingBasedAggregator() + ontology_graph = RDFGraph() + ontology_graph.add((known_conviction, RDF.type, class_type)) + + def force_known_and_fact_together(representations): + entities = set(representations.keys()) + if known_conviction in entities and fact_conviction in entities: + remaining = [ + entity + for entity in entities + if entity not in {known_conviction, fact_conviction} + ] + return [ + [known_conviction, fact_conviction], + *[[entity] for entity in remaining], + ], {} + return [list(entities)], {} + + monkeypatch.setattr( + aggregator.clusterer, + "cluster_entities", + force_known_and_fact_together, + ) + + result = aggregator.aggregate_graphs([unit], ontology_graph=ontology_graph) + + associated_targets = { + obj for obj in result.objects(None, associated_with) if isinstance(obj, URIRef) + } + assert associated_targets + assert all(str(obj).startswith(doc_iri) for obj in associated_targets) + assert known_conviction not in associated_targets + + uri_nodes = { + term for s, _, o in result for term in (s, o) if isinstance(term, URIRef) + } + assert all(not str(node).startswith(DEFAULT_IRI) for node in uri_nodes) + + +def test_fact_predicate_is_collected_and_rewritten_to_doc_namespace() -> None: + doc_iri = "https://example.org/docs/predicate-case" + predicate = URIRef("https://growgraph.dev/factsHasCase") + ttl = f""" + @prefix doc: <{doc_iri}/> . + @prefix facts: . + @prefix rdf: . + doc:CaseA facts:HasCase doc:CaseB . + doc:CaseA rdf:type doc:Case . + """ + unit = make_fact_unit("Predicate-only fact URI.", 0, doc_iri, ttl) + + result = EmbeddingBasedAggregator().aggregate_graphs([unit]) + + rewritten_predicates = { + p for _, p, _ in result if isinstance(p, URIRef) and str(p).startswith(doc_iri) + } + assert rewritten_predicates + assert any("HasCase" in str(p) for p in rewritten_predicates) + assert predicate not in set(result.predicates(None, None)) + + +def test_cross_chunk_entity_context_is_merged_for_representation(monkeypatch) -> None: + doc_iri = "https://example.org/docs/context-merge" + shared = URIRef("https://growgraph.dev/factsSharedEntity") + rel_a = URIRef("https://growgraph.dev/factsHasAlpha") + rel_b = URIRef("https://growgraph.dev/factsHasBeta") + ttl_chunk_0 = """ + @prefix facts: . + facts:SharedEntity facts:HasAlpha "A" . + """ + ttl_chunk_1 = """ + @prefix facts: . + facts:SharedEntity facts:HasBeta "B" . + """ + units = [ + make_fact_unit("First chunk", 0, doc_iri, ttl_chunk_0), + make_fact_unit("Second chunk", 1, doc_iri, ttl_chunk_1), + ] + aggregator = EmbeddingBasedAggregator() + original_create_representation = aggregator.normalizer.create_representation + seen_shared_context: dict[str, set[URIRef]] = {"properties": set()} + + def capture_representation(entity, graph): + representation = original_create_representation(entity, graph) + if entity == shared: + seen_shared_context["properties"] = set(representation.properties) + return representation + + monkeypatch.setattr( + aggregator.normalizer, + "create_representation", + capture_representation, + ) + + aggregator.aggregate_graphs(units) + + assert rel_a in seen_shared_context["properties"] + assert rel_b in seen_shared_context["properties"] + + +def test_doc_namespace_forcing_avoids_uri_collisions(monkeypatch) -> None: + doc_iri = "https://example.org/docs/collision-safe" + ttl = """ + @prefix facts: . + facts:EntityA facts:RelatedTo "left" . + facts:EntityB facts:RelatedTo "right" . + """ + unit = make_fact_unit("Collision case", 0, doc_iri, ttl) + aggregator = EmbeddingBasedAggregator() + + def singleton_clusters(representations): + return [[entity] for entity in representations], {} + + original_create_representations = aggregator.normalizer.create_representations_batch + + def force_same_normal_form(entities, entity_graphs): + representations = original_create_representations(entities, entity_graphs) + for entity in entities: + if str(entity).endswith("EntityA") or str(entity).endswith("EntityB"): + rep = representations[entity] + rep.normal_form = "collision" + rep.representation = "collision" + return representations + + monkeypatch.setattr(aggregator.clusterer, "cluster_entities", singleton_clusters) + monkeypatch.setattr( + aggregator.normalizer, + "create_representations_batch", + force_same_normal_form, + ) + + result = aggregator.aggregate_graphs([unit]) + + subject_targets = { + subject + for subject, _, obj in result + if isinstance(subject, URIRef) + and str(subject).startswith(doc_iri) + and isinstance(obj, Literal) + and str(obj) in {"left", "right"} + } + assert len(subject_targets) == 2 + assert len({str(target).split("/")[-1] for target in subject_targets}) == 2 + assert all(str(target).startswith(doc_iri) for target in subject_targets) + + +def test_select_ontology_anchor_candidates_preserves_trigger_doc_iri() -> None: + aggregator = EmbeddingBasedAggregator() + doc_a = URIRef("https://example.org/docs/a") + doc_b = URIRef("https://example.org/docs/b") + known_court = URIRef("https://growgraph.dev/fcaont#AppealCourtRouen") + tentative_a = URIRef("https://growgraph.dev/fcaont#AppealCourt_Rouen") + tentative_b = URIRef("https://growgraph.dev/fcaont#AppealCourtRouenAlias") + + ontology_graph = RDFGraph() + ontology_graph.add((known_court, RDFS.label, Literal("Appeal Court Rouen"))) + + tentative_graph = RDFGraph() + tentative_graph.add((tentative_a, RDFS.label, Literal("Appeal Court Rouen"))) + tentative_graph.add((tentative_b, RDFS.label, Literal("Appeal Court Rouen"))) + tentative_representations = aggregator.normalizer.create_representations_batch( + [tentative_a, tentative_b], + { + tentative_a: tentative_graph, + tentative_b: tentative_graph, + }, + ) + + selected = aggregator._select_ontology_anchor_candidates( + tentative_entities=[tentative_a, tentative_b], + tentative_representations=tentative_representations, + tentative_doc_iris={ + tentative_a: doc_a, + tentative_b: doc_b, + }, + ontology_graph=ontology_graph, + known_ontology_entities={known_court}, + ) + + assert selected[known_court] == doc_a + + +def test_jaccard_handles_empty_and_partial_overlap() -> None: + assert EmbeddingBasedAggregator._jaccard(set(), set()) == 1.0 + assert EmbeddingBasedAggregator._jaccard(set(), {"a"}) == 0.0 + assert EmbeddingBasedAggregator._jaccard({"a", "b"}, {"b", "c"}) == 1 / 3 + + +def test_fact_to_fact_candidate_rejected_when_symbolically_incompatible( + monkeypatch, +) -> None: + doc_iri = "https://example.org/docs/case-merge-gate-1" + criminal_court = URIRef(f"{DEFAULT_IRI}/CriminalCourt") + civil_court = URIRef(f"{DEFAULT_IRI}/CivilCourt") + + ttl = f""" + @prefix doc: <{doc_iri}/> . + @prefix facts: <{DEFAULT_IRI}/> . + @prefix rdf: . + @prefix rdfs: . + doc:Case1 facts:heardAt facts:CriminalCourt . + doc:Case2 facts:heardAt facts:CivilCourt . + facts:CriminalCourt rdf:type . + facts:CivilCourt rdf:type . + facts:CriminalCourt rdfs:label "Criminal Court" . + facts:CivilCourt rdfs:label "Civil Court" . + """ + unit = make_fact_unit("Two related courts", 0, doc_iri, ttl) + aggregator = EmbeddingBasedAggregator() + + def force_candidate_cluster(representations): + entities = set(representations.keys()) + if criminal_court in entities and civil_court in entities: + remaining = [ + entity + for entity in entities + if entity not in {criminal_court, civil_court} + ] + return [ + [criminal_court, civil_court], + *[[entity] for entity in remaining], + ], {} + return [list(entities)], {} + + monkeypatch.setattr( + aggregator.clusterer, + "cluster_entities", + force_candidate_cluster, + ) + + result = aggregator.aggregate_graphs([unit]) + heard_at_targets = { + subject + for subject in result.subjects(RDFS.label, Literal("Criminal Court")) + if isinstance(subject, URIRef) + } | { + subject + for subject in result.subjects(RDFS.label, Literal("Civil Court")) + if isinstance(subject, URIRef) + } + + assert len(heard_at_targets) == 2 + assert all(str(target).startswith(doc_iri) for target in heard_at_targets) + + +def test_fact_to_fact_candidate_merges_when_symbolically_compatible( + monkeypatch, +) -> None: + doc_iri = "https://example.org/docs/case-merge-gate-2" + united_states = URIRef(f"{DEFAULT_IRI}/UnitedStates") + united_states_alias = URIRef(f"{DEFAULT_IRI}/united_states") + ttl = f""" + @prefix doc: <{doc_iri}/> . + @prefix facts: <{DEFAULT_IRI}/> . + @prefix rdf: . + @prefix rdfs: . + facts:UnitedStates rdf:type . + facts:united_states rdf:type . + facts:UnitedStates rdfs:label "United States" . + facts:united_states rdfs:label "United States" . + facts:UnitedStates facts:population "331000000" . + facts:united_states facts:population "332000000" . + """ + unit = make_fact_unit("US aliases", 0, doc_iri, ttl) + aggregator = EmbeddingBasedAggregator() + + def force_candidate_cluster(representations): + entities = set(representations.keys()) + if united_states in entities and united_states_alias in entities: + remaining = [ + entity + for entity in entities + if entity not in {united_states, united_states_alias} + ] + return [ + [united_states, united_states_alias], + *[[entity] for entity in remaining], + ], {} + return [list(entities)], {} + + monkeypatch.setattr( + aggregator.clusterer, + "cluster_entities", + force_candidate_cluster, + ) + + result = aggregator.aggregate_graphs([unit]) + population_subjects = { + subject + for subject, _, obj in result + if isinstance(subject, URIRef) + and isinstance(obj, Literal) + and str(obj) in {"331000000", "332000000"} + } + + assert len(population_subjects) == 1 + target = next(iter(population_subjects)) + assert str(target).startswith(doc_iri) diff --git a/ontology_platform/vendored/ontocast/test/aggregation/test_clustering.py b/ontology_platform/vendored/ontocast/test/aggregation/test_clustering.py new file mode 100644 index 0000000..e60de8c --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/aggregation/test_clustering.py @@ -0,0 +1,89 @@ +from typing import cast +from unittest.mock import Mock + +from rdflib import URIRef + +from ontocast.onto.constants import DEFAULT_IRI +from ontocast.tool.agg.clustering import ClusterRepresentativeSelector +from ontocast.tool.agg.normalizer import EntityRepresentation + + +def test_simplicity_score_prefers_simple_uris( + cluster_representative_selector: ClusterRepresentativeSelector, +) -> None: + simple = URIRef("http://ex.org/Thing") + complex_uri = URIRef("http://example.org/deeply/nested/path/ComplexEntity_123") + + simple_score = cluster_representative_selector.compute_simplicity_score(simple) + complex_score = cluster_representative_selector.compute_simplicity_score( + complex_uri + ) + + assert simple_score < complex_score + + +def test_select_representative_prefers_ontology_entity( + cluster_representative_selector: ClusterRepresentativeSelector, +) -> None: + ont_entity = URIRef("http://ontology.org/Thing") + chunk_entity = URIRef(f"{DEFAULT_IRI}/entity_long_name") + + ont_rep = Mock(is_ontology_entity=True) + chunk_rep = Mock(is_ontology_entity=False) + reps = cast( + dict[URIRef, EntityRepresentation], + {ont_entity: ont_rep, chunk_entity: chunk_rep}, + ) + + selected = cluster_representative_selector.select_representative( + [ont_entity, chunk_entity], reps + ) + assert selected == ont_entity + + +def test_select_representative_prefers_simple_non_ontology_uri( + cluster_representative_selector: ClusterRepresentativeSelector, +) -> None: + simple = URIRef("http://chunk1.org/Thing") + complex_uri = URIRef("http://chunk2.org/very_long_complex_entity_name_123") + + simple_rep = Mock(is_ontology_entity=False) + complex_rep = Mock(is_ontology_entity=False) + reps = cast( + dict[URIRef, EntityRepresentation], + {simple: simple_rep, complex_uri: complex_rep}, + ) + + selected = cluster_representative_selector.select_representative( + [simple, complex_uri], reps + ) + assert selected == simple + + +def test_select_representative_returns_singleton( + cluster_representative_selector: ClusterRepresentativeSelector, +) -> None: + entity = URIRef("http://chunk1.org/Only") + rep = Mock(is_ontology_entity=False) + reps = cast(dict[URIRef, EntityRepresentation], {entity: rep}) + + selected = cluster_representative_selector.select_representative([entity], reps) + assert selected == entity + + +def test_create_mapping_maps_all_cluster_members( + cluster_representative_selector: ClusterRepresentativeSelector, +) -> None: + e1 = URIRef("http://chunk1.org/A") + e2 = URIRef("http://chunk1.org/B") + e3 = URIRef("http://chunk2.org/C") + + rep1 = Mock(is_ontology_entity=False) + rep2 = Mock(is_ontology_entity=False) + rep3 = Mock(is_ontology_entity=False) + + reps = cast(dict[URIRef, EntityRepresentation], {e1: rep1, e2: rep2, e3: rep3}) + mapping = cluster_representative_selector.create_mapping([[e1, e2], [e3]], reps) + + assert mapping[e1] == mapping[e2] + assert mapping[e3] == e3 diff --git a/ontology_platform/vendored/ontocast/test/aggregation/test_normalizer.py b/ontology_platform/vendored/ontocast/test/aggregation/test_normalizer.py new file mode 100644 index 0000000..c835a6c --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/aggregation/test_normalizer.py @@ -0,0 +1,61 @@ +from rdflib import RDF, RDFS, Literal, Namespace, URIRef + +from ontocast.onto.constants import DEFAULT_IRI +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.tool.agg.normalizer import EntityNormalizer + + +def test_normalize_string_camel_case(normalizer: EntityNormalizer) -> None: + assert normalizer.normalize_string("PLRedShift") == "pl red shift" + + +def test_normalize_string_snake_case(normalizer: EntityNormalizer) -> None: + assert normalizer.normalize_string("PL_red_shift_value") == "pl red shift value" + + +def test_normalize_string_diacritics(normalizer: EntityNormalizer) -> None: + assert normalizer.normalize_string("Café") == "cafe" + + +def test_normalize_uri_variants(normalizer: EntityNormalizer) -> None: + camel_uri = URIRef("http://example.org/PLRedShift") + snake_uri = URIRef("http://example.org/PL_red_shift_value") + assert normalizer.normalize_uri(camel_uri) == "pl red shift" + assert normalizer.normalize_uri(snake_uri) == "pl red shift value" + + +def test_is_ontology_entity(normalizer: EntityNormalizer) -> None: + assert normalizer.is_ontology_entity(URIRef("http://ontology.org/Thing")) is True + assert normalizer.is_ontology_entity(URIRef(f"{DEFAULT_IRI}/entity")) is False + + +def test_create_representation_collects_metadata(normalizer: EntityNormalizer) -> None: + graph = RDFGraph() + ex = Namespace("http://example.org/") + ont = Namespace("http://ontology.org/") + + entity = ex.TestEntity + graph.add((entity, RDF.type, ont.Thing)) + graph.add((entity, RDFS.label, Literal("Test Entity"))) + graph.add((entity, ex.hasValue, Literal("123"))) + + representation = normalizer.create_representation(entity, graph) + + assert representation.entity == entity + assert "test entity" in representation.normal_form + assert representation.types == [ont.Thing] + assert "Test Entity" in representation.labels + assert ex.hasValue in representation.properties + assert "type" in representation.representation + + +def test_create_representation_marks_ontology_entity( + normalizer: EntityNormalizer, +) -> None: + graph = RDFGraph() + ont = Namespace("http://ontology.org/") + entity = ont.SomeClass + graph.add((entity, RDF.type, RDFS.Class)) + + representation = normalizer.create_representation(entity, graph) + assert representation.is_ontology_entity is True diff --git a/ontology_platform/vendored/ontocast/test/aggregation/test_provenance.py b/ontology_platform/vendored/ontocast/test/aggregation/test_provenance.py new file mode 100644 index 0000000..0912b23 --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/aggregation/test_provenance.py @@ -0,0 +1,107 @@ +from rdflib import RDF, Literal, URIRef +from rdflib.namespace import XSD + +from ontocast.onto.constants import DEFAULT_IRI, PROV, RDF_REIFIES, SCHEMA +from ontocast.onto.content_unit import ContentUnit, OutputType +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.tool.agg.rewriter import GraphRewriter + + +def test_merge_graphs_with_provenance_adds_chunk_metadata( + graph_rewriter: GraphRewriter, +) -> None: + graph = RDFGraph() + entity = URIRef(f"{DEFAULT_IRI}/Entity1") + graph.add((entity, RDF.type, URIRef(f"{DEFAULT_IRI}/Thing"))) + + unit = ContentUnit( + text="test", + index=5, + doc_iri=URIRef("https://example.org/doc/abc123"), + graph=graph, + type=OutputType.FACTS, + ) + merged = graph_rewriter.merge_graphs_with_provenance([unit], mapping={}) + unit_uri = URIRef(unit.iri_absolute) + + assert (unit_uri, RDF.type, PROV.Entity) in merged + assert (unit_uri, SCHEMA.position, Literal(5, datatype=XSD.integer)) in merged + assert (unit_uri, SCHEMA.identifier, Literal(unit.hid)) in merged + + namespaces = {prefix: str(namespace) for prefix, namespace in merged.namespaces()} + + assert namespaces["prov"] == str(PROV) + assert namespaces["schema"] == str(SCHEMA) + assert namespaces["doc"] == "https://example.org/doc/abc123/" + + +def test_merge_graphs_with_provenance_reifies_mapped_triple( + graph_rewriter: GraphRewriter, +) -> None: + graph = RDFGraph() + old_subject = URIRef("http://chunk.org/OldEntity") + old_predicate = URIRef("http://chunk.org/prop") + value = Literal("value") + graph.add((old_subject, old_predicate, value)) + + new_subject = URIRef(f"{DEFAULT_IRI}/NewEntity") + new_predicate = URIRef(f"{DEFAULT_IRI}/prop") + unit = ContentUnit( + text="test", + index=0, + doc_iri=URIRef("https://example.org/doc"), + graph=graph, + type=OutputType.FACTS, + ) + + merged = graph_rewriter.merge_graphs_with_provenance( + [unit], + {old_subject: new_subject, old_predicate: new_predicate}, + ) + stmt_nodes = list(merged.subjects(RDF_REIFIES, None)) + assert len(stmt_nodes) == 1 + + reified = list(merged.objects(stmt_nodes[0], RDF_REIFIES)) + assert len(reified) == 1 + quoted = reified[0] + assert isinstance(quoted, tuple) + assert quoted[0] == new_subject + assert quoted[1] == new_predicate + assert str(quoted[2]) == str(value) + + +def test_shared_triple_accumulates_multiple_provenance_sources( + graph_rewriter: GraphRewriter, +) -> None: + triple = ( + URIRef(f"{DEFAULT_IRI}/Alice"), + URIRef(f"{DEFAULT_IRI}/knows"), + URIRef(f"{DEFAULT_IRI}/Bob"), + ) + graph_a = RDFGraph() + graph_b = RDFGraph() + graph_a.add(triple) + graph_b.add(triple) + + unit_a = ContentUnit( + text="chunk 0", + index=0, + doc_iri=URIRef("https://example.org/doc"), + graph=graph_a, + type=OutputType.FACTS, + ) + unit_b = ContentUnit( + text="chunk 1", + index=1, + doc_iri=URIRef("https://example.org/doc"), + graph=graph_b, + type=OutputType.FACTS, + ) + + merged = graph_rewriter.merge_graphs_with_provenance([unit_a, unit_b], mapping={}) + statements = list(merged.subjects(RDF_REIFIES, None)) + assert len(statements) == 1 + + sources = {str(src) for src in merged.objects(statements[0], PROV.wasDerivedFrom)} + assert str(URIRef(unit_a.iri_absolute)) in sources + assert str(URIRef(unit_b.iri_absolute)) in sources diff --git a/ontology_platform/vendored/ontocast/test/aggregation/test_rewriter.py b/ontology_platform/vendored/ontocast/test/aggregation/test_rewriter.py new file mode 100644 index 0000000..1306787 --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/aggregation/test_rewriter.py @@ -0,0 +1,123 @@ +from rdflib import OWL, RDF, Literal, URIRef + +from ontocast.onto.constants import DEFAULT_IRI +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.tool.agg.rewriter import GraphRewriter + + +def test_apply_mapping_to_triple(graph_rewriter: GraphRewriter) -> None: + e1 = URIRef("http://chunk1.org/e1") + p1 = URIRef("http://chunk1.org/p1") + e2 = URIRef("http://chunk1.org/e2") + + e1_new = URIRef(f"{DEFAULT_IRI}/Entity1") + p1_new = URIRef(f"{DEFAULT_IRI}/property1") + e2_new = URIRef(f"{DEFAULT_IRI}/Entity2") + + mapped = graph_rewriter.apply_mapping_to_triple( + e1, + p1, + e2, + {e1: e1_new, p1: p1_new, e2: e2_new}, + ) + assert mapped == (e1_new, p1_new, e2_new) + + +def test_apply_mapping_preserves_ontology_type_object( + graph_rewriter: GraphRewriter, +) -> None: + entity = URIRef("http://chunk1.org/entity") + ontology_type = URIRef("http://ontology.org/Thing") + mapped_entity = URIRef(f"{DEFAULT_IRI}/Entity") + + new_s, new_p, new_o = graph_rewriter.apply_mapping_to_triple( + entity, + RDF.type, + ontology_type, + {entity: mapped_entity}, + ) + assert (new_s, new_p, new_o) == (mapped_entity, RDF.type, ontology_type) + + +def test_rewrite_graph_applies_mapping(graph_rewriter: GraphRewriter) -> None: + graph = RDFGraph() + e1 = URIRef("http://chunk1.org/e1") + e2 = URIRef("http://chunk1.org/e2") + p = URIRef("http://chunk1.org/p") + ont_type = URIRef("http://ontology.org/Thing") + + graph.add((e1, p, e2)) + graph.add((e1, RDF.type, ont_type)) + + e1_new = URIRef(f"{DEFAULT_IRI}/Entity1") + e2_new = URIRef(f"{DEFAULT_IRI}/Entity2") + p_new = URIRef(f"{DEFAULT_IRI}/relatesTo") + + rewritten = graph_rewriter.rewrite_graph(graph, {e1: e1_new, e2: e2_new, p: p_new}) + assert (e1_new, p_new, e2_new) in rewritten + assert (e1_new, RDF.type, ont_type) in rewritten + + +def test_merge_graphs_deduplicates_triples(graph_rewriter: GraphRewriter) -> None: + graph1 = RDFGraph() + graph2 = RDFGraph() + e = URIRef("http://chunk1.org/e") + p = URIRef("http://chunk1.org/p") + value = Literal("value") + + graph1.add((e, p, value)) + graph2.add((e, p, value)) + + merged = graph_rewriter.merge_graphs( + [graph1, graph2], + mapping={ + e: URIRef(f"{DEFAULT_IRI}/Entity"), + p: URIRef(f"{DEFAULT_IRI}/hasValue"), + }, + base_namespace=DEFAULT_IRI, + ) + assert ( + len(list(merged.triples((URIRef(f"{DEFAULT_IRI}/Entity"), None, value)))) == 1 + ) + + +def test_rewrite_graph_adds_sameas_for_merged_entities( + graph_rewriter: GraphRewriter, +) -> None: + graph_rewriter = GraphRewriter(add_sameas_links=True) + graph = RDFGraph() + e1 = URIRef("http://chunk1.org/e1") + e2 = URIRef("http://chunk2.org/e2") + p = URIRef("http://chunk1.org/p") + canonical = URIRef(f"{DEFAULT_IRI}/Entity") + + graph.add((e1, p, Literal("a"))) + graph.add((e2, p, Literal("b"))) + + rewritten = graph_rewriter.rewrite_graph(graph, {e1: canonical, e2: canonical}) + assert len(list(rewritten.triples((canonical, OWL.sameAs, None)))) >= 1 + + +def test_rewriter_blocks_sameas_for_forbidden_namespace() -> None: + base = "https://growgraph.dev/facts" + graph_rewriter = GraphRewriter( + add_sameas_links=True, + blocked_sameas_namespaces=(base,), + ) + graph = RDFGraph() + original_fact = URIRef("https://growgraph.dev/factsPersonA") + original_doc = URIRef("https://example.org/docs/case-1/PersonA") + canonical_doc = URIRef("https://example.org/docs/case-1/PersonCanonical") + relation = URIRef("https://example.org/relation") + graph.add((original_doc, relation, Literal("value"))) + + rewritten = graph_rewriter.rewrite_graph( + graph, + { + original_doc: canonical_doc, + original_fact: canonical_doc, + }, + ) + + assert (canonical_doc, OWL.sameAs, original_doc) in rewritten + assert (canonical_doc, OWL.sameAs, original_fact) not in rewritten diff --git a/ontology_platform/vendored/ontocast/test/aggregation/test_uri_builder.py b/ontology_platform/vendored/ontocast/test/aggregation/test_uri_builder.py new file mode 100644 index 0000000..8f53d5d --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/aggregation/test_uri_builder.py @@ -0,0 +1,141 @@ +from rdflib import OWL, RDF, RDFS, URIRef + +from ontocast.onto.constants import DEFAULT_IRI +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.tool.agg.normalizer import EntityRepresentation +from ontocast.tool.agg.uri_builder import ( + EntityRole, + URIBuilder, + detect_role, + format_structured_id, + has_structured_id, + normalize_local_name, + to_lower_camel_case, + to_pascal_case, +) + + +def make_representation(uri: str, normal_form: str) -> EntityRepresentation: + return EntityRepresentation( + entity=URIRef(uri), + normal_form=normal_form, + types=[], + properties=[], + labels=[], + representation=normal_form, + is_ontology_entity=False, + ) + + +def test_pascal_case_and_lower_camel_helpers() -> None: + assert to_pascal_case("judicial decision") == "JudicialDecision" + assert to_pascal_case("case") == "Case" + assert to_lower_camel_case("has decision") == "hasDecision" + assert to_lower_camel_case("name") == "name" + + +def test_structured_id_helpers() -> None: + assert has_structured_id(URIRef("http://ex.org/Case_2023_456")) is True + assert has_structured_id(URIRef("http://ex.org/Person")) is False + assert ( + format_structured_id(URIRef("http://ex.org/case_2023_456")) == "Case_2023_456" + ) + + +def test_detect_role_for_class_property_and_instance() -> None: + graph = RDFGraph() + class_entity = URIRef("http://ex.org/Person") + prop_entity = URIRef("http://ex.org/hasAge") + instance_entity = URIRef("http://ex.org/Alice") + + graph.add((class_entity, RDF.type, RDFS.Class)) + graph.add((prop_entity, RDF.type, OWL.DatatypeProperty)) + graph.add((instance_entity, RDF.type, class_entity)) + + assert detect_role(class_entity, graph) == EntityRole.CLASS + assert detect_role(prop_entity, graph) == EntityRole.PROPERTY + assert detect_role(instance_entity, graph) == EntityRole.INSTANCE + + +def test_normalize_local_name_uses_role_specific_formatting() -> None: + class_rep = make_representation( + "http://ex.org/JudicialDecision", "judicial decision" + ) + prop_rep = make_representation("http://ex.org/hasDecision", "has decision") + structured_rep = make_representation("http://ex.org/Case_2023_456", "case 2023 456") + + assert normalize_local_name(class_rep, EntityRole.CLASS) == "JudicialDecision" + assert normalize_local_name(prop_rep, EntityRole.PROPERTY) == "hasDecision" + assert normalize_local_name(structured_rep, EntityRole.INSTANCE) == "Case_2023_456" + + +def test_build_uri_preserves_ontology_entities(uri_builder: URIBuilder) -> None: + entity = URIRef("http://ontology.org/Thing") + rep = EntityRepresentation( + entity=entity, + normal_form="thing", + types=[], + properties=[], + labels=[], + representation="thing", + is_ontology_entity=True, + ) + assert uri_builder.build_uri(entity, rep, EntityRole.CLASS) == entity + + +def test_compose_mappings_flattens_two_stage_mapping() -> None: + e1 = URIRef("http://chunk1.org/A") + e2 = URIRef("http://chunk2.org/B") + representative = URIRef("http://chunk1.org/A") + final = URIRef(f"{DEFAULT_IRI}/SomeEntity") + + composed = URIBuilder.compose_mappings( + {e1: representative, e2: representative}, + {representative: final}, + ) + + assert composed[e1] == final + assert composed[e2] == final + + +def test_create_entity_uri_mapping_uses_doc_namespace_and_avoids_collisions() -> None: + builder = URIBuilder(base_iri=DEFAULT_IRI) + doc_iri = URIRef("https://example.org/docs/case-1") + left = URIRef("https://growgraph.dev/factsEntityA") + right = URIRef("https://growgraph.dev/factsEntityB") + left_canonical = URIRef("https://growgraph.dev/factsCanonicalA") + right_canonical = URIRef("https://growgraph.dev/factsCanonicalB") + + shared_representation = EntityRepresentation( + entity=left_canonical, + normal_form="collision", + types=[], + properties=[], + labels=[], + representation="collision", + is_ontology_entity=False, + ) + + representations = { + left_canonical: shared_representation, + right_canonical: EntityRepresentation( + entity=right_canonical, + normal_form="collision", + types=[], + properties=[], + labels=[], + representation="collision", + is_ontology_entity=False, + ), + } + + mapping = builder.create_entity_uri_mapping( + identity_mapping={left: left_canonical, right: right_canonical}, + representations=representations, + entity_doc_iris={left: doc_iri, right: doc_iri}, + entity_is_ontology={left_canonical: False, right_canonical: False}, + ) + + assert str(mapping[left]).startswith(f"{doc_iri}/") + assert str(mapping[right]).startswith(f"{doc_iri}/") + assert mapping[left] != mapping[right] diff --git a/ontology_platform/vendored/ontocast/test/conftest.py b/ontology_platform/vendored/ontocast/test/conftest.py new file mode 100644 index 0000000..4b6a2db --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/conftest.py @@ -0,0 +1,549 @@ +"""Pytest configuration for test suite.""" + +import importlib +import json +import logging +import os +import warnings +from pathlib import Path +from typing import TYPE_CHECKING, Optional + +import pytest +from suthing import FileHandle + +if TYPE_CHECKING: + from langchain_huggingface import HuggingFaceEmbeddings + +from ontocast.config import ( + Config, + LLMConfig, + LLMProvider, + OpenAIModel, + PathConfig, + ToolConfig, +) +from ontocast.onto.constants import DEFAULT_DOMAIN +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.state import AgentState +from ontocast.tool import ( + FilesystemTripleStoreManager, + LLMTool, + OntologyManager, +) +from ontocast.tool.triple_manager.mock import ( + MockFusekiTripleStoreManager, + MockNeo4jTripleStoreManager, +) +from ontocast.toolbox import ToolBox + +logger = logging.getLogger(__name__) + +# Suppress deprecation warnings from third-party libraries that we cannot control +# Note: We adapt to new conventions where possible (e.g., using pyld directly for JSON-LD +# instead of rdflib's deprecated ConjunctiveGraph). These suppressions are only for +# warnings from external libraries that we cannot modify. + +warnings.filterwarnings( + "ignore", + category=DeprecationWarning, + message=".*@model_validator.*mode='after'.*", + module="docling_core", +) + + +def pytest_configure(config): + """Configure pytest to suppress known deprecation warnings from third-party libraries.""" + # Suppress Pydantic deprecation warnings from docling_core (third-party library we cannot modify) + config.addinivalue_line( + "filterwarnings", + "ignore::DeprecationWarning:docling_core", + ) + + +@pytest.fixture +def current_domain(): + return os.getenv("CURRENT_DOMAIN", DEFAULT_DOMAIN) + + +@pytest.fixture +def llm_base_url(): + return os.getenv("LLM_BASE_URL", None) + + +@pytest.fixture +def provider(): + return os.getenv("LLM_PROVIDER", LLMProvider.OPENAI) + + +@pytest.fixture +def model_name(): + return OpenAIModel(os.getenv("LLM_MODEL_NAME", OpenAIModel.GPT4_O_MINI)) + + +@pytest.fixture +def temperature(): + return 0.1 + + +@pytest.fixture +def test_ontology(): + from ontocast.onto.ontology import Ontology + + graph = RDFGraph._from_turtle_str( + """ + @prefix rdf: . + @prefix rdfs: . + @prefix owl: . + @prefix ex: . + @prefix schema: . + @prefix dcterms: . + + ex: rdf:type owl:Ontology ; + rdfs:label "Test Domain Ontology" ; + dcterms:title "test_onto"^^rdf:XMLLiteral ; + rdfs:comment "An ontology for testing that covers basic concepts and relationships in a test domain. Used for validating ontology processing functionality." . + + ex:SpaceTimeEvent a rdfs:Class ; + rdfs:label "Event" ; + rdfs:comment "Some kind of event with spacetime coordinates" ; + rdfs:subClassOf schema:Event . """ + ) + return Ontology(graph=graph) + + +@pytest.fixture +def ontology_path(): + return Path("data/ontologies") + + +@pytest.fixture +def working_directory(): + return None + # return Path("test/tmp") + + +@pytest.fixture +def llm_tool(provider, model_name, temperature, llm_base_url): + config = LLMConfig( + provider=LLMProvider(provider), + model_name=model_name, + temperature=temperature, + base_url=llm_base_url, + ) + llm_tool = LLMTool.create(config=config) + return llm_tool + + +@pytest.fixture +def tsm_tool(ontology_path, working_directory): + return FilesystemTripleStoreManager( + working_directory=working_directory, ontology_path=ontology_path + ) + + +@pytest.fixture +def tools( + ontology_path, + working_directory, + model_name, + temperature, + provider, + llm_base_url, + om_tool_fname, +) -> ToolBox: + # Create LLM config + llm_config = LLMConfig( + provider=LLMProvider(provider), + model_name=model_name, + temperature=temperature, + base_url=llm_base_url, + ) + + # Create path config + path_config = PathConfig( + working_directory=working_directory, + ontology_directory=ontology_path, + ) + + # Create tool config + tool_config = ToolConfig( + llm_config=llm_config, + path_config=path_config, + ) + + # Create main config + config = Config(tool_config=tool_config) + + tools: ToolBox = ToolBox(config=config) + import asyncio + + asyncio.run(tools.initialize()) + + # Load ontologies from JSON file if it exists (using Pydantic's load method) + json_path = Path(om_tool_fname) + if json_path.exists(): + try: + loaded_om = OntologyManager.load(json_path) + # Merge loaded ontologies into the toolbox's ontology manager + for iri, versions in loaded_om.ontology_versions.items(): + for ontology in versions: + tools.ontology_manager.add_ontology(ontology) + except Exception: + # Silently fail if JSON loading fails + pass + + return tools + + +@pytest.fixture +def state_chunked(state_chunked_filename): + return AgentState.load(state_chunked_filename) + + +@pytest.fixture +def state_ontology_selected(state_onto_selected_filename): + return AgentState.load(state_onto_selected_filename) + + +@pytest.fixture +def state_ontology_rendered(state_ontology_rendered_filename): + return AgentState.load(state_ontology_rendered_filename) + + +@pytest.fixture +def state_ontology_criticized(state_ontology_criticized_filename): + return AgentState.load(state_ontology_criticized_filename) + + +@pytest.fixture +def state_rendered_facts(state_rendered_facts_filename): + return AgentState.load(state_rendered_facts_filename) + + +@pytest.fixture +def state_sublimated(state_sublimated_filename): + return AgentState.load(state_sublimated_filename) + + +@pytest.fixture +def state_facts_failed(state_facts_failed_filename): + return AgentState.load(state_facts_failed_filename) + + +@pytest.fixture +def state_facts_success(state_facts_success_filename): + return AgentState.load(state_facts_success_filename) + + +@pytest.fixture +def agent_state_select_ontology_null(state_onto_null_filename): + return AgentState.load(state_onto_null_filename) + + +@pytest.fixture +def om_tool(om_tool_fname): + try: + return OntologyManager.load(om_tool_fname) + except (FileNotFoundError, Exception): + return OntologyManager() + + +@pytest.fixture +def max_iter(): + return 2 + + +@pytest.fixture +def apple_report(): + r = FileHandle.load(Path("data/json/fin.10Q.apple.json")) + return {"text": r["text"]} + + +@pytest.fixture +def random_report(): + return FileHandle.load(Path("data/json/random.json")) + + +@pytest.fixture +def agent_state_onto_fresh(): + return AgentState.load("test/data/state_onto_addendum.json") + + +@pytest.fixture(scope="session") +def neo4j_uri(): + return os.environ.get("NEO4J_URI", "bolt://localhost:7687") + + +@pytest.fixture(scope="session") +def neo4j_auth(): + return os.environ.get("NEO4J_AUTH", "neo4j/test") + + +@pytest.fixture(scope="session") +def neo4j_triple_store_manager(neo4j_uri, neo4j_auth): + """Mock Neo4j triple store manager for testing.""" + return MockNeo4jTripleStoreManager(uri=neo4j_uri, auth=neo4j_auth, clean=True) + + +@pytest.fixture(scope="session") +def fuseki_triple_store_manager(): + """Mock Fuseki triple store manager for testing.""" + uri = os.environ.get("FUSEKI_URI", "http://localhost:3030/test") + auth = os.environ.get("FUSEKI_AUTH", None) + if auth and "/" in auth: + auth = tuple(auth.split("/", 1)) + return MockFusekiTripleStoreManager(uri=uri, auth=auth, dataset="test", clean=True) + + +@pytest.fixture(scope="session") +def real_embeddings() -> Optional["HuggingFaceEmbeddings"]: + """Fixture providing real HuggingFace embeddings if available, otherwise None. + + Uses the same model as in split_chunks.py for consistency. + Session-scoped so the model is loaded only once per test session and reused. + """ + + try: + torch = importlib.import_module("torch") + from langchain_huggingface import HuggingFaceEmbeddings + + embeddings = HuggingFaceEmbeddings( + model_name="sentence-transformers/paraphrase-multilingual-mpnet-base-v2", + model_kwargs={ + "device": "cuda" + if torch is not None and torch.cuda.is_available() + else "cpu" + }, + encode_kwargs={"normalize_embeddings": False}, + ) + return embeddings + except ImportError as e: + logger.error(f"Could not import HuggingFaceEmbeddings: {e}") + return None + except Exception: + return None + + +@pytest.fixture(scope="session") +def mock_embeddings(): + try: + from langchain_core.embeddings import Embeddings + except ImportError as e: + logger.error(f"Could not import Embeddings: {e}") + + class MockEmbeddings(Embeddings): + """Mock embeddings for testing. + + Returns deterministic embeddings based on text content. + """ + + def __init__(self, embedding_dim: int = 384): + """Initialize mock embeddings. + + Args: + embedding_dim: Dimension of the embedding vectors. Defaults to 384. + """ + self.embedding_dim = embedding_dim + # Simple hash-based embedding for deterministic results + self._cache: dict[str, list[float]] = {} + + def embed_documents(self, texts: list[str]) -> list[list[float]]: + """Generate embeddings for a list of texts.""" + return [self.embed_query(text) for text in texts] + + def embed_query(self, text: str) -> list[float]: + """Generate an embedding for a single text.""" + if text in self._cache: + return self._cache[text] + + from ontocast.util import render_text_hash + + hash_int = int(render_text_hash(text, digits=None), 16) + + embedding = [] + for i in range(self.embedding_dim): + val = (hash_int + i * 17) % 1000 + embedding.append((val / 1000.0) - 0.5) + + self._cache[text] = embedding + return embedding + + return MockEmbeddings() + + +@pytest.fixture(scope="session") +def embeddings(real_embeddings, mock_embeddings): + """Fixture providing embeddings - prefers real embeddings, falls back to mock. + + Session-scoped so the model is loaded only once per test session. + """ + if real_embeddings is not None: + return real_embeddings + return mock_embeddings + + +@pytest.fixture +def sample_text(): + """Fixture providing realistic sample text (~10k characters) from clinical trial JSON.""" + json_file = ( + Path(__file__).parent.parent + / "data" + / "json" + / "clinical.trials.NCT01239745.json" + ) + if json_file.exists(): + data = json.load(open(json_file)) + + def json_to_md(data, depth=1): + md = [] + if isinstance(data, dict): + for key, value in data.items(): + if isinstance(value, (str, int, float, bool, type(None))): + md.append(f"{key}: {value}\n") + elif isinstance(value, dict): + md.append(f"{key}:\n") + md.extend(json_to_md(value, depth + 1)) + elif isinstance(value, list): + md.append(f"{key}:\n") + for item in value: + if isinstance(item, (str, int, float, bool, type(None))): + md.append(f" - {item}\n") + else: + md.extend(json_to_md(item, depth + 1)) + elif isinstance(data, list): + for item in data: + if isinstance(item, (str, int, float, bool, type(None))): + md.append(f"- {item}\n") + else: + md.extend(json_to_md(item, depth)) + return md + + text_lines = json_to_md(data) + text = "".join(text_lines) + return text[:10000] + + # Fallback + return ( + "This is the first sentence. " + "This is the second sentence. " + "This is the third sentence. " + "This is the fourth sentence. " + "This is the fifth sentence. " + "This is the sixth sentence. " + "This is the seventh sentence. " + "This is the eighth sentence. " + "This is the ninth sentence. " + "This is the tenth sentence." + ) * 100 + + +@pytest.fixture +def long_text(): + """Fixture providing longer text for testing min/max size constraints.""" + paragraphs = [] + for i in range(5): + sentences = [] + for j in range(10): + sentences.append( + f"This is paragraph {i + 1}, sentence {j + 1}. " + f"It contains some content to make it longer. " + f"Here is more text to ensure we have enough characters." + ) + paragraphs.append(" ".join(sentences)) + return "\n\n".join(paragraphs) + + +# --- Aggregator test fixtures (used by test_aggregator.py) --- + + +@pytest.fixture +def normalizer(): + """EntityNormalizer instance for aggregator tests.""" + from ontocast.tool.agg.normalizer import EntityNormalizer + + return EntityNormalizer() + + +@pytest.fixture +def cluster_representative_selector(): + """ClusterRepresentativeSelector instance for aggregator tests.""" + from ontocast.tool.agg.clustering import ClusterRepresentativeSelector + + return ClusterRepresentativeSelector() + + +@pytest.fixture +def uri_builder(): + """URIBuilder instance for aggregator tests.""" + from ontocast.tool.agg.uri_builder import URIBuilder + + return URIBuilder() + + +@pytest.fixture +def graph_rewriter(): + """GraphRewriter instance for aggregator tests (add_sameas_links=True).""" + from ontocast.tool.agg.rewriter import GraphRewriter + + return GraphRewriter(add_sameas_links=False) + + +def triple_store_roundtrip(manager, test_ontology): + # test_ontology is already an Ontology object, use it directly + ontology = test_ontology + # Store ontology + manager.serialize(ontology) + # Fetch ontologies + ontologies = manager.fetch_ontologies() + # There should be at least one ontology with the correct ontology_id + assert any(o.ontology_id == "to" for o in ontologies) + # The ontology graph should have the same number of triples as the input + assert len(ontologies[0].graph) == len(ontology.graph) + + +def triple_store_serialize_facts(manager): + """Test serializing facts (RDF triples) to triple store and retrieving them.""" + # Create test facts + facts = RDFGraph._from_turtle_str( + """ + @prefix rdf: . + @prefix rdfs: . + @prefix ex: . + @prefix schema: . + + ex:Person a rdfs:Class ; + rdfs:label "Person" ; + rdfs:comment "A human being" . + + ex:John a ex:Person ; + rdfs:label "John Doe" ; + schema:name "John Doe" ; + schema:email "john@example.com" . + + ex:Jane a ex:Person ; + rdfs:label "Jane Smith" ; + schema:name "Jane Smith" ; + schema:email "jane@example.com" . + + ex:knows a rdf:Property ; + rdfs:label "knows" ; + rdfs:comment "Relationship between people who know each other" . + + ex:John ex:knows ex:Jane . + """ + ) + # Verify we have the expected number of triples + expected_triple_count = len(facts) + assert expected_triple_count == 15, "Test facts should contain triples" + # Serialize facts to triple store + result = manager.serialize(facts) + assert result is not None, "serialize should return a result" + + +def triple_store_serialize_empty_facts(manager): + """Test serializing empty facts graph.""" + # Create empty facts + empty_facts = RDFGraph() + # Serialize empty facts - should not raise an error + result = manager.serialize(empty_facts) + assert result is not None, "serialize should return a result even for empty graph" diff --git a/참고/playwright-main/tests/assets/network-tab/style.css b/ontology_platform/vendored/ontocast/test/manual/__init__.py similarity index 100% rename from 참고/playwright-main/tests/assets/network-tab/style.css rename to ontology_platform/vendored/ontocast/test/manual/__init__.py diff --git a/ontology_platform/vendored/ontocast/test/manual/test_agent_live_llm.py b/ontology_platform/vendored/ontocast/test/manual/test_agent_live_llm.py new file mode 100644 index 0000000..7f45f79 --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/manual/test_agent_live_llm.py @@ -0,0 +1,214 @@ +import os +from pathlib import Path + +import pytest +from rdflib import URIRef + +from ontocast.agent.criticise_facts import criticise_facts +from ontocast.agent.criticise_ontology import criticise_ontology +from ontocast.agent.render_facts import render_facts +from ontocast.agent.render_ontology import render_ontology +from ontocast.config import Config, LLMProvider +from ontocast.onto.content_unit import ContentUnit +from ontocast.onto.enum import Status +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.unit_states import UnitFactsState, UnitOntologyState +from ontocast.toolbox import ToolBox + +RUN_MANUAL_TESTS = os.getenv("ONTOCAST_RUN_MANUAL_TESTS", "0") == "1" + +pytestmark = [ + pytest.mark.skipif( + not RUN_MANUAL_TESTS, + reason="Set ONTOCAST_RUN_MANUAL_TESTS=1 to run live manual tests.", + ), +] + + +def _require_env(name: str) -> str: + value = os.getenv(name) + if value is None or value.strip() == "": + pytest.fail(f"Missing required environment variable: {name}") + return value + + +def _create_tools_from_env() -> ToolBox: + _ = _require_env("LLM_PROVIDER") + _ = _require_env("LLM_MODEL_NAME") + provider = LLMProvider(_require_env("LLM_PROVIDER").lower()) + if provider == LLMProvider.OPENAI: + _ = _require_env("LLM_API_KEY") + elif provider == LLMProvider.OLLAMA: + _ = _require_env("LLM_BASE_URL") + + _ = _require_env("ONTOCAST_WORKING_DIRECTORY") + + config = Config() + config.validate_llm_config() + + if config.tool_config.path_config.working_directory is None: + pytest.fail("ONTOCAST_WORKING_DIRECTORY must be set to run manual agent tests.") + + config.tool_config.path_config.working_directory = Path( + config.tool_config.path_config.working_directory + ).expanduser() + config.tool_config.path_config.working_directory.mkdir(parents=True, exist_ok=True) + + if config.tool_config.path_config.ontology_directory is not None: + config.tool_config.path_config.ontology_directory = Path( + config.tool_config.path_config.ontology_directory + ).expanduser() + + return ToolBox(config) + + +@pytest.fixture(scope="module") +def live_tools() -> ToolBox: + return _create_tools_from_env() + + +@pytest.fixture +def realistic_text() -> str: + return ( + "ACME Robotics announced that it signed a three-year collaboration with " + "North Valley Hospital in Berlin to deploy autonomous delivery carts across " + "seven departments. The pilot starts in May 2026 and is co-funded by the " + "hospital innovation office and the regional health authority. " + "The agreement names Dr. Lena Fischer as clinical lead and ACME CTO " + "Rahul Mehta as technical lead. Success metrics include a 20 percent " + "reduction in nurse walking distance, fewer late medication rounds, and " + "weekly safety audits." + ) + + +def _build_seed_ontology() -> Ontology: + graph = RDFGraph() + graph.parse( + data=""" + @prefix ex: . + @prefix owl: . + @prefix rdfs: . + + ex:health a owl:Ontology ; + rdfs:label "Healthcare Collaboration Ontology" . + + ex:Organization a rdfs:Class . + ex:Hospital a rdfs:Class ; rdfs:subClassOf ex:Organization . + ex:Company a rdfs:Class ; rdfs:subClassOf ex:Organization . + ex:Person a rdfs:Class . + ex:collaboratesWith a owl:ObjectProperty . + ex:hasLead a owl:ObjectProperty . + ex:locatedIn a owl:ObjectProperty . + """, + format="turtle", + ) + return Ontology(graph=graph, iri="https://example.com/health") + + +def _build_content_unit(text: str, with_seed_facts: bool = False) -> ContentUnit: + unit = ContentUnit( + text=text, + index=0, + doc_iri=URIRef("https://example.com/doc/manual-live"), + ) + if with_seed_facts: + unit.graph.parse( + data=""" + @prefix ex: . + @prefix facts: . + facts:acme ex:collaboratesWith facts:north_valley_hospital . + """, + format="turtle", + ) + return unit + + +@pytest.mark.anyio +async def test_render_facts_live_llm(live_tools: ToolBox, realistic_text: str) -> None: + state = UnitFactsState( + content_unit=_build_content_unit(realistic_text), + ontology_snapshot=_build_seed_ontology(), + facts_user_instruction=( + "Extract organizations, people, timeline details, and measurable targets." + ), + ) + + result = await render_facts(state, live_tools.get_atomic_tools()) + + assert result.failure_stage is None + assert result.status == Status.SUCCESS + assert len(result.content_unit.graph) > 0 + assert result.budget_tracker.calls_count > 0 + + +@pytest.mark.anyio +async def test_criticise_facts_live_llm( + live_tools: ToolBox, realistic_text: str +) -> None: + state = UnitFactsState( + content_unit=_build_content_unit(realistic_text), + ontology_snapshot=_build_seed_ontology(), + facts_user_instruction=( + "Prioritize correct entities, relations, and measurable outcomes." + ), + ) + rendered = await render_facts(state, live_tools.get_atomic_tools()) + assert len(rendered.content_unit.graph) > 0 + + critiqued = await criticise_facts(rendered, live_tools.get_atomic_tools()) + + assert ( + critiqued.failure_stage is None + or critiqued.failure_stage.name == "FACTS_CRITIQUE" + ) + assert critiqued.status in (Status.SUCCESS, Status.FAILED) + assert critiqued.budget_tracker.calls_count > 0 + + +@pytest.mark.anyio +async def test_render_ontology_live_llm( + live_tools: ToolBox, realistic_text: str +) -> None: + null_ontology = Ontology() + state = UnitOntologyState( + content_unit=_build_content_unit(realistic_text), + ontology_snapshot=null_ontology, + ontology_user_instruction=( + "Create a compact ontology for healthcare logistics collaboration." + ), + ) + + result = await render_ontology(state, live_tools.get_atomic_tools()) + + assert result.failure_stage is None + assert result.status == Status.SUCCESS + assert not result.current_ontology.is_null() + assert len(result.current_ontology.graph) > 0 + assert result.budget_tracker.calls_count > 0 + + +@pytest.mark.anyio +async def test_criticise_ontology_live_llm( + live_tools: ToolBox, realistic_text: str +) -> None: + null_ontology = Ontology() + state = UnitOntologyState( + content_unit=_build_content_unit(realistic_text), + ontology_snapshot=null_ontology, + ontology_user_instruction=( + "Keep class hierarchy minimal and ensure relation naming consistency." + ), + ) + rendered = await render_ontology(state, live_tools.get_atomic_tools()) + assert not rendered.current_ontology.is_null() + assert len(rendered.current_ontology.graph) > 0 + + critiqued = await criticise_ontology(rendered, live_tools.get_atomic_tools()) + + assert ( + critiqued.failure_stage is None + or critiqued.failure_stage.name == "ONTOLOGY_CRITIQUE" + ) + assert critiqued.status in (Status.SUCCESS, Status.FAILED) + assert critiqued.budget_tracker.calls_count > 0 diff --git a/ontology_platform/vendored/ontocast/test/test_agent_facts.py b/ontology_platform/vendored/ontocast/test/test_agent_facts.py new file mode 100644 index 0000000..27770d9 --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/test_agent_facts.py @@ -0,0 +1,181 @@ +import importlib +from types import SimpleNamespace +from typing import cast + +import pytest +from rdflib import URIRef + +from ontocast.onto.content_unit import ContentUnit +from ontocast.onto.enum import FailureStage, Status +from ontocast.onto.model import ( + FactsCritiqueReport, + FactsRenderReport, + SemanticTriplesFactsReport, + TripleFix, +) +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.unit_states import UnitFactsState +from ontocast.tool.atomic import AtomicToolBox + +criticise_facts_module = importlib.import_module("ontocast.agent.criticise_facts") +render_facts_module = importlib.import_module("ontocast.agent.render_facts") + + +def _build_content_unit(with_graph: bool = False) -> ContentUnit: + unit = ContentUnit( + text="Alice works for ACME.", + index=0, + doc_iri=URIRef("https://example.com/doc/d1"), + ) + if with_graph: + unit.graph.parse( + data=""" + @prefix ex: . + ex:alice ex:worksFor ex:acme . + """, + format="turtle", + ) + return unit + + +def _build_ontology() -> Ontology: + ontology_graph = RDFGraph() + ontology_graph.parse( + data=""" + @prefix onto: . + @prefix owl: . + onto:CompanyOntology a owl:Ontology . + """, + format="turtle", + ) + return Ontology(graph=ontology_graph, iri="https://example.com/onto") + + +def _build_tools() -> AtomicToolBox: + async def get_llm_tool(_budget_tracker): + return object() + + return cast(AtomicToolBox, SimpleNamespace(get_llm_tool=get_llm_tool)) + + +@pytest.mark.anyio +async def test_render_facts_routes_to_fresh_when_graph_is_empty(monkeypatch) -> None: + calls = {"fresh": 0, "update": 0} + + async def fake_fresh(state: UnitFactsState, tools) -> UnitFactsState: + calls["fresh"] += 1 + return state + + async def fake_update(state: UnitFactsState, tools) -> UnitFactsState: + calls["update"] += 1 + return state + + monkeypatch.setattr(render_facts_module, "render_facts_fresh", fake_fresh) + monkeypatch.setattr(render_facts_module, "render_facts_update", fake_update) + + state = UnitFactsState( + content_unit=_build_content_unit(with_graph=False), + ontology_snapshot=_build_ontology(), + ) + result = await render_facts_module.render_facts(state, tools=_build_tools()) + + assert result is state + assert calls["fresh"] == 1 + assert calls["update"] == 0 + + +@pytest.mark.anyio +async def test_render_facts_fresh_sets_success_and_budget(monkeypatch) -> None: + async def fake_call_llm_with_retry(**kwargs): + rendered_graph = RDFGraph() + rendered_graph.parse( + data=""" + @prefix ex: . + ex:alice ex:worksFor ex:acme . + """, + format="turtle", + ) + return FactsRenderReport( + facts_report=SemanticTriplesFactsReport( + semantic_graph=rendered_graph, + ontology_relevance_score=95, + triples_generation_score=94, + ) + ) + + monkeypatch.setattr( + render_facts_module, "call_llm_with_retry", fake_call_llm_with_retry + ) + + state = UnitFactsState( + content_unit=_build_content_unit(with_graph=False), + ontology_snapshot=_build_ontology(), + ) + result = await render_facts_module.render_facts_fresh(state, tools=_build_tools()) + + assert result.status == Status.SUCCESS + assert result.failure_stage is None + assert len(result.content_unit.graph) == 1 + assert result.budget_tracker.facts_operations_count == 1 + assert result.budget_tracker.facts_triples_generated == 1 + + +@pytest.mark.anyio +async def test_criticise_facts_marks_failed_and_sets_suggestions(monkeypatch) -> None: + async def fake_call_llm_with_retry(**kwargs): + return FactsCritiqueReport( + success=False, + score=35, + actionable_triple_fixes=[ + TripleFix( + text_fragment="Alice works for ACME.", + action="ADD", + severity="important", + explanation="Missing employment relation triple.", + correct_value="ex:alice ex:worksFor ex:acme .", + ) + ], + systemic_critique_summary="Misses key relations.", + ) + + monkeypatch.setattr( + criticise_facts_module, "call_llm_with_retry", fake_call_llm_with_retry + ) + + state = UnitFactsState( + content_unit=_build_content_unit(with_graph=True), + ontology_snapshot=_build_ontology(), + ) + result = await criticise_facts_module.criticise_facts(state, tools=_build_tools()) + + assert result.status == Status.FAILED + assert result.failure_stage == FailureStage.FACTS_CRITIQUE + assert len(result.suggestions.actionable_fixes) == 1 + assert result.failure_reason == "Facts Critic suggests improvements" + + +@pytest.mark.anyio +async def test_criticise_facts_accepts_high_score_even_when_success_false( + monkeypatch, +) -> None: + async def fake_call_llm_with_retry(**kwargs): + return FactsCritiqueReport( + success=False, + score=95, + actionable_triple_fixes=[], + systemic_critique_summary="", + ) + + monkeypatch.setattr( + criticise_facts_module, "call_llm_with_retry", fake_call_llm_with_retry + ) + + state = UnitFactsState( + content_unit=_build_content_unit(with_graph=True), + ontology_snapshot=_build_ontology(), + ) + result = await criticise_facts_module.criticise_facts(state, tools=_build_tools()) + + assert result.status == Status.SUCCESS + assert result.failure_stage is None diff --git a/ontology_platform/vendored/ontocast/test/test_aggregation_config.py b/ontology_platform/vendored/ontocast/test/test_aggregation_config.py new file mode 100644 index 0000000..17065b0 --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/test_aggregation_config.py @@ -0,0 +1,16 @@ +from ontocast.config import AggregationConfig, Config + + +def test_aggregation_config_defaults() -> None: + config = AggregationConfig() + assert config.embedding_model == "paraphrase-multilingual-MiniLM-L12-v2" + assert config.similarity_threshold == 0.80 + + +def test_aggregation_config_reads_env(monkeypatch) -> None: + monkeypatch.setenv("AGG_EMBEDDING_MODEL", "all-MiniLM-L6-v2") + monkeypatch.setenv("AGG_SIMILARITY_THRESHOLD", "0.73") + + config = Config() + assert config.tool_config.aggregation.embedding_model == "all-MiniLM-L6-v2" + assert config.tool_config.aggregation.similarity_threshold == 0.73 diff --git a/ontology_platform/vendored/ontocast/test/test_graph_update.py b/ontology_platform/vendored/ontocast/test/test_graph_update.py new file mode 100644 index 0000000..3920bb2 --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/test_graph_update.py @@ -0,0 +1,527 @@ +"""Test for GraphUpdate SPARQL query generation and execution. + +This test verifies that GraphUpdate.generate_sparql_queries() generates valid SPARQL +queries that can be executed on RDFGraph instances using rdflib's update() method. +""" + +from rdflib import Literal, URIRef + +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.sparql_models import ( + GenericSparqlQuery, + GraphUpdate, + TripleOp, +) + + +def test_rdfgraph_recovers_dangling_semicolon_at_eof() -> None: + """RDFGraph should recover from common LLM-truncated Turtle at EOF.""" + ttl = """ + @prefix ex: . + @prefix rdf: . + + ex:Case85_968 a ex:Appeal ; + ex:appealsTo ex:Cassation ; + """ + + graph = RDFGraph._from_turtle_str(ttl) + + assert len(graph) == 2 + assert ( + URIRef("http://example.org/Case85_968"), + URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://example.org/Appeal"), + ) in graph + assert ( + URIRef("http://example.org/Case85_968"), + URIRef("http://example.org/appealsTo"), + URIRef("http://example.org/Cassation"), + ) in graph + + +def test_graph_update_with_language_tags(): + """Test GraphUpdate with language-tagged literals.""" + # Create initial RDFGraph + graph = RDFGraph._from_turtle_str( + """ + @prefix rdf: . + @prefix rdfs: . + @prefix ex: . + + ex:Test a rdfs:Class . + """ + ) + + initial_triple_count = len(graph) + + # Create Turtle with language-tagged literals + triples = """ + @prefix ex: . + @prefix rdfs: . + + ex:Test rdfs:label "Test Label"@en ; + rdfs:comment "Un commentaire"@fr . + """ + + graph_update = GraphUpdate( + triple_operations=[ + TripleOp( + type="insert", + graph=triples, # type: ignore[arg-type] + prefixes={"ex": "http://example.org/"}, + ) + ] + ) + + # Generate SPARQL queries + queries = graph_update.generate_sparql_queries() + + # Should generate one query + assert len(queries) == 1 + + # Execute the query on the graph + graph.update(queries[0]) + + # Verify new triples were added + assert len(graph) == initial_triple_count + 2 + + +def test_graph_update_insert_operation(): + """Test GraphUpdate with TripleOp insert operations using Turtle format.""" + # Create initial RDFGraph + graph = RDFGraph._from_turtle_str( + """ + @prefix ex: . + @prefix rdf: . + @prefix rdfs: . + + ex:Person a rdfs:Class ; + rdfs:label "Person" . + """ + ) + + initial_triple_count = len(graph) + + # Create triples in Turtle format + triples = """ + @prefix ex: . + @prefix rdf: . + @prefix rdfs: . + + ex:John a ex:Person ; + rdfs:label "John Doe" . + """ + + graph_update = GraphUpdate( + triple_operations=[ + TripleOp( + type="insert", + graph=triples, # type: ignore[arg-type] + prefixes={"ex": "http://example.org/"}, + ) + ] + ) + + # Generate SPARQL queries + queries = graph_update.generate_sparql_queries() + + # Should generate one query + assert len(queries) == 1 + + # Execute the query on the graph + graph.update(queries[0]) + + # Verify new triples were added + assert len(graph) == initial_triple_count + 2 + assert ( + URIRef("http://example.org/John"), + URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://example.org/Person"), + ) in graph + assert ( + URIRef("http://example.org/John"), + URIRef("http://www.w3.org/2000/01/rdf-schema#label"), + Literal("John Doe"), + ) in graph + + +def test_graph_update_extract_insert_graph() -> None: + """Test GraphUpdate.extract_insert_graph returns only insert triples.""" + insert_ttl = """ + @prefix ex: . + @prefix rdf: . + @prefix rdfs: . + ex:Person a rdfs:Class . + ex:Person rdfs:label "Person" . + """ + delete_ttl = """ + @prefix ex: . + @prefix rdfs: . + ex:Obsolete a rdfs:Class . + """ + gu = GraphUpdate( + triple_operations=[ + TripleOp(type="insert", graph=insert_ttl), # type: ignore[arg-type] + TripleOp(type="delete", graph=delete_ttl), # type: ignore[arg-type] + ] + ) + insert_graph = gu.extract_insert_graph() + assert len(insert_graph) == 2 + person_uri = URIRef("http://example.org/Person") + rdf_type = URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type") + rdfs_class = URIRef("http://www.w3.org/2000/01/rdf-schema#Class") + rdfs_label = URIRef("http://www.w3.org/2000/01/rdf-schema#label") + assert (person_uri, rdf_type, rdfs_class) in insert_graph + assert (person_uri, rdfs_label, Literal("Person")) in insert_graph + obsolete_uri = URIRef("http://example.org/Obsolete") + assert (obsolete_uri, rdf_type, rdfs_class) not in insert_graph + + +def test_graph_update_delete_operation(): + """Test GraphUpdate with TripleOp delete operations.""" + # Create RDFGraph with existing triples + graph = RDFGraph._from_turtle_str( + """ + @prefix ex: . + @prefix rdf: . + @prefix rdfs: . + + ex:Person a rdfs:Class ; + rdfs:label "Person" . + + ex:John a ex:Person ; + rdfs:label "John Doe" . + + ex:Jane a ex:Person ; + rdfs:label "Jane Smith" . + """ + ) + + initial_triple_count = len(graph) + + # Create GraphUpdate with TripleOp using Turtle format + triples = """ + @prefix ex: . + @prefix rdf: . + @prefix rdfs: . + + ex:John a ex:Person ; + rdfs:label "John Doe" . + """ + + graph_update = GraphUpdate( + triple_operations=[ + TripleOp( + type="delete", + graph=triples, # type: ignore[arg-type] + prefixes={"ex": "http://example.org/"}, + ) + ] + ) + + # Generate SPARQL queries + queries = graph_update.generate_sparql_queries() + + # Should generate one query + assert len(queries) == 1 + + # Execute the query on the graph + graph.update(queries[0]) + + # Verify triples were removed + assert len(graph) == initial_triple_count - 2 + assert ( + URIRef("http://example.org/John"), + URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://example.org/Person"), + ) not in graph + assert ( + URIRef("http://example.org/John"), + URIRef("http://www.w3.org/2000/01/rdf-schema#label"), + Literal("John Doe"), + ) not in graph + # Jane should still be there + assert ( + URIRef("http://example.org/Jane"), + URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://example.org/Person"), + ) in graph + + +def test_graph_update_with_prefixes(): + """Test GraphUpdate with TripleOp operations that declare custom prefixes.""" + # Create initial RDFGraph + graph = RDFGraph._from_turtle_str( + """ + @prefix ex: . + @prefix rdf: . + + ex:Person a rdf:Class . + """ + ) + + initial_triple_count = len(graph) + + # Create GraphUpdate with custom prefixes using Turtle format + triples = """ + @prefix ex: . + @prefix rdf: . + @prefix schema: . + + ex:John a ex:Person ; + schema:name "John Doe" . + """ + + graph_update = GraphUpdate( + triple_operations=[ + TripleOp( + type="insert", + graph=triples, # type: ignore[arg-type] + prefixes={ + "ex": "http://example.org/", + "schema": "https://schema.org/", + }, + ), + ] + ) + + # Generate SPARQL queries + queries = graph_update.generate_sparql_queries() + + # Should generate one query + assert len(queries) == 1 + + # Verify the query includes PREFIX declarations + assert "PREFIX schema: " in queries[0] + + # Execute the query on the graph + graph.update(queries[0]) + + # Verify new triples were added + assert len(graph) == initial_triple_count + 2 + assert ( + URIRef("http://example.org/John"), + URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://example.org/Person"), + ) in graph + assert ( + URIRef("http://example.org/John"), + URIRef("https://schema.org/name"), + Literal("John Doe"), + ) in graph + + +def test_graph_update_mixed_operations_ordered(): + """Test GraphUpdate with mixed operations in specific order.""" + # Create initial RDFGraph + graph = RDFGraph._from_turtle_str( + """ + @prefix ex: . + @prefix rdf: . + @prefix rdfs: . + + ex:Person a rdfs:Class ; + rdfs:label "Person" . + + ex:John a ex:Person ; + rdfs:label "John Doe" . + """ + ) + + initial_triple_count = len(graph) + + # Create GraphUpdate with mixed operations using Turtle format + insert_jane = """ + @prefix ex: . + @prefix rdf: . + @prefix schema: . + + ex:Jane a ex:Person ; + schema:name "Jane Smith" . + """ + delete_john_label = """ + @prefix ex: . + @prefix rdfs: . + + ex:John rdfs:label "John Doe" . + """ + insert_john_label = """ + @prefix ex: . + @prefix rdfs: . + + ex:John rdfs:label "John Updated" . + """ + + graph_update = GraphUpdate( + triple_operations=[ + # First: Insert new person with custom schema prefix + TripleOp( + type="insert", + graph=insert_jane, # type: ignore[arg-type] + prefixes={ + "ex": "http://example.org/", + "schema": "https://schema.org/", + }, + ), + # Second: Delete John's label + TripleOp( + type="delete", + graph=delete_john_label, # type: ignore[arg-type] + prefixes={"ex": "http://example.org/"}, + ), + # Third: Insert new label for John + TripleOp( + type="insert", + graph=insert_john_label, # type: ignore[arg-type] + prefixes={"ex": "http://example.org/"}, + ), + ] + ) + + # Generate SPARQL queries + queries = graph_update.generate_sparql_queries() + + # Should generate 3 queries (one for each TripleOp) + assert len(queries) == 3 + + # Execute queries in order + for query in queries: + graph.update(query) + + # Verify final state + # Should have: 4 initial + 2 added (Jane) - 1 deleted (John's old label) + 1 added (John's new label) = 6 triples + assert ( + len(graph) == initial_triple_count + 2 + ) # +2 net change: +2 for Jane, -1 for John's old label, +1 for John's new label + + # Verify John's label was updated + assert ( + URIRef("http://example.org/John"), + URIRef("http://www.w3.org/2000/01/rdf-schema#label"), + Literal("John Updated"), + ) in graph + assert ( + URIRef("http://example.org/John"), + URIRef("http://www.w3.org/2000/01/rdf-schema#label"), + Literal("John Doe"), + ) not in graph + + # Verify Jane was added + assert ( + URIRef("http://example.org/Jane"), + URIRef("http://www.w3.org/1999/02/22-rdf-syntax-ns#type"), + URIRef("http://example.org/Person"), + ) in graph + assert ( + URIRef("http://example.org/Jane"), + URIRef("https://schema.org/name"), + Literal("Jane Smith"), + ) in graph + + +def test_graph_update_generic_sparql_query(): + """Test GraphUpdate with GenericSparqlQuery operation.""" + # Create initial RDFGraph + graph = RDFGraph._from_turtle_str( + """ + @prefix ex: . + @prefix rdf: . + @prefix rdfs: . + + ex:Person a rdfs:Class ; + rdfs:label "Person" . + + ex:John a ex:Person ; + rdfs:label "John Doe" . + """ + ) + + initial_triple_count = len(graph) + + # Create GraphUpdate with GenericSparqlQuery + # Note: GenericSparqlQuery handles its own prefix declarations + graph_update = GraphUpdate( + sparql_operations=[ + GenericSparqlQuery( + query="PREFIX ex: \nPREFIX schema: \nPREFIX rdf: \nINSERT { ex:John schema:age 30 } WHERE { ex:John rdf:type ex:Person }" + ), + ] + ) + + # Generate SPARQL queries + queries = graph_update.generate_sparql_queries() + + # Should generate one query + assert len(queries) == 1 + + # Verify the query includes the custom SPARQL with prefixes + assert "INSERT { ex:John schema:age 30 }" in queries[0] + assert "WHERE { ex:John rdf:type ex:Person }" in queries[0] + + # Execute the query on the graph + graph.update(queries[0]) + + # Verify the custom query was executed + assert len(graph) == initial_triple_count + 1 + assert ( + URIRef("http://example.org/John"), + URIRef("https://schema.org/age"), + Literal(30), + ) in graph + + +def test_graph_update_empty_operations(): + """Test GraphUpdate with empty operations list.""" + graph = RDFGraph._from_turtle_str( + """ + @prefix ex: . + @prefix rdf: . + + ex:Person a rdf:Class . + """ + ) + + initial_triple_count = len(graph) + + # Create GraphUpdate with empty operations + graph_update = GraphUpdate(triple_operations=[]) + + # Generate SPARQL queries + queries = graph_update.generate_sparql_queries() + + # Should generate no queries + assert len(queries) == 0 + + # Graph should remain unchanged + assert len(graph) == initial_triple_count + + +def test_graph_update_operations_with_empty_triples(): + """Test GraphUpdate with operations that have empty triples lists.""" + graph = RDFGraph._from_turtle_str( + """ + @prefix ex: . + @prefix rdf: . + + ex:Person a rdf:Class . + """ + ) + + initial_triple_count = len(graph) + + # Create GraphUpdate with operations that have empty triples + graph_update = GraphUpdate( + triple_operations=[ + TripleOp(type="insert", graph=RDFGraph()), + TripleOp(type="delete", graph=RDFGraph()), + ] + ) + + # Generate SPARQL queries + queries = graph_update.generate_sparql_queries() + + # Should generate no queries (empty triples are skipped) + assert len(queries) == 0 + + # Graph should remain unchanged + assert len(graph) == initial_triple_count diff --git a/ontology_platform/vendored/ontocast/test/test_merge_ontologies.py b/ontology_platform/vendored/ontocast/test/test_merge_ontologies.py new file mode 100644 index 0000000..884ca36 --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/test_merge_ontologies.py @@ -0,0 +1,285 @@ +"""Tests for ontology merging functionality.""" + +import logging +from datetime import datetime, timezone + +import pytest +from rdflib import DCTERMS, OWL, PROV, RDF, RDFS, Literal, URIRef + +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.tool.ontology_manager import OntologyManager + +logger = logging.getLogger(__name__) + + +@pytest.fixture +def ontology_manager(): + """Create an ontology manager for testing.""" + return OntologyManager() + + +@pytest.fixture +def base_ontology(): + """Create a base ontology for testing.""" + graph = RDFGraph() + iri = URIRef("http://example.org/test") + graph.add((iri, RDF.type, OWL.Ontology)) + graph.add((iri, RDFS.label, Literal("Test Ontology"))) + + # Add some classes + class1 = URIRef("http://example.org/test#Class1") + graph.add((class1, RDF.type, OWL.Class)) + graph.add((class1, RDFS.label, Literal("Class 1"))) + + ontology = Ontology( + graph=graph, + iri=str(iri), + ontology_id="test", + title="Test Ontology", + version="1.0.0", + created_at=datetime(2024, 1, 1, tzinfo=timezone.utc), + ) + # Ensure hash is computed + if not ontology.hash: + ontology._compute_and_set_hash() + return ontology + + +@pytest.fixture +def branch1_ontology(base_ontology): + """Create a branch 1 ontology (child of base).""" + # Create a copy of the base graph + graph = base_ontology.graph.copy() + + # Add new class + class2 = URIRef("http://example.org/test#Class2") + graph.add((class2, RDF.type, OWL.Class)) + graph.add((class2, RDFS.label, Literal("Class 2"))) + + ontology = Ontology( + graph=graph, + iri=base_ontology.iri, + ontology_id=base_ontology.ontology_id, + title=base_ontology.title, + version="1.1.0", + parent_hashes=[base_ontology.hash] if base_ontology.hash else [], + created_at=datetime(2024, 1, 2, tzinfo=timezone.utc), + ) + return ontology + + +@pytest.fixture +def branch2_ontology(base_ontology): + """Create a branch 2 ontology (child of base).""" + # Create a copy of the base graph + graph = base_ontology.graph.copy() + + # Add different new class + class3 = URIRef("http://example.org/test#Class3") + graph.add((class3, RDF.type, OWL.Class)) + graph.add((class3, RDFS.label, Literal("Class 3"))) + + ontology = Ontology( + graph=graph, + iri=base_ontology.iri, + ontology_id=base_ontology.ontology_id, + title=base_ontology.title, + version="1.2.0", + parent_hashes=[base_ontology.hash] if base_ontology.hash else [], + created_at=datetime(2024, 1, 3, tzinfo=timezone.utc), + ) + return ontology + + +def test_merge_ontologies_basic(ontology_manager, branch1_ontology, branch2_ontology): + """Test basic ontology merging.""" + from ontocast.onto.ontology_operations import merge_ontologies + + # Ensure hashes are computed + if not branch1_ontology.hash: + branch1_ontology._compute_and_set_hash() + if not branch2_ontology.hash: + branch2_ontology._compute_and_set_hash() + + # Merge + merged = merge_ontologies(branch1_ontology, branch2_ontology) + + # Check that merged ontology has both parents + assert merged.parent_hashes == [branch1_ontology.hash, branch2_ontology.hash] + assert merged.iri == branch1_ontology.iri + assert merged.created_at is not None + assert merged.hash is not None + + # Check that merged graph contains content triples from both (excluding metadata) + # Metadata (version, title, description, created_at, hash, parent_hash) is not compared + # as it may differ in the merged ontology + def get_content_triples(graph, onto_iri): + """Get content triples (excluding metadata) from a graph.""" + content_triples = set() + onto_iri_ref = URIRef(onto_iri) + for s, p, o in graph: + # Skip metadata triples for the ontology IRI + if s == onto_iri_ref: + if ( + p == DCTERMS.identifier + and isinstance(o, Literal) + and str(o).startswith("hash:") + ): + continue + if p == PROV.wasDerivedFrom: + continue + if p == DCTERMS.created: + continue + if p == OWL.versionInfo: + continue + if p == RDFS.label: + continue + if p == DCTERMS.title: + continue + if p == DCTERMS.description: + continue + if p == RDFS.comment: + continue + content_triples.add((s, p, o)) + return content_triples + + branch1_content = get_content_triples(branch1_ontology.graph, branch1_ontology.iri) + branch2_content = get_content_triples(branch2_ontology.graph, branch2_ontology.iri) + merged_content = get_content_triples(merged.graph, merged.iri) + + # All content triples from both branches should be in merged + assert branch1_content.issubset(merged_content), ( + f"Missing triples from branch1: {branch1_content - merged_content}" + ) + assert branch2_content.issubset(merged_content), ( + f"Missing triples from branch2: {branch2_content - merged_content}" + ) + + +def test_merge_ontologies_with_contradictions(ontology_manager): + """Test merging ontologies with contradictions.""" + from ontocast.onto.ontology_operations import merge_ontologies + + # Create two ontologies with conflicting property values + graph1 = RDFGraph() + iri = URIRef("http://example.org/test") + graph1.add((iri, RDF.type, OWL.Ontology)) + class1 = URIRef("http://example.org/test#Class1") + graph1.add((class1, RDF.type, OWL.Class)) + graph1.add((class1, RDFS.label, Literal("Class One"))) # Different label + + graph2 = RDFGraph() + graph2.add((iri, RDF.type, OWL.Ontology)) + graph2.add((class1, RDF.type, OWL.Class)) + graph2.add((class1, RDFS.label, Literal("Class 1"))) # Different label + + onto1 = Ontology( + graph=graph1, + iri=str(iri), + ontology_id="test", + created_at=datetime(2024, 1, 1, tzinfo=timezone.utc), + ) + onto2 = Ontology( + graph=graph2, + iri=str(iri), + ontology_id="test", + created_at=datetime(2024, 1, 2, tzinfo=timezone.utc), + ) + + # Merge should succeed (both values kept in RDF) + merged = merge_ontologies(onto1, onto2) + + # Both label values should be in merged graph + labels = [o for _, _, o in merged.graph.triples((class1, RDFS.label, None))] + assert len(labels) == 2 + label_strings = {str(label) for label in labels} + assert "Class One" in label_strings or '"Class One"' in label_strings + assert "Class 1" in label_strings or '"Class 1"' in label_strings + + +def test_merge_terminal_ontologies_pairwise( + ontology_manager, base_ontology, branch1_ontology, branch2_ontology +): + """Test merging terminal ontologies pair-wise.""" + from ontocast.onto.ontology_operations import merge_ontologies + + # Add all ontologies to manager + ontology_manager.add_ontology(base_ontology) + ontology_manager.add_ontology(branch1_ontology) + ontology_manager.add_ontology(branch2_ontology) + + # Get terminal ontologies (should be branch1 and branch2) + terminals = ontology_manager.get_terminal_ontologies_by_iri(base_ontology.iri) + assert len(terminals) == 2 + + # Sort by created_at + terminals.sort( + key=lambda x: x.created_at or datetime.min.replace(tzinfo=timezone.utc) + ) + + # Merge the two terminals + merged = merge_ontologies(terminals[0], terminals[1]) + + # Add merged to manager + ontology_manager.add_ontology(merged) + + # Check that we now have one terminal + new_terminals = ontology_manager.get_terminal_ontologies_by_iri(base_ontology.iri) + assert len(new_terminals) == 1 + assert new_terminals[0].hash == merged.hash + + +def test_merge_ontologies_preserves_namespaces(ontology_manager): + """Test that merging preserves namespace bindings.""" + from ontocast.onto.ontology_operations import merge_ontologies + + graph1 = RDFGraph() + graph1.bind("ex", "http://example.org/") + iri = URIRef("http://example.org/test") + graph1.add((iri, RDF.type, OWL.Ontology)) + + graph2 = RDFGraph() + graph2.bind("test", "http://test.org/") + graph2.add((iri, RDF.type, OWL.Ontology)) + + onto1 = Ontology(graph=graph1, iri=str(iri), created_at=datetime.now(timezone.utc)) + onto2 = Ontology(graph=graph2, iri=str(iri), created_at=datetime.now(timezone.utc)) + + merged = merge_ontologies(onto1, onto2) + + # Check that both namespaces are present + namespaces = dict(merged.graph.namespaces()) + assert "ex" in namespaces + assert "test" in namespaces + + +def test_merge_ontologies_created_at_set(ontology_manager): + """Test that merged ontology has created_at set to merge time.""" + from ontocast.onto.ontology_operations import merge_ontologies + + graph1 = RDFGraph() + iri = URIRef("http://example.org/test") + graph1.add((iri, RDF.type, OWL.Ontology)) + + graph2 = RDFGraph() + graph2.add((iri, RDF.type, OWL.Ontology)) + + onto1 = Ontology( + graph=graph1, + iri=str(iri), + created_at=datetime(2024, 1, 1, tzinfo=timezone.utc), + ) + onto2 = Ontology( + graph=graph2, + iri=str(iri), + created_at=datetime(2024, 1, 2, tzinfo=timezone.utc), + ) + + before_merge = datetime.now(timezone.utc) + merged = merge_ontologies(onto1, onto2) + after_merge = datetime.now(timezone.utc) + + # Created_at should be set to merge time (between before and after) + assert merged.created_at is not None + assert before_merge <= merged.created_at <= after_merge diff --git a/ontology_platform/vendored/ontocast/test/test_ontology_lineage_refresh.py b/ontology_platform/vendored/ontocast/test/test_ontology_lineage_refresh.py new file mode 100644 index 0000000..88fe08e --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/test_ontology_lineage_refresh.py @@ -0,0 +1,108 @@ +from datetime import datetime +from typing import cast + +from rdflib import DCTERMS, OWL, RDF, XSD, Literal, URIRef + +from ontocast.agent.normalize_ontology import normalize_ontology_units +from ontocast.onto.constants import PROV +from ontocast.onto.content_unit import ContentUnit, OutputType +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.toolbox import ToolBox + + +def _make_base_ontology() -> Ontology: + base_iri = URIRef("https://example.org/onto") + graph = RDFGraph() + graph.add((base_iri, RDF.type, OWL.Ontology)) + graph.add((URIRef(f"{base_iri}#Person"), RDF.type, OWL.Class)) + return Ontology(graph=graph, iri=str(base_iri)) + + +def test_derive_updated_version_refreshes_lineage_metadata() -> None: + base = _make_base_ontology() + assert base.hash is not None + base_hash = base.hash + + onto_iri = URIRef(base.iri) + updated_graph = base.graph.copy() + updated_graph.add((URIRef(f"{base.iri}#Organization"), RDF.type, OWL.Class)) + updated_graph.add((onto_iri, PROV.wasDerivedFrom, URIRef("urn:hash:stale-parent"))) + updated_graph.add((onto_iri, DCTERMS.identifier, Literal("hash:stale-hash"))) + updated_graph.add( + ( + onto_iri, + DCTERMS.created, + Literal("2001-01-01T00:00:00+00:00", datatype=XSD.dateTime), + ) + ) + + updated = base.derive_updated_version(updated_graph) + + assert updated.hash is not None + assert updated.hash != base_hash + assert updated.parent_hashes == [base_hash] + assert updated.created_at is not None + + hash_identifiers = { + str(obj) + for _, _, obj in updated.graph.triples((onto_iri, DCTERMS.identifier, None)) + if str(obj).startswith("hash:") + } + parent_uris = { + str(obj) + for _, _, obj in updated.graph.triples((onto_iri, PROV.wasDerivedFrom, None)) + } + created_values = [ + str(obj) + for _, _, obj in updated.graph.triples((onto_iri, DCTERMS.created, None)) + ] + + assert hash_identifiers == {f"hash:{updated.hash}"} + assert "hash:stale-hash" not in hash_identifiers + assert parent_uris == {f"urn:hash:{base_hash}"} + assert "urn:hash:stale-parent" not in parent_uris + assert len(created_values) == 1 + assert datetime.fromisoformat(created_values[0]) == updated.created_at + + +class _DummyTools: + pass + + +def test_normalize_ontology_units_refreshes_lineage_for_updated_base() -> None: + base = _make_base_ontology() + assert base.hash is not None + base_hash = base.hash + + doc_iri = URIRef("https://example.org/doc/alpha") + delta_graph = RDFGraph() + delta_graph.add((URIRef(f"{base.iri}#Case"), RDF.type, OWL.Class)) + unit = ContentUnit( + text="delta", + index=0, + doc_iri=doc_iri, + graph=delta_graph, + type=OutputType.ONTOLOGIES, + ) + + normalized, applied, provenance = normalize_ontology_units( + units=[unit], + tools=cast(ToolBox, _DummyTools()), + base_ontology=base, + require_base=True, + ) + + onto_iri = URIRef(base.iri) + assert len(applied) == 1 + assert normalized.hash is not None + assert normalized.hash != base_hash + assert normalized.parent_hashes == [base_hash] + assert normalized.created_at is not None + assert len(provenance) == 0 + assert (URIRef(f"{base.iri}#Case"), RDF.type, OWL.Class) in normalized.graph + assert ( + onto_iri, + PROV.wasDerivedFrom, + URIRef(f"urn:hash:{base_hash}"), + ) in normalized.graph diff --git a/ontology_platform/vendored/ontocast/test/test_ontology_manager.py b/ontology_platform/vendored/ontocast/test/test_ontology_manager.py new file mode 100644 index 0000000..d6c83a0 --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/test_ontology_manager.py @@ -0,0 +1,659 @@ +"""Test suite for OntologyManager. + +This test suite ensures that: +1. Every ontology in the manager has a created_at field set (not None) +2. Version tracking works correctly +3. Terminal ontology detection works +4. Freshest ontology selection works +5. Lineage graphs are built correctly +""" + +from datetime import datetime, timezone + +import pytest + +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.tool import OntologyManager + + +@pytest.fixture +def ontology_manager(): + """Create a fresh OntologyManager for each test.""" + return OntologyManager() + + +@pytest.fixture +def sample_ontology(): + """Create a sample ontology with minimal required fields.""" + graph = RDFGraph() + graph.parse( + data=""" + @prefix rdf: . + @prefix owl: . + @prefix rdfs: . + + a owl:Ontology ; + rdfs:label "Test Ontology" . + """, + format="turtle", + ) + ontology = Ontology( + graph=graph, + ontology_id="test", + iri="https://example.org/test", + title="Test Ontology", + version="1.0.0", + ) + # Compute hash if not set + if not ontology.hash: + ontology._compute_and_set_hash() + return ontology + + +@pytest.fixture +def ontology_with_parent(sample_ontology): + """Create an ontology that has sample_ontology as parent.""" + graph = RDFGraph() + graph.parse( + data=""" + @prefix rdf: . + @prefix owl: . + @prefix rdfs: . + + a owl:Ontology ; + rdfs:label "Test Ontology v2" . + + a owl:Class ; + rdfs:label "New Class" . + """, + format="turtle", + ) + ontology = Ontology( + graph=graph, + ontology_id="test", + iri="https://example.org/test", + title="Test Ontology v2", + version="2.0.0", + parent_hashes=[sample_ontology.hash] if sample_ontology.hash else [], + ) + if not ontology.hash: + ontology._compute_and_set_hash() + return ontology + + +class TestOntologyManagerCreatedAt: + """Test that created_at is always set when adding ontologies.""" + + def test_add_ontology_sets_created_at_if_missing( + self, ontology_manager, sample_ontology + ): + """Test that add_ontology sets created_at if it's None.""" + assert sample_ontology.created_at is None + ontology_manager.add_ontology(sample_ontology) + + # Check that created_at was set + assert sample_ontology.created_at is not None + assert isinstance(sample_ontology.created_at, datetime) + + # Check that it's in the manager + versions = ontology_manager.get_ontology_versions("test") + assert len(versions) == 1 + assert versions[0].created_at is not None + + def test_add_ontology_preserves_existing_created_at( + self, ontology_manager, sample_ontology + ): + """Test that add_ontology preserves existing created_at.""" + original_time = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + sample_ontology.created_at = original_time + + ontology_manager.add_ontology(sample_ontology) + + # Check that created_at was preserved + assert sample_ontology.created_at == original_time + + versions = ontology_manager.get_ontology_versions("test") + assert versions[0].created_at == original_time + + def test_all_ontologies_have_created_at(self, ontology_manager, sample_ontology): + """Test that all ontologies in manager have created_at set.""" + ontology_manager.add_ontology(sample_ontology) + + # Check all ontologies property + ontologies = ontology_manager.ontologies + assert len(ontologies) > 0 + for ontology in ontologies: + assert ontology.created_at is not None, ( + f"Ontology {ontology.ontology_id} (hash: {ontology.hash}) " + "should have created_at set" + ) + + def test_get_ontology_versions_all_have_created_at( + self, ontology_manager, sample_ontology, ontology_with_parent + ): + """Test that all versions returned have created_at set.""" + ontology_manager.add_ontology(sample_ontology) + ontology_manager.add_ontology(ontology_with_parent) + + versions = ontology_manager.get_ontology_versions("test") + assert len(versions) == 2 + for version in versions: + assert version.created_at is not None, ( + f"Version with hash {version.hash} should have created_at set" + ) + + +class TestOntologyManagerVersionTracking: + """Test version tracking functionality.""" + + def test_add_ontology_creates_version_tree(self, ontology_manager, sample_ontology): + """Test that adding an ontology creates a version tree.""" + ontology_manager.add_ontology(sample_ontology) + + assert sample_ontology.iri in ontology_manager.ontology_versions + assert len(ontology_manager.ontology_versions[sample_ontology.iri]) == 1 + + def test_add_duplicate_hash_not_added(self, ontology_manager, sample_ontology): + """Test that adding the same ontology twice doesn't create duplicates.""" + ontology_manager.add_ontology(sample_ontology) + ontology_manager.add_ontology(sample_ontology) + + versions = ontology_manager.get_ontology_versions("test") + assert len(versions) == 1 + + def test_add_ontology_without_hash_rejected(self, ontology_manager): + """Test that adding ontology without hash is rejected.""" + ontology = Ontology( + ontology_id="test", + iri="https://example.org/test", + ) + assert ontology.hash is None + + ontology_manager.add_ontology(ontology) + + # Should not be added + assert ontology.iri not in ontology_manager.ontology_versions + + def test_add_ontology_without_iri_rejected(self, ontology_manager, sample_ontology): + """Test that adding ontology without valid IRI is rejected.""" + sample_ontology.iri = None + + ontology_manager.add_ontology(sample_ontology) + + # Should not be added + assert len(ontology_manager.ontology_versions) == 0 + + +class TestTerminalOntologies: + """Test terminal ontology detection.""" + + def test_single_ontology_is_terminal(self, ontology_manager, sample_ontology): + """Test that a single ontology is terminal.""" + ontology_manager.add_ontology(sample_ontology) + + terminals = ontology_manager.get_terminal_ontologies("test") + assert len(terminals) == 1 + assert terminals[0].hash == sample_ontology.hash + + def test_parent_is_not_terminal_when_child_exists( + self, ontology_manager, sample_ontology, ontology_with_parent + ): + """Test that parent is not terminal when child exists.""" + ontology_manager.add_ontology(sample_ontology) + ontology_manager.add_ontology(ontology_with_parent) + + terminals = ontology_manager.get_terminal_ontologies("test") + assert len(terminals) == 1 + assert terminals[0].hash == ontology_with_parent.hash + assert sample_ontology.hash not in [t.hash for t in terminals] + + def test_multiple_terminals_for_different_ontology_ids( + self, ontology_manager, sample_ontology + ): + """Test that we can have terminals for different ontology_ids.""" + # Create second ontology with different ID + graph2 = RDFGraph() + graph2.parse( + data=""" + @prefix rdf: . + @prefix owl: . + @prefix rdfs: . + + a owl:Ontology ; + rdfs:label "Test Ontology 2" . + """, + format="turtle", + ) + ontology2 = Ontology( + graph=graph2, + ontology_id="test2", + iri="https://example.org/test2", + ) + if not ontology2.hash: + ontology2._compute_and_set_hash() + + ontology_manager.add_ontology(sample_ontology) + ontology_manager.add_ontology(ontology2) + + terminals = ontology_manager.get_terminal_ontologies() + assert len(terminals) == 2 + assert {t.ontology_id for t in terminals} == {"test", "test2"} + + +class TestFreshestTerminalOntology: + """Test freshest terminal ontology selection.""" + + def test_freshest_single_ontology(self, ontology_manager, sample_ontology): + """Test that freshest returns the only ontology when there's one.""" + ontology_manager.add_ontology(sample_ontology) + + freshest = ontology_manager.get_freshest_terminal_ontology("test") + assert freshest is not None + assert freshest.hash == sample_ontology.hash + + def test_freshest_selects_most_recent( + self, ontology_manager, sample_ontology, ontology_with_parent + ): + """Test that freshest selects the most recently created ontology.""" + # Set explicit timestamps + sample_ontology.created_at = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + ontology_with_parent.created_at = datetime( + 2024, 1, 2, 12, 0, 0, tzinfo=timezone.utc + ) + + ontology_manager.add_ontology(sample_ontology) + ontology_manager.add_ontology(ontology_with_parent) + + freshest = ontology_manager.get_freshest_terminal_ontology("test") + assert freshest is not None + assert freshest.hash == ontology_with_parent.hash + + def test_freshest_handles_no_timestamps(self, ontology_manager, sample_ontology): + """Test that freshest falls back when no timestamps.""" + # This shouldn't happen in practice since add_ontology sets created_at, + # but test the fallback logic + sample_ontology.created_at = None + # Manually add to bypass add_ontology's created_at setting + if sample_ontology.iri not in ontology_manager.ontology_versions: + ontology_manager.ontology_versions[sample_ontology.iri] = [] + ontology_manager.ontology_versions[sample_ontology.iri].append(sample_ontology) + + freshest = ontology_manager.get_freshest_terminal_ontology("test") + # Should still return something (fallback to first) + assert freshest is not None + + def test_freshest_returns_none_when_no_ontologies(self, ontology_manager): + """Test that freshest returns None when no ontologies exist.""" + freshest = ontology_manager.get_freshest_terminal_ontology("nonexistent") + assert freshest is None + + +class TestOntologiesProperty: + """Test the ontologies property.""" + + def test_ontologies_returns_freshest_per_ontology_id( + self, ontology_manager, sample_ontology, ontology_with_parent + ): + """Test that ontologies property returns one per ontology_id.""" + sample_ontology.created_at = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + ontology_with_parent.created_at = datetime( + 2024, 1, 2, 12, 0, 0, tzinfo=timezone.utc + ) + + ontology_manager.add_ontology(sample_ontology) + ontology_manager.add_ontology(ontology_with_parent) + + ontologies = ontology_manager.ontologies + assert len(ontologies) == 1 # One per ontology_id + assert ontologies[0].hash == ontology_with_parent.hash + + def test_ontologies_all_have_created_at(self, ontology_manager, sample_ontology): + """Test that all ontologies returned have created_at.""" + ontology_manager.add_ontology(sample_ontology) + + ontologies = ontology_manager.ontologies + for ontology in ontologies: + assert ontology.created_at is not None + + def test_ontologies_cache_is_updated_incrementally( + self, ontology_manager, sample_ontology + ): + """Test that cache is updated incrementally when adding ontologies.""" + # Initially empty + assert not ontology_manager.has_ontologies + assert len(ontology_manager.ontologies) == 0 + + # Add first ontology + ontology_manager.add_ontology(sample_ontology) + assert ontology_manager.has_ontologies + assert len(ontology_manager.ontologies) == 1 + assert sample_ontology.iri in ontology_manager._cached_ontologies + assert ( + ontology_manager._cached_ontologies[sample_ontology.iri] + == sample_ontology.hash + ) + + # Add second ontology with different ID + graph2 = RDFGraph() + graph2.parse( + data=""" + @prefix rdf: . + @prefix owl: . + @prefix rdfs: . + + a owl:Ontology . + """, + format="turtle", + ) + ontology2 = Ontology( + graph=graph2, + ontology_id="test2", + iri="https://example.org/test2", + ) + if not ontology2.hash: + ontology2._compute_and_set_hash() + + ontology_manager.add_ontology(ontology2) + assert len(ontology_manager.ontologies) == 2 + assert sample_ontology.iri in ontology_manager._cached_ontologies + assert ontology2.iri in ontology_manager._cached_ontologies + assert ontology_manager._cached_ontologies[ontology2.iri] == ontology2.hash + + def test_ontologies_cache_updates_when_new_version_added( + self, ontology_manager, sample_ontology, ontology_with_parent + ): + """Test that cache is updated when a new version is added for existing ontology_id.""" + # Add initial ontology + sample_ontology.created_at = datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc) + ontology_manager.add_ontology(sample_ontology) + + # Check cache has initial hash + assert ( + ontology_manager._cached_ontologies[sample_ontology.iri] + == sample_ontology.hash + ) + + # Add newer version (same IRI) + ontology_with_parent.created_at = datetime( + 2024, 1, 2, 12, 0, 0, tzinfo=timezone.utc + ) + ontology_manager.add_ontology(ontology_with_parent) + + # Cache should be updated to newer hash + assert ( + ontology_manager._cached_ontologies[sample_ontology.iri] + == ontology_with_parent.hash + ) + assert ( + ontology_manager._cached_ontologies[sample_ontology.iri] + != sample_ontology.hash + ) + + +class TestHasOntologies: + """Test the has_ontologies property.""" + + def test_has_ontologies_false_when_empty(self, ontology_manager): + """Test that has_ontologies returns False when no ontologies.""" + assert not ontology_manager.has_ontologies + + def test_has_ontologies_true_when_ontologies_exist( + self, ontology_manager, sample_ontology + ): + """Test that has_ontologies returns True when ontologies exist.""" + ontology_manager.add_ontology(sample_ontology) + assert ontology_manager.has_ontologies + + def test_has_ontologies_works_with_cache(self, ontology_manager, sample_ontology): + """Test that has_ontologies works correctly with caching.""" + # Initially false + assert not ontology_manager.has_ontologies + + # Add ontology + ontology_manager.add_ontology(sample_ontology) + assert ontology_manager.has_ontologies + + # Should still be true after accessing ontologies property + _ = ontology_manager.ontologies + assert ontology_manager.has_ontologies + + +class TestLineageGraph: + """Test lineage graph building.""" + + def test_get_lineage_graph_creates_graph( + self, ontology_manager, sample_ontology, ontology_with_parent + ): + """Test that lineage graph is created correctly.""" + ontology_manager.add_ontology(sample_ontology) + ontology_manager.add_ontology(ontology_with_parent) + + lineage = ontology_manager.get_lineage_graph("test") + assert lineage is not None + import networkx as nx + + assert isinstance(lineage, nx.DiGraph) + + # Check nodes exist + assert sample_ontology.hash in lineage.nodes() + assert ontology_with_parent.hash in lineage.nodes() + + # Check edge from child to parent + assert lineage.has_edge(ontology_with_parent.hash, sample_ontology.hash) + + def test_get_lineage_graph_returns_none_for_missing_id(self, ontology_manager): + """Test that lineage graph returns None for missing ontology_id.""" + lineage = ontology_manager.get_lineage_graph("nonexistent") + assert lineage is None + + +class TestGetOntology: + """Test get_ontology method.""" + + def test_get_ontology_by_hash(self, ontology_manager, sample_ontology): + """Test getting ontology by hash.""" + ontology_manager.add_ontology(sample_ontology) + + retrieved = ontology_manager.get_ontology(hash=sample_ontology.hash) + assert retrieved.hash == sample_ontology.hash + assert retrieved.created_at is not None + + def test_get_ontology_by_ontology_id_returns_terminal( + self, ontology_manager, sample_ontology, ontology_with_parent + ): + """Test that getting by ontology_id returns terminal.""" + ontology_manager.add_ontology(sample_ontology) + ontology_manager.add_ontology(ontology_with_parent) + + retrieved = ontology_manager.get_ontology(ontology_id="test") + assert retrieved.hash == ontology_with_parent.hash + assert retrieved.created_at is not None + + def test_get_ontology_by_iri(self, ontology_manager, sample_ontology): + """Test getting ontology by IRI.""" + ontology_manager.add_ontology(sample_ontology) + + retrieved = ontology_manager.get_ontology(ontology_iri=sample_ontology.iri) + assert retrieved.iri == sample_ontology.iri + assert retrieved.created_at is not None + + +class TestGetOntologyNames: + """Test get_ontology_names method.""" + + def test_get_ontology_names_returns_all_ids( + self, ontology_manager, sample_ontology + ): + """Test that get_ontology_names returns all ontology IDs.""" + ontology_manager.add_ontology(sample_ontology) + + # Create second ontology + graph2 = RDFGraph() + graph2.parse( + data=""" + @prefix rdf: . + @prefix owl: . + @prefix rdfs: . + + a owl:Ontology . + """, + format="turtle", + ) + ontology2 = Ontology( + graph=graph2, + ontology_id="test2", + iri="https://example.org/test2", + ) + if not ontology2.hash: + ontology2._compute_and_set_hash() + ontology_manager.add_ontology(ontology2) + + names = ontology_manager.get_ontology_names() + assert "test" in names + assert "test2" in names + assert len(names) == 2 + + +class TestContains: + """Test __contains__ method.""" + + def test_contains_by_ontology_id(self, ontology_manager, sample_ontology): + """Test checking containment by ontology_id.""" + ontology_manager.add_ontology(sample_ontology) + + assert "test" in ontology_manager + assert "nonexistent" not in ontology_manager + + def test_contains_by_iri(self, ontology_manager, sample_ontology): + """Test checking containment by IRI.""" + ontology_manager.add_ontology(sample_ontology) + + assert sample_ontology.iri in ontology_manager + assert "https://example.org/nonexistent" not in ontology_manager + + +class TestRecreateFromRDFGraph: + """Test recreating Ontology from RDF graph with parent_hashes and created_at.""" + + def test_recreate_ontology_with_parent_hashes_and_created_at(self): + """Test that parent_hashes and created_at are correctly read from RDF graph.""" + # Create an ontology with parent_hashes and created_at + original_ontology = Ontology( + graph=RDFGraph(), + ontology_id="test", + iri="https://example.org/test", + title="Test Ontology", + version="1.0.0", + ) + if not original_ontology.hash: + original_ontology._compute_and_set_hash() + + # Set parent_hashes and created_at + parent_hash = "parent1234567890abcdef" + original_ontology.parent_hashes = [parent_hash] + original_ontology.created_at = datetime( + 2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc + ) + + # Sync to graph to add triples + original_ontology.sync_properties_to_graph() + + # Now recreate ontology from the graph + # This simulates loading from a triple store or file + recreated_ontology = Ontology(graph=original_ontology.graph) + + # Verify parent_hashes was read correctly + assert len(recreated_ontology.parent_hashes) == 1 + assert parent_hash in recreated_ontology.parent_hashes + + # Verify created_at was read correctly + assert recreated_ontology.created_at is not None + assert recreated_ontology.created_at == datetime( + 2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc + ) + + def test_recreate_ontology_with_multiple_parent_hashes(self): + """Test that multiple parent_hashes are correctly read from RDF graph.""" + # Create an ontology with multiple parent_hashes + original_ontology = Ontology( + graph=RDFGraph(), + ontology_id="test", + iri="https://example.org/test", + title="Test Ontology", + version="1.0.0", + ) + if not original_ontology.hash: + original_ontology._compute_and_set_hash() + + # Set multiple parent_hashes (simulating a merge) + parent_hashes = ["parent1", "parent2", "parent3"] + original_ontology.parent_hashes = parent_hashes + original_ontology.created_at = datetime( + 2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc + ) + + # Sync to graph + original_ontology.sync_properties_to_graph() + + # Recreate from graph + recreated_ontology = Ontology(graph=original_ontology.graph) + + # Verify all parent_hashes were read + assert len(recreated_ontology.parent_hashes) == 3 + assert set(recreated_ontology.parent_hashes) == set(parent_hashes) + + def test_recreate_ontology_with_empty_parent_hashes(self): + """Test that empty parent_hashes (root ontology) is correctly handled.""" + # Create a root ontology (no parents) + original_ontology = Ontology( + graph=RDFGraph(), + ontology_id="test", + iri="https://example.org/test", + title="Test Ontology", + version="1.0.0", + ) + if not original_ontology.hash: + original_ontology._compute_and_set_hash() + + # Ensure parent_hashes is empty (root ontology) + original_ontology.parent_hashes = [] + original_ontology.created_at = datetime( + 2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc + ) + + # Sync to graph + original_ontology.sync_properties_to_graph() + + # Recreate from graph + recreated_ontology = Ontology(graph=original_ontology.graph) + + # Verify parent_hashes is empty + assert recreated_ontology.parent_hashes == [] + assert len(recreated_ontology.parent_hashes) == 0 + + def test_recreate_ontology_preserves_existing_created_at(self): + """Test that existing created_at is preserved when recreating from graph.""" + # Create ontology with created_at + original_ontology = Ontology( + graph=RDFGraph(), + ontology_id="test", + iri="https://example.org/test", + title="Test Ontology", + version="1.0.0", + ) + if not original_ontology.hash: + original_ontology._compute_and_set_hash() + + original_time = datetime(2023, 12, 25, 15, 45, 0, tzinfo=timezone.utc) + original_ontology.created_at = original_time + + # Sync to graph + original_ontology.sync_properties_to_graph() + + # Recreate from graph + recreated_ontology = Ontology(graph=original_ontology.graph) + + # Verify created_at was preserved + assert recreated_ontology.created_at is not None + assert recreated_ontology.created_at == original_time diff --git a/ontology_platform/vendored/ontocast/test/test_pipeline.py b/ontology_platform/vendored/ontocast/test/test_pipeline.py new file mode 100644 index 0000000..48c526e --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/test_pipeline.py @@ -0,0 +1,711 @@ +import importlib +from types import SimpleNamespace +from typing import cast + +import pytest +from rdflib import OWL, RDF, BNode, Literal, URIRef + +from ontocast.agent.normalize_ontology import normalize_ontology_units +from ontocast.onto.constants import ONTOLOGY_NULL_IRI, PROV, RDF_REIFIES, SCHEMA +from ontocast.onto.content_unit import ContentUnit, OutputType +from ontocast.onto.enum import RenderMode, Status, WorkflowNode +from ontocast.onto.model import ( + ExternalEvidenceCacheEntry, + ExternalEvidencePlan, + ExternalEvidenceRequest, + GraphUpdateRenderReport, + OntologyCritiqueReport, +) +from ontocast.onto.ontology import Ontology +from ontocast.onto.rdfgraph import RDFGraph +from ontocast.onto.sparql_models import GenericSparqlQuery, GraphUpdate +from ontocast.onto.state import AgentState +from ontocast.onto.unit_states import UnitFactsState, UnitOntologyState +from ontocast.stategraph.node_factories import make_normalize_ontology_node +from ontocast.stategraph.routing import route_after_ontology_consolidation +from ontocast.tool.aggregate import EmbeddingBasedAggregator +from ontocast.tool.atomic import AtomicToolBox, SearchHit +from ontocast.toolbox import ToolBox + +render_ontology_module = importlib.import_module("ontocast.agent.render_ontology") +criticise_ontology_module = importlib.import_module("ontocast.agent.criticise_ontology") +select_ontology_module = importlib.import_module("ontocast.agent.select_ontology") +unit_loops = importlib.import_module("ontocast.stategraph.atomic") +external_evidence_module = importlib.import_module("ontocast.agent.external_evidence") + + +def _build_content_unit() -> ContentUnit: + return ContentUnit( + text="Alice works for ACME.", + index=0, + doc_iri=URIRef("https://example.com/doc/d1"), + ) + + +def _build_ontology() -> Ontology: + graph = RDFGraph() + graph.parse( + data=""" + @prefix onto: . + @prefix owl: . + onto:CompanyOntology a owl:Ontology . + """, + format="turtle", + ) + return Ontology(graph=graph, iri="https://example.com/onto") + + +def test_unit_facts_loop_isolates_input_state() -> None: + """Unit loop uses model_copy(deep=True), so input state is not mutated.""" + state = UnitFactsState( + content_unit=_build_content_unit(), ontology_snapshot=_build_ontology() + ) + original_text = state.content_unit.text + # Simulate what the loop does: it copies before processing + copied = state.model_copy(deep=True) + copied.content_unit.text = "MUTATED" + assert state.content_unit.text == original_text + + +@pytest.mark.anyio +async def test_run_unit_facts_loop_uses_dedicated_state(monkeypatch) -> None: + async def fake_render(state: UnitFactsState, tools) -> UnitFactsState: + state.status = Status.SUCCESS + return state + + async def fake_critic(state: UnitFactsState, tools) -> UnitFactsState: + state.status = Status.SUCCESS + return state + + monkeypatch.setattr(unit_loops, "render_facts", fake_render) + monkeypatch.setattr(unit_loops, "criticise_facts", fake_critic) + + state = UnitFactsState( + content_unit=_build_content_unit(), ontology_snapshot=_build_ontology() + ) + tools = cast(AtomicToolBox, object()) + result = await unit_loops.facts_loop(state, tools=tools) + + assert result.status == Status.SUCCESS + assert result.content_unit.hid == state.content_unit.hid + + +@pytest.mark.anyio +async def test_run_unit_ontology_loop_emits_updates(monkeypatch) -> None: + async def fake_render(state: UnitOntologyState, tools) -> UnitOntologyState: + state.status = Status.SUCCESS + state.ontology_updates = [GraphUpdate()] + state.current_ontology = Ontology( + graph=RDFGraph(), iri="https://example.com/onto" + ) + return state + + async def fake_critic(state: UnitOntologyState, tools) -> UnitOntologyState: + state.status = Status.SUCCESS + return state + + monkeypatch.setattr(unit_loops, "render_ontology", fake_render) + monkeypatch.setattr(unit_loops, "criticise_ontology", fake_critic) + + state = UnitOntologyState( + content_unit=_build_content_unit(), + ontology_snapshot=Ontology(iri=ONTOLOGY_NULL_IRI), + ) + tools = cast(AtomicToolBox, object()) + result = await unit_loops.ontology_loop(state, tools=tools) + + assert result.status == Status.SUCCESS + assert len(result.all_updates) == 1 + + +def test_reduce_ontology_units_returns_ontology_when_no_units() -> None: + tools = ToolBox.__new__(ToolBox) + tools.aggregator = EmbeddingBasedAggregator() + reduced, applied, provenance = normalize_ontology_units(units=[], tools=tools) + + assert reduced is not None + assert reduced.iri is not None + assert applied == [] + assert len(provenance) == 0 + + +def test_reduce_ontology_units_merges_unit_graphs_without_aggregator() -> None: + tools = ToolBox.__new__(ToolBox) + tools.aggregator = EmbeddingBasedAggregator() + unit1 = ContentUnit( + text="Alice works at ACME", + index=0, + doc_iri=URIRef("https://example.com/doc/d1"), + graph=_build_ontology().graph, + type=OutputType.ONTOLOGIES, + ) + reduced, applied, provenance = normalize_ontology_units(units=[unit1], tools=tools) + + assert reduced is not None + assert len(reduced.graph) > 0 + assert len(applied) == 1 + assert len(applied[0].triple_operations) == 1 + assert len(provenance) == 0 + assert isinstance(applied, list) + + +def test_reduce_ontology_units_creates_base_when_required() -> None: + tools = cast(ToolBox, ToolBox.__new__(ToolBox)) + tools.aggregator = EmbeddingBasedAggregator() + delta_graph = RDFGraph() + delta_graph.parse( + data=""" + @prefix ex: . + @prefix rdf: . + @prefix rdfs: . + ex:Company rdf:type rdfs:Class . + """, + format="turtle", + ) + unit = ContentUnit( + text="Company ontology snippet", + index=0, + doc_iri=URIRef("https://example.com/doc/d1"), + graph=delta_graph, + type=OutputType.ONTOLOGIES, + ) + reduced, applied, provenance = normalize_ontology_units( + units=[unit], + tools=tools, + base_ontology=None, + require_base=True, + ) + + assert not reduced.is_null() + assert len(reduced.graph) > 0 + assert len(provenance) == 0 + assert isinstance(applied, list) + + +def test_reduce_ontology_units_strips_provenance_and_stores_artifact() -> None: + tools = ToolBox.__new__(ToolBox) + tools.aggregator = EmbeddingBasedAggregator() + doc_iri = URIRef("https://growgraph.dev/doc/test") + court = URIRef("https://growgraph.dev/fcaont#Court") + appeal_court = URIRef("https://growgraph.dev/fcaont#AppealCourt") + reifier = BNode() + source_chunk = URIRef(f"{doc_iri}/chunk-1") + + graph = RDFGraph(store="oxigraph") + graph.add((appeal_court, RDF.type, court)) + graph.add((appeal_court, OWL.sameAs, court)) + graph.add((source_chunk, RDF.type, PROV.Entity)) + graph.add((source_chunk, SCHEMA.identifier, Literal("chunk-1"))) + graph.add((reifier, RDF_REIFIES, Literal("quoted-triple"))) + graph.add((reifier, PROV.wasDerivedFrom, source_chunk)) + + unit = ContentUnit( + text="Appeal court ontology unit", + index=0, + doc_iri=doc_iri, + graph=graph, + type=OutputType.ONTOLOGIES, + ) + reduced, _, provenance = normalize_ontology_units(units=[unit], tools=tools) + + assert (appeal_court, RDF.type, court) in reduced.graph + assert (appeal_court, OWL.sameAs, court) not in reduced.graph + assert (source_chunk, SCHEMA.identifier, Literal("chunk-1")) not in reduced.graph + + assert (appeal_court, OWL.sameAs, court) in provenance + assert list(provenance.triples((None, RDF_REIFIES, None))) + assert list(provenance.triples((None, PROV.wasDerivedFrom, source_chunk))) + + +def test_normalize_ontology_node_feeds_clean_graph_to_consolidation() -> None: + class DummyTools: + aggregator = EmbeddingBasedAggregator() + + normalize_node = make_normalize_ontology_node(cast(ToolBox, DummyTools())) + + doc_iri = URIRef("https://growgraph.dev/doc/test-node") + class_uri = URIRef("https://growgraph.dev/fcaont#Judgement") + source_chunk = URIRef(f"{doc_iri}/chunk-1") + graph = RDFGraph() + graph.add( + (class_uri, RDF.type, URIRef("http://www.w3.org/2000/01/rdf-schema#Class")) + ) + graph.add((source_chunk, RDF.type, PROV.Entity)) + graph.add((source_chunk, SCHEMA.identifier, Literal("chunk-1"))) + graph.add((class_uri, OWL.sameAs, URIRef("https://growgraph.dev/fcaont#Judgment"))) + + state = AgentState(render_mode=RenderMode.ONTOLOGY) + state.current_ontology = _build_ontology() + state.ontology_units = [ + ContentUnit( + text="Ontology delta", + index=0, + doc_iri=doc_iri, + graph=graph, + type=OutputType.ONTOLOGIES, + ) + ] + + updated = normalize_node(state) + ontology_ttl = updated.current_ontology.graph.serialize(format="turtle") + + assert "rdf:reifies" not in ontology_ttl + assert f"{doc_iri}/chunk-1" not in ontology_ttl + assert "owl:sameAs" not in ontology_ttl + assert len(updated.ontology_provenance_artifact) > 0 + + +@pytest.mark.anyio +async def test_select_ontology_none_keeps_success_status(monkeypatch) -> None: + class SelectorResult: + answer_index = 0 + + async def fake_call_llm_with_retry(**kwargs): + return SelectorResult() + + monkeypatch.setattr( + select_ontology_module, "call_llm_with_retry", fake_call_llm_with_retry + ) + + state = AgentState() + state.content_units = [_build_content_unit()] + tools = SimpleNamespace( + llm=object(), + ontology_manager=SimpleNamespace( + has_ontologies=True, ontologies=[_build_ontology()] + ), + ) + result = await select_ontology_module.select_ontology(state, tools) # type: ignore[arg-type] + + assert result.status == Status.SUCCESS + assert result.current_ontology.is_null() + + +@pytest.mark.anyio +async def test_render_ontology_uses_update_when_snapshot_exists(monkeypatch) -> None: + calls = {"fresh": 0, "update": 0} + + async def fake_fresh(state: UnitOntologyState, tools) -> UnitOntologyState: + calls["fresh"] += 1 + return state + + async def fake_update(state: UnitOntologyState, tools) -> UnitOntologyState: + calls["update"] += 1 + return state + + monkeypatch.setattr(render_ontology_module, "render_ontology_fresh", fake_fresh) + monkeypatch.setattr(render_ontology_module, "render_ontology_update", fake_update) + + state = UnitOntologyState( + content_unit=_build_content_unit(), + ontology_snapshot=_build_ontology(), + ) + # Simulate accidental null current ontology while a valid snapshot exists. + state.current_ontology = Ontology(iri=ONTOLOGY_NULL_IRI) + result = await render_ontology_module.render_ontology( + state, tools=cast(AtomicToolBox, object()) + ) + + assert result is state + assert calls["update"] == 1 + assert calls["fresh"] == 0 + + +@pytest.mark.anyio +async def test_render_ontology_update_adds_external_evidence_when_enabled( + monkeypatch, +) -> None: + captured_prompt_kwargs: dict[str, object] = {} + + async def fake_call_llm_with_retry(**kwargs): + captured_prompt_kwargs.update(kwargs["prompt_kwargs"]) + return GraphUpdateRenderReport(graph_update=GraphUpdate()) + + async def fake_get_llm_tool(_budget_tracker): + return object() + + monkeypatch.setattr( + render_ontology_module, "call_llm_with_retry", fake_call_llm_with_retry + ) + tools = cast( + AtomicToolBox, + SimpleNamespace( + get_llm_tool=fake_get_llm_tool, + ), + ) + state = UnitOntologyState( + content_unit=_build_content_unit(), + ontology_snapshot=_build_ontology(), + ) + state.external_evidence_text = ( + "### EXTERNAL EVIDENCE (WEB SEARCH)\n" + "1. Ontology engineering patterns | https://example.org/ontology\n" + " Use consistent subclass hierarchies and explicit domains." + ) + + await render_ontology_module.render_ontology_update(state, tools=tools) + + external_evidence = str(captured_prompt_kwargs.get("external_evidence", "")) + assert "EXTERNAL EVIDENCE" in external_evidence + assert "https://example.org/ontology" in external_evidence + + +@pytest.mark.anyio +async def test_criticise_ontology_skips_external_evidence_when_disabled( + monkeypatch, +) -> None: + captured_prompt_kwargs: dict[str, object] = {} + + async def fake_call_llm_with_retry(**kwargs): + captured_prompt_kwargs.update(kwargs["prompt_kwargs"]) + return OntologyCritiqueReport( + success=True, + score=95, + systemic_critique_summary="Looks good.", + actionable_ontology_fixes=[], + ) + + async def fake_get_llm_tool(_budget_tracker): + return object() + + monkeypatch.setattr( + criticise_ontology_module, "call_llm_with_retry", fake_call_llm_with_retry + ) + tools = cast( + AtomicToolBox, + SimpleNamespace( + get_llm_tool=fake_get_llm_tool, + ), + ) + state = UnitOntologyState( + content_unit=_build_content_unit(), + ontology_snapshot=_build_ontology(), + ) + + await criticise_ontology_module.criticise_ontology(state, tools=tools) + + assert captured_prompt_kwargs.get("external_evidence") == "" + + +@pytest.mark.anyio +async def test_plan_external_evidence_uses_fallback_when_planner_disabled() -> None: + tools = cast( + AtomicToolBox, + SimpleNamespace( + web_grounding_enabled_for_node=lambda _node: True, + web_search_reuse_evidence_across_attempt=False, + web_search_planner_enabled=False, + web_search_planner_min_query_chars=8, + web_search_planner_max_queries=3, + web_search_planner_min_confidence=0.35, + ), + ) + state = UnitOntologyState( + content_unit=_build_content_unit(), + ontology_snapshot=_build_ontology(), + ontology_user_instruction="Clarify company ontology terms.", + ) + state.set_external_evidence_request( + WorkflowNode.TEXT_TO_ONTOLOGY, + ExternalEvidenceRequest( + initiate_search=True, + rationale="Need targeted terminology lookup for ontology refinement.", + ), + ) + + planned = await external_evidence_module.plan_external_evidence_for_node( + state, tools, WorkflowNode.TEXT_TO_ONTOLOGY + ) + + assert planned.external_evidence_plan.should_search is True + assert planned.external_evidence_plan.queries + assert planned.external_evidence_planned_at_node == WorkflowNode.TEXT_TO_ONTOLOGY + + +@pytest.mark.anyio +async def test_fetch_external_evidence_filters_domains_and_dedupes() -> None: + async def fake_search(query: str, max_results: int | None = None): + _ = query, max_results + return [ + SearchHit( + title="Good result", + url="https://example.org/ontology", + snippet="This is a sufficiently detailed snippet for ontology guidance.", + ), + SearchHit( + title="Duplicate URL", + url="https://example.org/ontology", + snippet="Different text but same URL should be deduped.", + ), + SearchHit( + title="Other domain", + url="https://noise.test/entry", + snippet="This snippet is long enough but should be filtered by allowlist.", + ), + ] + + tools = cast( + AtomicToolBox, + SimpleNamespace( + web_grounding_enabled_for_node=lambda _node: True, + search=fake_search, + web_search_allowed_domains={"example.org"}, + web_search_blocked_domains=set(), + web_search_min_snippet_chars=20, + web_search_max_snippet_chars=180, + web_search_max_total_chars=1200, + ), + ) + state = UnitOntologyState( + content_unit=_build_content_unit(), + ontology_snapshot=_build_ontology(), + ) + state.set_external_evidence_request( + WorkflowNode.TEXT_TO_ONTOLOGY, + ExternalEvidenceRequest( + initiate_search=True, + rationale="Need clarification", + query_hints=["ontology engineering patterns"], + confidence=0.9, + ), + ) + state.set_external_evidence_cache_entry( + WorkflowNode.TEXT_TO_ONTOLOGY, + ExternalEvidenceCacheEntry( + plan=ExternalEvidencePlan( + should_search=True, + rationale="Need clarification", + intent="definition", + confidence=0.9, + queries=["ontology engineering patterns"], + ), + ), + ) + + fetched = await external_evidence_module.fetch_external_evidence_for_node( + state, tools, WorkflowNode.TEXT_TO_ONTOLOGY + ) + + assert fetched.external_evidence_source_count == 1 + assert fetched.external_evidence_domains == ["example.org"] + assert "https://example.org/ontology" in fetched.external_evidence_text + + +@pytest.mark.anyio +async def test_ontology_loop_runs_external_evidence_nodes(monkeypatch) -> None: + called_nodes: list[WorkflowNode] = [] + + async def fake_plan(state: UnitOntologyState, tools, target_node: WorkflowNode): + _ = tools + called_nodes.append(target_node) + return state + + async def fake_fetch(state: UnitOntologyState, tools, target_node: WorkflowNode): + _ = tools, target_node + return state + + async def fake_render(state: UnitOntologyState, tools) -> UnitOntologyState: + _ = tools + state.status = Status.SUCCESS + return state + + async def fake_critic(state: UnitOntologyState, tools) -> UnitOntologyState: + _ = tools + state.status = Status.SUCCESS + return state + + monkeypatch.setattr(unit_loops, "plan_external_evidence_for_node", fake_plan) + monkeypatch.setattr(unit_loops, "fetch_external_evidence_for_node", fake_fetch) + monkeypatch.setattr(unit_loops, "render_ontology", fake_render) + monkeypatch.setattr(unit_loops, "criticise_ontology", fake_critic) + + state = UnitOntologyState( + content_unit=_build_content_unit(), + ontology_snapshot=Ontology(iri=ONTOLOGY_NULL_IRI), + ) + tools = cast(AtomicToolBox, object()) + result = await unit_loops.ontology_loop(state, tools=tools) + + assert result.status == Status.SUCCESS + assert called_nodes == [] + + +@pytest.mark.anyio +async def test_ontology_loop_plans_search_when_critic_requests_it(monkeypatch) -> None: + called_nodes: list[WorkflowNode] = [] + + async def fake_plan(state: UnitOntologyState, tools, target_node: WorkflowNode): + _ = tools + called_nodes.append(target_node) + return state + + async def fake_fetch(state: UnitOntologyState, tools, target_node: WorkflowNode): + _ = tools + called_nodes.append(target_node) + return state + + async def fake_render(state: UnitOntologyState, tools) -> UnitOntologyState: + _ = tools + state.status = Status.SUCCESS + return state + + critic_calls = {"count": 0} + + async def fake_critic(state: UnitOntologyState, tools) -> UnitOntologyState: + _ = tools + critic_calls["count"] += 1 + if critic_calls["count"] == 1: + state.status = Status.FAILED + state.set_external_evidence_request( + WorkflowNode.CRITICISE_ONTOLOGY, + ExternalEvidenceRequest( + initiate_search=True, + rationale="Need domain standard disambiguation.", + query_hints=["ontology modeling standard pattern"], + ), + ) + return state + state.status = Status.SUCCESS + return state + + monkeypatch.setattr(unit_loops, "plan_external_evidence_for_node", fake_plan) + monkeypatch.setattr(unit_loops, "fetch_external_evidence_for_node", fake_fetch) + monkeypatch.setattr(unit_loops, "render_ontology", fake_render) + monkeypatch.setattr(unit_loops, "criticise_ontology", fake_critic) + + state = UnitOntologyState( + content_unit=_build_content_unit(), + ontology_snapshot=Ontology(iri=ONTOLOGY_NULL_IRI), + ) + tools = cast(AtomicToolBox, object()) + result = await unit_loops.ontology_loop(state, tools=tools) + + assert result.status == Status.SUCCESS + assert called_nodes == [ + WorkflowNode.CRITICISE_ONTOLOGY, + WorkflowNode.CRITICISE_ONTOLOGY, + ] + + +def test_agent_state_render_mode_properties() -> None: + facts_only = AgentState(render_mode=RenderMode.FACTS) + assert facts_only.render_mode == RenderMode.FACTS + assert facts_only.render_facts is True + assert facts_only.render_ontology is False + + ontology_only = AgentState(render_mode=RenderMode.ONTOLOGY) + assert ontology_only.render_mode == RenderMode.ONTOLOGY + assert ontology_only.render_facts is False + assert ontology_only.render_ontology is True + + both = AgentState(render_mode=RenderMode.ONTOLOGY_AND_FACTS) + assert both.render_mode == RenderMode.ONTOLOGY_AND_FACTS + assert both.render_facts is True + assert both.render_ontology is True + + +def test_route_after_ontology_consolidation_respects_ontology_only_mode() -> None: + ontology_only = AgentState(render_mode=RenderMode.ONTOLOGY) + assert route_after_ontology_consolidation(ontology_only) == WorkflowNode.SERIALIZE + + ontology_and_facts = AgentState(render_mode=RenderMode.ONTOLOGY_AND_FACTS) + assert ( + route_after_ontology_consolidation(ontology_and_facts) + == WorkflowNode.RENDER_FACTS + ) + + +def test_toolbox_serialize_skips_facts_in_ontology_only_mode() -> None: + class RecordingOntologyManager: + def __init__(self) -> None: + self.added = 0 + + def add_ontology(self, ontology: Ontology) -> None: + self.added += 1 + + class RecordingStore: + def __init__(self) -> None: + self.calls: list[tuple[object, str | None]] = [] + + def serialize(self, payload: object, graph_uri: str | None = None) -> None: + self.calls.append((payload, graph_uri)) + + state = AgentState(render_mode=RenderMode.ONTOLOGY) + state.current_ontology = _build_ontology() + store = RecordingStore() + toolbox = SimpleNamespace( + ontology_manager=RecordingOntologyManager(), + filesystem_manager=store, + triple_store_manager=None, + ) + + ToolBox.serialize(cast(ToolBox, toolbox), state) + + assert len(store.calls) == 1 + assert isinstance(store.calls[0][0], Ontology) + assert store.calls[0][1] is None + + +def test_toolbox_serialize_includes_facts_when_render_facts_enabled() -> None: + class RecordingOntologyManager: + def add_ontology(self, ontology: Ontology) -> None: + return None + + class RecordingStore: + def __init__(self) -> None: + self.calls: list[tuple[object, str | None]] = [] + + def serialize(self, payload: object, graph_uri: str | None = None) -> None: + self.calls.append((payload, graph_uri)) + + state = AgentState(render_mode=RenderMode.ONTOLOGY_AND_FACTS) + state.current_ontology = _build_ontology() + store = RecordingStore() + toolbox = SimpleNamespace( + ontology_manager=RecordingOntologyManager(), + filesystem_manager=store, + triple_store_manager=None, + ) + + ToolBox.serialize(cast(ToolBox, toolbox), state) + + assert len(store.calls) == 2 + assert isinstance(store.calls[0][0], Ontology) + assert isinstance(store.calls[1][0], RDFGraph) + assert store.calls[1][1] == state.graph_uri + + +def test_render_updated_graph_splits_compound_sparql_insert_updates() -> None: + graph = RDFGraph() + graph.parse( + data=""" + @prefix ex: . + ex:Existing ex:kept ex:Value . + """, + format="turtle", + ) + update = GraphUpdate( + sparql_operations=[ + GenericSparqlQuery( + query=( + "PREFIX ex: \n" + "INSERT DATA { ex:Person ex:label ex:Alice }\n" + "INSERT DATA { ex:Person ex:status ex:Active }" + ) + ) + ] + ) + + updated_graph, was_applied = AgentState.render_updated_graph(graph, [update]) + + assert was_applied is True + assert ( + URIRef("http://example.org/Person"), + URIRef("http://example.org/label"), + URIRef("http://example.org/Alice"), + ) in updated_graph + assert ( + URIRef("http://example.org/Person"), + URIRef("http://example.org/status"), + URIRef("http://example.org/Active"), + ) in updated_graph diff --git a/ontology_platform/vendored/ontocast/test/test_rdfgraph_iadd.py b/ontology_platform/vendored/ontocast/test/test_rdfgraph_iadd.py new file mode 100644 index 0000000..4b354b7 --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/test_rdfgraph_iadd.py @@ -0,0 +1,148 @@ +"""Test for RDFGraph __iadd__ method. + +This test verifies that the __iadd__ method properly reuses __add__ and binds prefixes. +""" + +from rdflib import Graph, Literal, Namespace, URIRef + +from ontocast.onto.rdfgraph import RDFGraph + + +def test_rdfgraph_iadd_reuses_add_and_binds_prefixes(): + """Test that __iadd__ reuses __add__ and properly binds prefixes.""" + + # Create two RDFGraph instances with different namespaces + graph1 = RDFGraph() + graph2 = RDFGraph() + + # Define namespacesЙ + ns1 = Namespace("http://example.org/ns1/") + ns2 = Namespace("http://example.org/ns2/") + + # Add some triples to graph1 with ns1 namespace + graph1.add((ns1.subject1, ns1.predicate1, Literal("value1"))) + graph1.add((ns1.subject2, ns1.predicate2, Literal("value2"))) + graph1.bind("ns1", ns1) + + # Add some triples to graph2 with ns2 namespace + graph2.add((ns2.subject1, ns2.predicate1, Literal("value3"))) + graph2.add((ns2.subject2, ns2.predicate2, Literal("value4"))) + graph2.bind("ns2", ns2) + + # Test __iadd__ method + graph1 += graph2 + + # Verify that all triples are present + assert len(graph1) == 4 + assert (ns1.subject1, ns1.predicate1, Literal("value1")) in graph1 + assert (ns1.subject2, ns1.predicate2, Literal("value2")) in graph1 + assert (ns2.subject1, ns2.predicate1, Literal("value3")) in graph1 + assert (ns2.subject2, ns2.predicate2, Literal("value4")) in graph1 + + # Verify that namespace bindings are preserved + namespaces = dict(graph1.namespaces()) + assert "ns1" in namespaces + assert "ns2" in namespaces + assert str(namespaces["ns1"]) == "http://example.org/ns1/" + assert str(namespaces["ns2"]) == "http://example.org/ns2/" + + +def test_rdfgraph_iadd_with_regular_graph(): + """Test that __iadd__ works with regular rdflib.Graph objects.""" + + # Create RDFGraph and regular Graph + rdf_graph = RDFGraph() + regular_graph = Graph() + + # Define namespace + ns = Namespace("http://example.org/test/") + + # Add triples to both graphs + rdf_graph.add((ns.subject1, ns.predicate1, Literal("value1"))) + rdf_graph.bind("test", ns) + + regular_graph.add((ns.subject2, ns.predicate2, Literal("value2"))) + + # Test __iadd__ method + rdf_graph += regular_graph + + # Verify that all triples are present + assert len(rdf_graph) == 2 + assert (ns.subject1, ns.predicate1, Literal("value1")) in rdf_graph + assert (ns.subject2, ns.predicate2, Literal("value2")) in rdf_graph + + # Verify that namespace binding is preserved + namespaces = dict(rdf_graph.namespaces()) + assert "test" in namespaces + assert str(namespaces["test"]) == "http://example.org/test/" + + +def test_rdfgraph_iadd_returns_self(): + """Test that __iadd__ returns self for chaining.""" + + graph1 = RDFGraph() + graph2 = RDFGraph() + + # Add some triples + graph1.add( + ( + URIRef("http://example.org/subject1"), + URIRef("http://example.org/predicate1"), + Literal("value1"), + ) + ) + graph2.add( + ( + URIRef("http://example.org/subject2"), + URIRef("http://example.org/predicate2"), + Literal("value2"), + ) + ) + + # Test that __iadd__ returns self + result = graph1.__iadd__(graph2) + + # Verify that result is the same object as graph1 + assert result is graph1 + assert len(graph1) == 2 + + +def test_rdfgraph_iadd_equivalent_to_add(): + """Test that __iadd__ produces the same result as __add__.""" + + # Create two graphs + graph1 = RDFGraph() + graph2 = RDFGraph() + + # Define namespaces + ns1 = Namespace("http://example.org/ns1/") + ns2 = Namespace("http://example.org/ns2/") + + # Add triples and bind namespaces + graph1.add((ns1.subject1, ns1.predicate1, Literal("value1"))) + graph1.bind("ns1", ns1) + + graph2.add((ns2.subject1, ns2.predicate1, Literal("value2"))) + graph2.bind("ns2", ns2) + + # Create copies for comparison + graph1_copy = RDFGraph() + for triple in graph1: + graph1_copy.add(triple) + for prefix, uri in graph1.namespaces(): + graph1_copy.bind(prefix, uri) + + # Test __add__ method + result_add = graph1_copy + graph2 + + # Test __iadd__ method + graph1 += graph2 + + # Verify that both methods produce the same result + assert len(graph1) == len(result_add) + assert set(graph1) == set(result_add) + + # Verify namespace bindings are the same + namespaces1 = dict(graph1.namespaces()) + namespaces_add = dict(result_add.namespaces()) + assert namespaces1 == namespaces_add diff --git a/ontology_platform/vendored/ontocast/test/test_semantic_chunker.py b/ontology_platform/vendored/ontocast/test/test_semantic_chunker.py new file mode 100644 index 0000000..1a6f3ce --- /dev/null +++ b/ontology_platform/vendored/ontocast/test/test_semantic_chunker.py @@ -0,0 +1,185 @@ +"""Test suite for SemanticChunker. + +This test suite ensures that: +1. Chunks, when joined, reproduce the original text (length and content) +2. If max_size and min_size are provided, all chunks are >= min_size and <= max_size +""" + +import json +import re +from pathlib import Path + +import pytest +from langchain_core.embeddings import Embeddings + +from ontocast.config import ChunkConfig +from ontocast.tool.chunk.util import SENTENCE_SPLIT_REGEX, SemanticChunker + + +class TestSemanticChunker: + """Core tests for SemanticChunker focusing on text reconstruction and size constraints.""" + + def test_chunks_reproduce_original_text_when_joined( + self, embeddings: Embeddings, sample_text: str + ): + """Test that chunks, when joined, reproduce the original text.""" + chunk_config = ChunkConfig( + min_size=1, # Very small min_size to allow any chunk size + max_size=100000, # Very large max_size to allow any chunk size + ) + chunker = SemanticChunker( + embeddings=embeddings, + chunk_config=chunk_config, + sentence_split_regex=SENTENCE_SPLIT_REGEX, + ) + + chunks = chunker.split_text(sample_text) + joined_text = "".join(chunks) + + # Verify length is approximately the same + length_diff = abs(len(joined_text) - len(sample_text)) + assert length_diff <= len(chunks), ( + f"Joined text length difference ({length_diff}) is too large. " + f"Original: {len(sample_text)}, Joined: {len(joined_text)}" + ) + + # Verify content is preserved (normalize whitespace for comparison) + original_normalized = re.sub(r"\s+", " ", sample_text.strip()) + joined_normalized = re.sub(r"\s+", " ", joined_text.strip()) + + # Check word coverage + original_words = set(re.findall(r"\b\w+\b", original_normalized.lower())) + joined_words = set(re.findall(r"\b\w+\b", joined_normalized.lower())) + missing_words = original_words - joined_words + coverage = ( + 1 - (len(missing_words) / len(original_words)) if original_words else 1 + ) + + assert coverage >= 0.95, ( + f"Word coverage too low: {coverage:.1%}. " + f"Missing {len(missing_words)} words: {list(missing_words)[:10]}" + ) + + def test_chunks_respect_min_and_max_size( + self, embeddings: Embeddings, long_text: str + ): + """Test that chunks respect both min_size and max_size constraints.""" + min_size = 200 + max_size = 1000 + chunk_config = ChunkConfig( + min_size=min_size, + max_size=max_size, + ) + chunker = SemanticChunker( + embeddings=embeddings, + chunk_config=chunk_config, + sentence_split_regex=SENTENCE_SPLIT_REGEX, + ) + + chunks = chunker.split_text(long_text) + + assert len(chunks) > 0, "Should produce at least one chunk" + for i, chunk in enumerate(chunks): + # All chunks must respect max_size + assert len(chunk) <= max_size, ( + f"Chunk {i} has length {len(chunk)} which exceeds max_size {max_size}" + ) + # All but last chunk should meet min_size + if i < len(chunks) - 1: + assert len(chunk) >= min_size, ( + f"Chunk {i} has length {len(chunk)} which is less than min_size {min_size}" + ) + + # Verify joined text exactly reproduces original + joined_text = "".join(chunks) + assert joined_text == long_text, ( + f"Joined text does not exactly match original text. " + f"Length difference: {abs(len(joined_text) - len(long_text))} characters. " + f"Original length: {len(long_text)}, Joined length: {len(joined_text)}. " + f"First difference at position: {next((i for i, (a, b) in enumerate(zip(long_text, joined_text)) if a != b), min(len(long_text), len(joined_text)))}" + ) + + def test_chunker_test_json_with_strict_size_constraints( + self, embeddings: Embeddings + ): + """Test with chunker.test.json using strict size constraints (min_size=2000, max_size=4000). + + This test reproduces a bug where: + 1. Chunks smaller than min_size are produced + 2. Chunks are almost exactly max_size (suggesting brute force cutting) + """ + # Load test data + json_file = Path(__file__).parent / "data" / "chunker.test.json" + if not json_file.exists(): + pytest.skip(f"Test data file not found: {json_file}") + + data = json.load(open(json_file)) + text = data.get("text", "") + if not text: + pytest.skip("No text found in test data") + + min_size = 2000 + max_size = 4000 + chunk_config = ChunkConfig( + min_size=min_size, + max_size=max_size, + ) + chunker = SemanticChunker( + embeddings=embeddings, + chunk_config=chunk_config, + sentence_split_regex=SENTENCE_SPLIT_REGEX, + ) + + chunks = chunker.split_text(text) + chunk_sizes = [len(c) for c in chunks] + + # Verify all chunks respect max_size + for i, chunk in enumerate(chunks): + assert len(chunk) <= max_size, ( + f"Chunk {i} has length {len(chunk)} which exceeds max_size {max_size}. " + f"Chunk sizes: {chunk_sizes}" + ) + + # Verify chunks meet min_size (except possibly the last one) + # All but the last chunk should meet min_size + # The last chunk may be smaller if remaining text is less than min_size + if len(chunks) > 1: + for i in range(len(chunks) - 1): + assert len(chunks[i]) >= min_size, ( + f"Chunk {i} (not last) has length {len(chunks[i])} which is less than " + f"min_size {min_size}. Chunk sizes: {chunk_sizes}" + ) + + # Even the last chunk should be reasonably sized (at least 50% of min_size) + # unless the total remaining text is very small + if len(chunks) > 0: + last_chunk_size = len(chunks[-1]) + if last_chunk_size < min_size * 0.5 and len(chunks) > 1: + # Check if this is really the last chunk or if there's a problem + total_remaining = sum(len(c) for c in chunks if len(c) < min_size) + if total_remaining >= min_size: + pytest.fail( + f"Last chunk has length {last_chunk_size} which is too small. " + f"Total size of small chunks: {total_remaining} >= {min_size}, " + f"so they should have been merged. Chunk sizes: {chunk_sizes}" + ) + + # Check for brute force cutting - chunks should not all be clustered near max_size + chunks_near_max = sum(1 for size in chunk_sizes if size >= max_size * 0.98) + ratio_near_max = chunks_near_max / len(chunks) if chunks else 0 + + # If more than 60% of chunks are near max_size, it suggests brute force cutting + assert ratio_near_max < 0.6, ( + f"Too many chunks ({chunks_near_max}/{len(chunks)} = {ratio_near_max:.1%}) " + f"are near max_size ({max_size * 0.98:.0f}), suggesting brute force cutting. " + f"Chunk sizes: {chunk_sizes}" + ) + + # Verify joined text exactly reproduces original + joined_text = "".join(chunks) + assert joined_text == text, ( + f"Joined text does not exactly match original text. " + f"Length difference: {abs(len(joined_text) - len(text))} characters. " + f"Original length: {len(text)}, Joined length: {len(joined_text)}. " + f"First difference at position: {next((i for i, (a, b) in enumerate(zip(text, joined_text)) if a != b), min(len(text), len(joined_text)))}" + ) diff --git a/start_webserver.bat b/start_webserver.bat new file mode 100644 index 0000000..bfc2734 --- /dev/null +++ b/start_webserver.bat @@ -0,0 +1,54 @@ +@echo off +setlocal + +set "SCRIPT_DIR=%~dp0" +cd /d "%SCRIPT_DIR%" + +set "HOST=127.0.0.1" +set "PORT=8000" +set "URL=http://%HOST%:%PORT%/" +set "BUNDLED_PYTHON=C:\Users\lasta\.cache\codex-runtimes\codex-primary-runtime\dependencies\python\python.exe" +set "PYTHON_EXE=" + +if exist "%SCRIPT_DIR%.venv\Scripts\python.exe" set "PYTHON_EXE=%SCRIPT_DIR%.venv\Scripts\python.exe" +if not defined PYTHON_EXE if exist "%SCRIPT_DIR%venv\Scripts\python.exe" set "PYTHON_EXE=%SCRIPT_DIR%venv\Scripts\python.exe" +if not defined PYTHON_EXE if exist "%BUNDLED_PYTHON%" set "PYTHON_EXE=%BUNDLED_PYTHON%" + +if not defined PYTHON_EXE ( + echo [OCP] Python executable not found. + echo [OCP] Put Python 3.11+ in .venv\Scripts\python.exe or install the Codex bundled runtime. + pause + exit /b 1 +) + +"%PYTHON_EXE%" -c "import fastapi, uvicorn" >nul 2>nul +if errorlevel 1 ( + echo [OCP] Missing Python dependencies. + echo [OCP] Run: "%PYTHON_EXE%" -m pip install -r requirements.txt + pause + exit /b 1 +) + +powershell -NoProfile -Command "try { $r = Invoke-RestMethod -Uri '%URL%health' -TimeoutSec 2; if ($r.ok) { exit 0 } else { exit 1 } } catch { exit 1 }" >nul 2>nul +if not errorlevel 1 ( + echo [OCP] Existing server is already running at %URL% + start "" "%URL%" + exit /b 0 +) + +set "CRAWLER_DATABASE_URL=sqlite:///crawler_platform.db" +title Ontology Crawler Web Server + +echo [OCP] Starting server with: %PYTHON_EXE% +echo [OCP] URL: %URL% +echo [OCP] Press Ctrl+C to stop the server. + +start "" "%URL%" +"%PYTHON_EXE%" -m uvicorn crawler_platform.app.main:app --host %HOST% --port %PORT% + +set "EXIT_CODE=%ERRORLEVEL%" +echo. +echo [OCP] Server stopped. +if not "%EXIT_CODE%"=="0" echo [OCP] Exit code: %EXIT_CODE% +pause +exit /b %EXIT_CODE% diff --git a/tests/test_content_quality.py b/tests/test_content_quality.py index 55424c1..4b9d19b 100644 --- a/tests/test_content_quality.py +++ b/tests/test_content_quality.py @@ -12,7 +12,9 @@ from crawler_platform.app.core.extractor.base import ( ExtractionBundle, ) from crawler_platform.app.core.extractor.ai_provider import parse_json_content +from crawler_platform.app.core.extractor.ai_provider import LLMJsonExtractor from crawler_platform.app.core.extractor.validation import attach_page_context, validate_extraction_bundle +from crawler_platform.app.domains.perfume.extractor import PerfumeRuleBasedExtractor def test_cleaner_prefers_product_content_and_removes_boilerplate(): @@ -175,7 +177,7 @@ def test_validation_rejects_product_detail_claims_on_category_page_context(): assert result.rejected_claims[0]["reason"] == "predicate hasPrice is not allowed for page type CategoryPage" -def test_repository_logs_rule_candidates_without_persisting_ontology_rows(): +def test_repository_persists_rule_candidates_without_merging_graph_relations(): config = load_project_config("configs/perfume_subscription.yaml") engine = make_engine("sqlite:///:memory:") models.Base.metadata.create_all(engine) @@ -194,23 +196,41 @@ def test_repository_logs_rule_candidates_without_persisting_ontology_rows(): "Neroli Summer", "Perfume", "hasTopNote", - "Bergamot", - "Note", - evidence_text="Top notes: Bergamot", - ) + "Bergamot", + "Note", + evidence_text="Top notes: Bergamot", + confidence=0.9, + ) ], extractor_name="perfume_rule_based", provider="rule_based", raw_output={}, ) + context = ExtractionPageContext( + url="https://example.com/product", + final_url="https://example.com/product", + title="Product", + page_type="ProductPage", + clean_text="Neroli Summer\nTop notes: Bergamot", + source_zones=[ + { + "zone_type": "product_description", + "selector": ".description", + "text": "Top notes: Bergamot", + "claim_allowed": True, + } + ], + ) - claims = repo.save_extraction_bundle(project.id, source, page, bundle, config) + claims = repo.save_extraction_bundle(project.id, source, page, attach_page_context(bundle, context), config) session.commit() - assert claims == [] - assert session.query(models.Claim).count() == 0 - assert session.query(models.Entity).count() == 0 + assert len(claims) == 1 + assert claims[0].status == "rule_candidate" + assert session.query(models.Claim).count() == 1 + assert session.query(models.Entity).count() == 2 assert session.query(models.ExtractionLog).count() == 1 + assert session.query(models.Relation).count() == 0 session.close() @@ -222,6 +242,93 @@ def test_parse_json_content_accepts_markdown_fenced_json(): assert parse_json_content(raw) == {"entities": [], "claims": []} +def test_lm_studio_extractor_merges_rule_claims_even_when_ai_claim_is_invalid(): + config = load_project_config("configs/perfume_subscription.yaml") + + class StubExtractor(LLMJsonExtractor): + def complete_json(self, page_text, project_config, compact=False, context=None): + return { + "entities": [ + { + "entity_type": "Brand", + "name": "912", + "attributes": {}, + "confidence": 0.9, + "evidence_text": "912", + } + ], + "claims": [ + { + "subject_name": "912", + "subject_type": "Brand", + "predicate": "hasBrand", + "object_name": "912", + "object_type": "Brand", + "object_value": None, + "evidence_text": "912", + "evidence_summary": "bad self-brand claim", + "confidence": 0.9, + "confidence_reason": "stub", + "source_zone": "product_title", + } + ], + } + + text = "Neroli Summer\nTop notes: Bergamot\nPrice $89" + bundle = StubExtractor("perfume", "lm_studio", model="stub").extract(text, config) + + assert any(claim.predicate == "hasTopNote" for claim in bundle.claims) + + +def test_the912_product_rule_fallback_uses_product_title_brand_and_price(): + config = load_project_config("configs/perfume_subscription.yaml") + text = """ + 새로운 향수의 시작, 클론 향수 + [2+1 기획] 912 클론 니치향수 모음 40ml + 4.8 + 114,414 + 일반 구매가격 + 72,000원 + 할인 적용금액 + 37,000원 + 최종 구매금액 + 35,000원 + 현재 위치 + 전체 상품 + 니치향수 + 오드 퍼퓸 + """ + bundle = PerfumeRuleBasedExtractor().extract(text, config) + context = ExtractionPageContext( + url="https://the912.co.kr/product/detail.html?product_no=513", + final_url="https://the912.co.kr/product/detail.html?product_no=513", + title="[2+1 기획] 912 클론 니치향수 모음 40ml - 912", + page_type="ProductPage", + clean_text=text, + source_zones=[ + { + "zone_type": "product_title", + "selector": "h1", + "text": "[2+1 기획] 912 클론 니치향수 모음 40ml", + "claim_allowed": True, + }, + { + "zone_type": "product_summary", + "selector": ".detail", + "text": "일반 구매가격\n72,000원\n할인 적용금액\n37,000원\n최종 구매금액\n35,000원", + "claim_allowed": True, + }, + ], + ) + + result = validate_extraction_bundle(attach_page_context(bundle, context), config) + predicates = {claim.predicate for claim in result.bundle.claims} + perfume_names = {entity.name for entity in result.bundle.entities if entity.entity_type == "Perfume"} + + assert "[2+1 기획] 912 클론 니치향수 모음 40ml" in perfume_names + assert {"hasBrand", "hasPrice"} <= predicates + + def test_parse_json_content_requires_structured_claim_schema(): raw = { "entities": [ diff --git a/tests/test_site_crawler.py b/tests/test_site_crawler.py index 3581750..aca6b52 100644 --- a/tests/test_site_crawler.py +++ b/tests/test_site_crawler.py @@ -4,7 +4,9 @@ from crawler_platform.app.core.crawler.site_crawler import ( normalize_url, should_analyze_page, ) -from crawler_platform.app.core.crawler.fetchers import detect_crawl_status +from crawler_platform.app.core.crawler.discovery import discover_links, normalize_cafe24_product_url +from crawler_platform.app.core.crawler.fetchers import FallbackFetcher, FetchResult, RobotsPolicy, detect_crawl_status +from crawler_platform.app.api.routes import CrawlRequest def test_classify_perfume_product_page(): @@ -39,6 +41,67 @@ def test_should_analyze_page_supports_legacy_names(): assert not should_analyze_page("CommunityPage", {"product", "brand", "review"}) +def test_robots_policy_disabled_skips_check(): + decision = RobotsPolicy().check("https://example.com/products/1", respect_robots_txt=False) + + assert decision.allowed + assert decision.status == "disabled" + assert not decision.checked + + +def test_crawl_request_defaults_to_no_robots_check(): + request = CrawlRequest( + config_path="configs/perfume_subscription.yaml", + source_name="official_brand_site", + url="https://example.com", + ) + + assert request.check_robots_txt is False + + +def test_robots_policy_allows_when_robots_unavailable(monkeypatch): + class UnavailableRobotsParser: + def set_url(self, url): + self.url = url + + def read(self): + raise OSError("network unavailable") + + monkeypatch.setattr( + "crawler_platform.app.core.crawler.fetchers.RobotFileParser", + UnavailableRobotsParser, + ) + + decision = RobotsPolicy().check("https://example.com/products/1") + + assert decision.allowed + assert decision.status == "unavailable" + assert "allowing crawl" in decision.reason + + +def test_robots_policy_reports_block_reason(monkeypatch): + class BlockingRobotsParser: + def set_url(self, url): + self.url = url + + def read(self): + return None + + def can_fetch(self, user_agent, url): + return False + + monkeypatch.setattr( + "crawler_platform.app.core.crawler.fetchers.RobotFileParser", + BlockingRobotsParser, + ) + + decision = RobotsPolicy().check("https://example.com/private") + + assert not decision.allowed + assert decision.status == "blocked" + assert "blocks crawling" in decision.reason + + def test_classify_board_page_before_content_analysis(): assert classify_page("https://example.com/board/free/read.html", "Notice", "Price $89") == "NoticePage" @@ -56,3 +119,44 @@ def test_classify_product_list_and_search_as_non_detail_pages(): assert classify_page("https://example.com/product/search.html?keyword=cotton", "Search", text) == "SearchPage" assert not should_analyze_page("CategoryPage", {"ProductPage", "BrandStoryPage", "ReviewPage"}) assert not should_analyze_page("SearchPage", {"ProductPage", "BrandStoryPage", "ReviewPage"}) + + +def test_cafe24_product_detail_with_category_segment_is_product_page(): + url = "https://the912.co.kr/product/21-기획-912-클론-니치향수-모음-40ml/513/category/1/display/2/" + + assert classify_page(url, "912 clone perfume", "") == "ProductPage" + + +def test_discovery_skips_utility_pages_seen_on_the912(): + html = """ + wish + faq + event + search + product + """ + + links = discover_links(html, "https://the912.co.kr", limit=10) + + assert [link.url for link in links] == ["https://the912.co.kr/product/detail.html?product_no=123"] + + +def test_cafe24_product_urls_are_canonicalized_for_dedupe(): + url = "https://the912.co.kr/product/21-기획-912-클론-니치향수-모음-40ml/513/category/1/display/2/?icid=x" + + assert normalize_cafe24_product_url(url) == "https://the912.co.kr/product/detail.html?product_no=513" + + +def test_fallback_fetcher_continues_when_primary_fails(): + class FailingFetcher: + def fetch(self, url): + raise PermissionError("[WinError 5] access denied") + + class WorkingFetcher: + def fetch(self, url): + return FetchResult(url=url, status_code=200, html="ok") + + result = FallbackFetcher(FailingFetcher(), WorkingFetcher(), fallback_label="requests").fetch("https://example.com") + + assert result.status_code == 200 + assert any("primary fetcher failed" in warning for warning in result.warnings) diff --git a/오픈소스분석자료/Crawl4AI_분석_및_기능명세.md b/오픈소스분석자료/Crawl4AI_분석_및_기능명세.md new file mode 100644 index 0000000..a294c59 --- /dev/null +++ b/오픈소스분석자료/Crawl4AI_분석_및_기능명세.md @@ -0,0 +1,934 @@ +# Crawl4AI 프로젝트 분석 및 기능명세 + +분석 대상: `C:\Users\lasta\MyProject\AI\참고\crawl4ai-main` +분석일: 2026-05-13 +목적: 향후 웹 크롤링/추출 플랫폼의 기준 소스로 삼기 위한 구조 분석 및 정확한 기능명세 정리 + +## 1. 프로젝트 개요 + +Crawl4AI는 Python 기반 오픈소스 웹 크롤러/스크레이퍼 SDK이다. 핵심 목표는 일반 웹페이지, 동적 웹앱, 로컬 파일, 원시 HTML을 수집한 뒤 LLM/RAG/에이전트 파이프라인에 적합한 Markdown, 구조화 JSON, 링크/미디어/메타데이터로 변환하는 것이다. + +주요 특징은 다음과 같다. + +- 비동기 크롤링: `AsyncWebCrawler` 중심의 async SDK +- 브라우저 크롤링: Playwright/Patchright 기반 동적 페이지 처리 +- HTTP 크롤링: 브라우저 없이 빠른 HTTP fetch 처리 +- Markdown 생성: 원문 Markdown, citation 포함 Markdown, 필터링된 fit Markdown 지원 +- 구조화 추출: LLM 기반 의미 추출을 주요 방식으로 지원하며, CSS/XPath/LXML/Regex 기반 결정적 추출도 함께 제공 +- 딥 크롤링: BFS, DFS, Best-First 전략 및 URL 필터/스코어러 +- URL 시딩: sitemap, Common Crawl, HEAD 메타데이터, BM25 기반 URL 후보 생성 +- 안티봇 보조: stealth, undetected browser adapter, proxy retry, fallback fetch hook +- 배포형 API: Docker/FastAPI 서버, REST API, streaming, job API, MCP bridge, monitor dashboard +- 운영 기능: 캐시, smart cache validation, browser pool, rate limit, Redis job state, webhook + +## 2. 기술 스택 + +- 언어: Python 3.10 이상 +- 브라우저 엔진: Playwright, Patchright +- HTTP: aiohttp, httpx +- HTML 처리: lxml, BeautifulSoup, cssselect +- 데이터 모델: Pydantic v2, dataclass +- 저장소/캐시: aiosqlite 기반 로컬 캐시, Docker 서버는 Redis job state 사용 +- LLM 연동: `unclecode-litellm` +- 검색/랭킹: rank-bm25, snowballstemmer, numpy +- 이미지/문서: Pillow, optional PDF parser +- API 서버: FastAPI, slowapi, prometheus-fastapi-instrumentator +- 배포: Dockerfile, docker-compose, supervisord + +## 3. 최상위 구조 + +```text +crawl4ai-main/ + crawl4ai/ # SDK 본체 + crawl4ai/deep_crawling/ # 딥 크롤링 전략, 필터, 스코어러 + crawl4ai/crawlers/ # 특화 크롤러 예: google_search, amazon_product + crawl4ai/processors/pdf/ # PDF 처리 전략 + crawl4ai/script/ # C4A script 컴파일러/검증기 + deploy/docker/ # FastAPI 서버, browser pool, job, monitor, MCP + docs/ # 공식 문서/예제/릴리즈 노트 + tests/ # 단위/통합/회귀/Docker/브라우저 테스트 +``` + +패키지 공개 API는 `crawl4ai/__init__.py`에서 대부분 export한다. 앞으로 우리 프로젝트에서 사용하거나 래핑할 핵심 객체는 +`AsyncWebCrawler`, `BrowserConfig`, `CrawlerRunConfig`, `CacheMode`, 추출 전략류, 딥 크롤링 전략류, `CrawlResult`이다. + +## 4. 핵심 런타임 아키텍처 + +### 4.1 기본 흐름 + +```mermaid +flowchart TD + A["사용자: URL + BrowserConfig + CrawlerRunConfig"] --> B["AsyncWebCrawler.arun"] + B --> C["CacheContext: 캐시 읽기/쓰기 판단"] + C --> D{"캐시 사용 가능?"} + D -- yes --> E["캐시 결과 로드 및 optional freshness 검증"] + D -- no --> F["CrawlerStrategy.crawl"] + F --> G["Playwright 또는 HTTP fetch"] + G --> H["apocess_html: HTML 정리/스크랩/Markdown/추출"] + E --> H + H --> I["CrawlResult 반환"] +``` + +### 4.2 주요 컴포넌트 + +- `AsyncWebCrawler`: SDK 중심 클래스. lifecycle, 캐시, robots.txt, proxy retry, anti-bot retry, HTML 후처리, 단일/다중/딥 크롤링 진입점을 담당한다. +- `AsyncCrawlerStrategy`: 실제 fetch 계층의 추상화. +- `AsyncPlaywrightCrawlerStrategy`: 동적 브라우저 페이지 처리, JS 실행, wait, iframe, screenshot, PDF, MHTML, shadow DOM, network/console capture 등을 담당한다. +- `AsyncHTTPCrawlerStrategy`: 브라우저 없는 HTTP 기반 수집. raw/file/http 다운로드 및 text/file 판단을 처리한다. +- `ContentScrapingStrategy`: HTML에서 cleaned HTML, media, links, metadata, tables 등을 산출한다. +- `MarkdownGenerationStrategy`: cleaned HTML을 LLM 친화 Markdown으로 변환한다. +- `ExtractionStrategy`: Markdown/HTML/text를 구조화 데이터로 추출한다. 특히 `LLMExtractionStrategy`는 이 프로젝트가 지향하는 LLM 친화 크롤링의 핵심 추출 방식이다. +- `DeepCrawlStrategy`: 단일 URL이 아닌 graph traversal 방식의 다중 URL 크롤링을 수행한다. +- `BaseDispatcher`: 다중 URL 크롤링 시 concurrency, memory, rate limit, retry, streaming을 담당한다. + +## 5. SDK 기능명세 + +### 5.1 `AsyncWebCrawler` + +기능: + +- `async with AsyncWebCrawler(...)` context manager 지원 +- `start()`, `close()` 명시 lifecycle 지원 +- `arun(url, config)` 단일 URL/파일/raw HTML 크롤링 +- `arun_many(urls, config, dispatcher)` 다중 URL 크롤링 +- `aseed_urls(...)` URL 후보 생성 +- `aprocess_html(...)` fetch 이후 HTML 처리 파이프라인 +- `thread_safe=True`일 때 내부 lock으로 동시 접근 직렬화 +- `base_directory/.crawl4ai/cache` 캐시 디렉터리 생성 +- `robots.txt` 검사 옵션 지원 +- `deep_crawl_strategy`가 들어오면 `arun` 호출이 딥 크롤링으로 장식됨 + +입력 URL 형식: + +- `http://...`, `https://...` +- `file://...` +- `raw:`, `raw://` + +반환: + +- 일반 단일 크롤: `CrawlResult` +- 딥 크롤 또는 streaming 설정: 전략/설정에 따라 `CrawlResult` 목록 또는 async stream + +### 5.2 `BrowserConfig` + +브라우저 인스턴스/컨텍스트 설정이다. + +주요 항목: + +- 브라우저 종류: `browser_type=chromium|firefox|webkit` +- 실행 형태: `headless`, `browser_mode=dedicated|builtin|docker|custom` +- CDP 연결: `cdp_url`, `browser_context_id`, `target_id`, `cache_cdp_connection` +- persistent context: `use_persistent_context`, `user_data_dir`, `storage_state` +- viewport: `viewport_width`, `viewport_height`, `viewport`, `device_scale_factor` +- proxy: `proxy_config`, deprecated `proxy` +- 다운로드: `accept_downloads`, `downloads_path` +- 인증/헤더: `cookies`, `headers`, `user_agent` +- user-agent 생성: `user_agent_mode=random`, `user_agent_generator_config` +- 성능 모드: `text_mode`, `light_mode`, `memory_saving_mode`, `max_pages_before_recycle` +- 안티봇: `enable_stealth` +- 리소스 차단: `avoid_ads`, `avoid_css` +- 초기 스크립트: `init_scripts` + +주의: + +- `enable_stealth`는 builtin managed browser와 함께 사용할 수 없도록 검증된다. +- `proxy` 문자열은 deprecated이며 내부적으로 `ProxyConfig`로 변환된다. +- `browser_mode=builtin|docker|custom`은 managed browser/CDP 경로를 사용한다. + +### 5.3 `CrawlerRunConfig` + +개별 크롤 요청 단위 설정이다. 앞으로 우리 프로젝트에서 가장 자주 매핑해야 할 객체다. + +콘텐츠 처리: + +- `word_count_threshold` +- `css_selector` +- `target_elements` +- `excluded_tags` +- `excluded_selector` +- `only_text` +- `keep_data_attributes` +- `keep_attrs` +- `remove_forms` +- `prettiify` +- `parser_type` +- `scraping_strategy` + +추출/Markdown: + +- `extraction_strategy` +- `chunking_strategy` +- `markdown_generator` +- `table_extraction` +- `table_score_threshold` + +캐시: + +- `cache_mode=ENABLED|DISABLED|READ_ONLY|WRITE_ONLY|BYPASS` +- `check_cache_freshness` +- `cache_validation_timeout` +- legacy 옵션 `bypass_cache`, `disable_cache`, `no_cache_read`, `no_cache_write`는 deprecated 접근 시 에러 유도 + +세션/프록시: + +- `session_id` +- `proxy_config` +- `proxy_rotation_strategy` +- `proxy_session_id` +- `proxy_session_ttl` +- `proxy_session_auto_release` + +브라우저 지역/정체성: + +- `locale` +- `timezone_id` +- `geolocation` +- `user_agent` +- `user_agent_mode` + +페이지 로딩/대기: + +- `wait_until` +- `page_timeout` +- `wait_for` +- `wait_for_timeout` +- `wait_for_images` +- `delay_before_return_html` +- `mean_delay` +- `max_range` +- `semaphore_count` + +상호작용: + +- `js_code` +- `js_code_before_wait` +- `c4a_script` +- `js_only` +- `scan_full_page` +- `scroll_delay` +- `max_scroll_steps` +- `process_iframes` +- `flatten_shadow_dom` +- `remove_overlay_elements` +- `remove_consent_popups` +- `simulate_user` +- `override_navigator` +- `magic` +- `adjust_viewport_to_content` + +미디어/아카이브: + +- `screenshot` +- `screenshot_wait_for` +- `screenshot_height_threshold` +- `force_viewport_screenshot` +- `pdf` +- `capture_mhtml` +- `exclude_external_images` +- `exclude_all_images` +- `image_description_min_word_threshold` +- `image_score_threshold` + +링크: + +- `exclude_external_links` +- `exclude_internal_links` +- `exclude_social_media_links` +- `exclude_social_media_domains` +- `exclude_domains` +- `score_links` +- `preserve_https_for_internal_links` +- `link_preview_config` + +디버깅/관찰: + +- `verbose` +- `log_console` +- `capture_network_requests` +- `capture_console_messages` + +연결/실행: + +- `method` +- `stream` +- `prefetch` +- `process_in_browser` +- `check_robots_txt` + +딥 크롤링/매칭: + +- `deep_crawl_strategy` +- `virtual_scroll_config` +- `url_matcher` +- `match_mode=OR|AND` + +안티봇 재시도: + +- `max_retries` +- `fallback_fetch_function` + +### 5.4 `CrawlResult` + +크롤 결과 모델이다. + +주요 필드: + +- `url` +- `success` +- `html` +- `cleaned_html` +- `markdown` +- `extracted_content` +- `media` +- `links` +- `metadata` +- `tables` +- `screenshot` +- `pdf` +- `mhtml` +- `downloaded_files` +- `js_execution_result` +- `session_id` +- `status_code` +- `response_headers` +- `redirected_url` +- `redirected_status_code` +- `ssl_certificate` +- `network_requests` +- `console_messages` +- `dispatch_result` +- `head_fingerprint` +- `cached_at` +- `cache_status` +- `crawl_stats` +- `error_message` + +`markdown`는 문자열처럼 동작하면서 내부적으로 `MarkdownGenerationResult`를 제공한다. + +`MarkdownGenerationResult` 필드: + +- `raw_markdown` +- `markdown_with_citations` +- `references_markdown` +- `fit_markdown` +- `fit_html` + +## 6. 추출 기능명세 + +Crawl4AI의 추출 계층은 LLM 기반 의미 추출을 중심 기능으로 제공하고, CSS/XPath/LXML/Regex 기반 추출을 보완적인 결정적 전략으로 함께 둔다. 즉 이 프로젝트는 단순 HTML 파서가 아니라, 수집한 웹 콘텐츠를 LLM이 바로 이해하고 구조화할 수 있는 형태로 변환하는 것을 주요 방식으로 삼는다. + +### 6.1 LLM 기반 추출 + +클래스: `LLMExtractionStrategy` + +기능: + +- LLM provider, instruction, schema 기반 구조화 추출 +- chunk 단위 분할 후 병렬/순차 LLM 호출 +- token usage 집계 +- JSON schema 기반 결과 유도 가능 +- Docker API의 `/llm`, `/llm/job`, `/ask`에서도 사용 + +사용처: + +- 크롤링 결과의 기본 의미 추출 방식 +- 비정형/반정형 페이지에서 의미 기반 필드 추출 +- 사용자가 자연어 instruction으로 원하는 데이터 구조를 지정하는 추출 +- RAG용 요약/질의응답 +- schema 기반 JSON 생성 및 schema 자동 생성 보조 + +### 6.2 CSS/XPath/LXML 기반 JSON 추출 + +클래스: + +- `JsonCssExtractionStrategy` +- `JsonXPathExtractionStrategy` +- `JsonLxmlExtractionStrategy` + +기능: + +- 반복 요소 base selector 지정 +- field별 selector, type, attribute, transform 지정 +- text/html/attribute/source 추출 +- nested/list field 추출 +- LLM을 이용한 schema 생성 보조 메서드 제공 + +권장 사용: + +- 쇼핑몰 상품 목록, 뉴스 목록, 테이블형 반복 카드처럼 DOM 구조가 안정적인 사이트 +- LLM 호출 비용을 줄여야 하거나 완전히 반복적인 DOM 패턴이 검증된 경우 + +### 6.3 Regex 추출 + +클래스: `RegexExtractionStrategy` + +기능: + +- email, url, phone 등 정규식 패턴 기반 추출 +- 사용자 정의 패턴 지원 +- plain text 변환 후 추출 가능 + +### 6.4 Cosine/Embedding 추출 + +클래스: `CosineStrategy` + +기능: + +- 문서 chunk embedding +- query와 유사한 문서 조각 필터링 +- hierarchical clustering 보조 + +주의: + +- optional dependency가 필요할 수 있다. +- LLM 없이 관련 섹션만 좁히는 용도에 적합하다. + +## 7. Markdown 및 콘텐츠 필터링 + +### 7.1 Markdown 생성 + +클래스: `DefaultMarkdownGenerator` + +기능: + +- HTML을 Markdown으로 변환 +- 링크 citation 및 reference 목록 생성 +- content filter 적용 후 `fit_markdown`, `fit_html` 생성 +- 표/코드/헤딩/링크가 LLM 입력에 적합하도록 정리 + +### 7.2 Content Filter + +클래스: + +- `PruningContentFilter`: 휴리스틱 기반 noise 제거 +- `BM25ContentFilter`: query 기반 관련 콘텐츠 선별 +- `LLMContentFilter`: LLM 기반 관련 콘텐츠 선별 + +사용 기준: + +- 단순 문서 정리: `PruningContentFilter` +- 사용자 질의 중심 수집: `BM25ContentFilter` +- 의미적 판단이 중요한 고품질 추출: `LLMContentFilter` +- 우리 프로젝트의 기본 의미 필터링/추출 정책: LLM 우선, 필요 시 BM25/CSS/XPath로 비용과 속도를 보완 + +## 8. 브라우저 크롤링 기능명세 + +`AsyncPlaywrightCrawlerStrategy`가 담당한다. + +지원 기능: + +- Playwright browser/context/page lifecycle +- dedicated browser, managed browser, CDP 연결 +- persistent profile 및 storage state +- JS 실행: 크롤 전/후 스크립트, C4A script 컴파일 결과 +- selector/function 기반 wait +- iframe 처리 +- shadow DOM flatten +- overlay/consent popup 제거 +- full page scan 및 virtual scroll +- lazy image 대기 +- screenshot 캡처 +- PDF export +- MHTML 캡처 +- file download 처리 +- network request capture +- console message capture +- SSL certificate fetch +- navigator override 및 simulated user 동작 +- stealth 적용 + +브라우저 관리자: + +- `ManagedBrowser`는 CDP endpoint를 제공하는 browser process를 직접 띄우거나 기존 CDP에 연결한다. +- memory saving, light mode, text mode, proxy flag, debugging port, user data dir를 관리한다. + +## 9. HTTP 크롤링 기능명세 + +`AsyncHTTPCrawlerStrategy`가 담당한다. + +지원 기능: + +- HTTP/HTTPS 요청 +- `file://` 로컬 파일 처리 +- `raw:` HTML 처리 +- content-type 기반 text/file 판단 +- 파일 다운로드명 추출 +- proxy formatting +- hook 실행 +- browser 없이 빠른 HTML 수집 + +제약: + +- JS 렌더링, 실제 브라우저 DOM 변화, screenshot/PDF 등은 브라우저 전략 필요 + +## 10. 딥 크롤링 기능명세 + +### 10.1 전략 + +- `BFSDeepCrawlStrategy`: breadth-first 탐색 +- `DFSDeepCrawlStrategy`: depth-first 탐색 +- `BestFirstCrawlingStrategy`: URL score 기반 우선순위 탐색 + +공통 기능: + +- `max_depth` +- `max_pages` +- stream/batch 실행 +- cancellation +- link discovery +- visited/seen 관리 +- resume/export state 일부 지원 +- crawler의 `arun`을 decorator로 감싸 단일 호출 인터페이스와 통합 + +### 10.2 필터 + +- `FilterChain`: 여러 URL filter 조합 +- `URLPatternFilter`: glob/패턴 기반 include/exclude +- `DomainFilter`: allowed/blocked domain, subdomain 판단 +- `ContentTypeFilter`: 확장자/content type 기반 판단 +- `ContentRelevanceFilter`: BM25 기반 관련도 +- `SEOFilter`: title, meta description, canonical, schema.org, URL 품질 기반 score + +### 10.3 스코어러 + +- `KeywordRelevanceScorer` +- `PathDepthScorer` +- `ContentTypeScorer` +- `FreshnessScorer` +- `DomainAuthorityScorer` +- `CompositeScorer` + +Best-first crawling에서 우선순위 계산에 사용한다. + +## 11. 다중 URL 크롤링/Dispatcher + +클래스: + +- `BaseDispatcher` +- `MemoryAdaptiveDispatcher` +- `SemaphoreDispatcher` +- `RateLimiter` + +기능: + +- URL별 config 선택: `url_matcher`와 `match_mode` +- concurrency 제한 +- memory threshold 기반 backpressure +- domain별 rate limit +- retry +- task status, memory usage, peak memory 기록 +- streaming result 지원 +- dispatcher monitor 연계 + +권장: + +- 소량 병렬: `SemaphoreDispatcher` +- 대량/장시간 크롤: `MemoryAdaptiveDispatcher` + +## 12. URL Seeder 기능명세 + +클래스: `AsyncUrlSeeder` + +기능: + +- sitemap 기반 URL 수집 +- Common Crawl index 기반 URL 수집 +- URL pattern 필터링 +- live validation +- HEAD 요청으로 title/meta/canonical 등 head data 수집 +- BM25/query 기반 URL relevance scoring +- nonsense URL 필터링 +- cache 사용 +- 여러 domain에 대한 batch seeding + +설정 객체: `SeedingConfig` + +주요 사용 시나리오: + +- “사이트 전체 중 특정 주제/상품/문서 URL 후보를 먼저 뽑고, 선별된 URL만 실제 크롤링” +- 대규모 사이트에서 full crawl 전에 seed 후보를 줄이는 단계 + +## 13. Adaptive Crawler 기능명세 + +클래스: + +- `AdaptiveCrawler` +- `AdaptiveConfig` +- `CrawlState` +- `StatisticalStrategy` +- `EmbeddingStrategy` + +기능: + +- 수집 상태를 누적하며 confidence 계산 +- query coverage, consistency, saturation 기반 stop 판단 +- 링크 relevance/novelty/authority 기반 ranking +- embedding 기반 semantic exploration +- state save/load + +사용 시나리오: + +- 고정 depth/page 수가 아니라 “원하는 정보가 충분히 모였을 때 멈추는” 연구형 크롤러 + +## 14. 캐시 기능명세 + +캐시 모드: + +- `ENABLED`: 읽기/쓰기 +- `DISABLED`: 캐시 사용 안 함 +- `READ_ONLY`: 읽기만 +- `WRITE_ONLY`: 쓰기만 +- `BYPASS`: 해당 작업에서 캐시 우회 + +Smart Cache: + +- `check_cache_freshness=True`일 때 ETag, Last-Modified, head fingerprint로 freshness 검증 +- fresh면 `cache_status=hit_validated` +- 검증 실패 시 fallback으로 cached result 사용 가능 +- stale/unknown이면 재크롤 + +캐시 대상: + +- web URL과 file URL은 cacheable +- raw HTML은 기본적으로 cacheable 아님 + +## 15. 프록시 및 안티봇 기능명세 + +프록시: + +- `ProxyConfig` +- 문자열/dict/env 기반 생성 +- list proxy 지원 +- `ProxyRotationStrategy`, `RoundRobinProxyStrategy` +- sticky proxy session: `proxy_session_id`, `proxy_session_ttl` +- NSTProxy API 연동 helper + +안티봇: + +- HTML/status 기반 block detection +- `max_retries` +- 여러 proxy 순회 +- 실패 통계 `crawl_stats` +- 최후 수단 `fallback_fetch_function` +- `enable_stealth` +- `UndetectedAdapter` +- browser flags에서 automation 흔적 일부 완화 + +주의: + +- CAPTCHA 해결 자체는 본체 기능이 아니라 예제에 가까운 외부 서비스 연동 형태다. +- 안티봇 우회는 사이트 약관/법적 제한을 반드시 확인해야 한다. + +## 16. Docker/FastAPI 서버 기능명세 + +경로: `deploy/docker` + +### 16.1 서버 구성 + +- `server.py`: FastAPI entrypoint +- `api.py`: crawl/md/llm 처리 로직 +- `crawler_pool.py`: browser pool +- `job.py`: 비동기 job API +- `monitor.py`, `monitor_routes.py`: dashboard/metrics +- `auth.py`: JWT token 발급/검증 +- `webhook.py`: job 완료 webhook 전달 +- `mcp_bridge.py`: MCP schema/tool bridge +- `schemas.py`: request/response schema + +### 16.2 REST endpoint + +- `GET /`: playground redirect +- `POST /token`: JWT token 발급 +- `POST /config/dump`: config object serialization +- `POST /md`: URL을 Markdown으로 변환 +- `POST /html`: HTML 반환 +- `POST /screenshot`: screenshot 반환/저장 +- `POST /pdf`: PDF 반환/저장 +- `POST /execute_js`: 지정 JS 실행 +- `GET /llm/{url:path}`: URL + query 기반 LLM QA +- `GET /schema`: 서버/API schema +- `GET /hooks/info`: hook 지원 정보 +- `GET /health`: health check +- `GET /metrics`: Prometheus metrics +- `POST /crawl`: 다중 URL 크롤 +- `POST /crawl/stream`: streaming crawl +- `GET /ask`: 질문/응답형 endpoint +- `POST /llm/job`: LLM extraction background job 생성 +- `GET /llm/job/{task_id}`: LLM job 조회 +- `POST /crawl/job`: crawl background job 생성 +- `GET /crawl/job/{task_id}`: crawl job 조회 + +Monitor endpoint: + +- `GET /dashboard` +- `GET /health` +- `GET /requests` +- `GET /browsers` +- `GET /endpoints/stats` +- `GET /timeline` +- `GET /logs/janitor` +- `GET /logs/errors` +- `POST /actions/cleanup` +- `POST /actions/kill_browser` +- `POST /actions/restart_browser` +- `POST /stats/reset` +- `WebSocket /ws` + +### 16.3 API 요청 모델 + +`CrawlRequest`: + +- `urls: List[str]`, 1~100개 +- `browser_config: Dict` +- `crawler_config: Dict` + +`CrawlRequestWithHooks`: + +- `CrawlRequest` + optional `hooks` + +`MarkdownRequest`: + +- `url` +- `f=fit|raw|bm25|llm` +- `q` +- `c` +- `provider` +- `temperature` +- `base_url` + +`ScreenshotRequest`: + +- `url` +- `screenshot_wait_for` +- `wait_for_images` +- `output_path` + +`PDFRequest`: + +- `url` +- `output_path` + +`JSEndpointRequest`: + +- `url` +- `scripts` + +### 16.4 보안/운영 + +- JWT token 인증 +- hooks는 기본 비활성화: `CRAWL4AI_HOOKS_ENABLED=false` +- hook code 실행은 RCE 위험이 있으므로 운영 환경에서는 비활성 권장 +- global page semaphore로 동시 page 수 제한 +- rate limiting +- TrustedHost/HTTPS middleware 옵션 +- Redis 기반 task state 및 TTL +- Prometheus metrics +- playground와 monitor dashboard 정적 파일 제공 + +## 17. CLI 기능명세 + +entrypoint: + +- `crwl = crawl4ai.cli:main` +- `crawl4ai-setup` +- `crawl4ai-doctor` +- `crawl4ai-download-models` +- `crawl4ai-migrate` + +README 기준 CLI 예: + +```bash +crwl https://www.nbcnews.com/business -o markdown +crwl https://docs.crawl4ai.com --deep-crawl bfs --max-pages 10 +crwl https://www.example.com/products -q "Extract all product prices" +``` + +역할: + +- 빠른 단일 URL 크롤 +- Markdown 출력 +- 딥 크롤 옵션 +- 질의 기반 LLM 추출 +- 설정 파일 기반 실행 예제 제공 + +## 18. C4A Script 기능명세 + +경로: `crawl4ai/script` + +공개 API: + +- `c4a_compile` +- `c4a_validate` +- `c4a_compile_file` +- `CompilationResult` +- `ValidationResult` +- `ErrorDetail` + +역할: + +- 사람이 읽기 쉬운 C4A script를 JavaScript로 컴파일 +- `CrawlerRunConfig(c4a_script=...)`에 넣으면 `js_code`로 변환되어 브라우저에서 실행 +- 폼 입력, 클릭, 스크롤, 로그인 흐름 등 반복 브라우저 작업 자동화에 적합 + +## 19. 특화 크롤러 + +경로: + +- `crawl4ai/crawlers/google_search` +- `crawl4ai/crawlers/amazon_product` + +역할: + +- 공통 SDK 위에 특정 사이트/도메인 추출 로직을 래핑한 예시 +- 향후 우리 프로젝트에서 도메인별 크롤러를 만들 때 참고할 구조 + +## 20. 테스트 자산 + +테스트는 다음 범위를 포괄한다. + +- 기본 async crawler +- browser manager/context/CDP/profile +- raw HTML/file/http 처리 +- caching/smart cache +- markdown/content filter +- extraction strategies +- table extraction +- link/media extraction +- deep crawling, filters, scorers, resume/cancel +- Docker API/server/hooks/security/webhook +- proxy/sticky sessions +- memory/stress +- regression tests + +이 프로젝트를 기반으로 개발할 때는 기존 테스트명을 기능별 체크리스트로 활용할 수 있다. + +## 21. 우리 프로젝트에 적용할 때의 권장 아키텍처 + +### 21.1 권장 래핑 계층 + +우리 코드에서 Crawl4AI를 직접 전역적으로 흩뿌려 쓰기보다 아래 계층으로 감싸는 것을 권장한다. + +```text +우리 서비스 + CrawlJob API / Queue + Domain Crawler Service + Crawl4AI Adapter + - BrowserConfig factory + - CrawlerRunConfig factory + - ExtractionStrategy factory + - Result normalizer + Crawl4AI SDK +``` + +### 21.2 우리가 정의해야 할 내부 표준 + +- 크롤 목적별 profile: + - `fast_static`: HTTP 또는 text/light mode + - `dynamic_page`: Playwright + JS/wait + - `full_capture`: screenshot/pdf/mhtml/network + - `structured_extract`: CSS/XPath schema + - `semantic_extract`: LLM/BM25 + - `deep_discovery`: URL seeder + deep crawl + +- 결과 저장 표준: + - raw html + - cleaned html + - raw markdown + - fit markdown + - extracted JSON + - media/links/tables + - crawl metadata/status/error + +- 실패 표준: + - DNS/network timeout + - HTTP error + - robots blocked + - anti-bot blocked + - extraction empty + - schema mismatch + - LLM provider failure + +## 22. 장점 + +- SDK/API/CLI/Docker를 모두 제공해 개발-운영 경로가 넓다. +- 동적 페이지 처리 기능이 풍부하다. +- Markdown과 구조화 추출이 기본 내장되어 LLM/RAG 파이프라인과 맞다. +- 딥 크롤링, URL 시딩, adaptive crawling까지 있어 단순 scraper보다 확장성이 높다. +- 캐시/dispatcher/browser pool/monitor 등 운영 기능도 상당히 갖추어져 있다. + +## 23. 리스크 및 주의사항 + +- 코드베이스가 크고 기능이 빠르게 확장된 흔적이 있어 일부 API가 deprecated 상태다. +- README 일부 문자는 인코딩이 깨져 있어 원문 문서만 보고 자동 처리하기 어렵다. +- hook code 실행은 보안상 위험하다. +- LLM extraction은 이 오픈소스가 제공하는 주요 추출 방식이다. 우리 프로젝트에서는 크롤링 옵션으로 추출 방식을 선택할 수 있게 하되, 기본 정책은 LLM 기반 추출 우선으로 둔다. CSS/XPath/Regex/schema 기반 추출은 비용, 속도, 반복 DOM 안정성이 중요한 경우 선택 가능한 보완 전략으로 사용한다. +- 브라우저 기반 대량 크롤링은 메모리 누수/컨텍스트 정리/프로세스 recycle 정책이 중요하다. +- anti-bot/stealth/proxy 기능은 기술적으로 제공되지만 법적/약관 리스크를 별도로 관리해야 한다. +- Docker 서버는 Redis, browser pool, auth, monitor 등 운영 의존성이 있어 단순 SDK 사용보다 배포 복잡도가 높다. + +## 24. 향후 개발 기준 기능명세 + +이 소스를 기반으로 우리 프로젝트를 진행할 때 최소 기능 기준은 다음과 같이 잡는 것을 권장한다. + +### 24.1 MVP 필수 + +- 단일 URL 크롤 +- 다중 URL 크롤 +- 동적 페이지 JS 렌더링 +- wait selector/function +- raw HTML 처리 +- Markdown 변환 +- LLM 기반 의미 추출 +- CSS/XPath 기반 JSON 추출 옵션 +- 캐시 모드 +- screenshot 선택 캡처 +- 링크/미디어/메타데이터 수집 +- 표 추출 +- 에러/상태/HTTP status 기록 + +### 24.2 1차 확장 + +- URL seeding +- BFS/DFS deep crawl +- domain/pattern/content-type filter +- BM25 query 기반 content filter +- proxy config +- session reuse +- persistent browser profile +- network/console capture +- smart cache validation + +### 24.3 운영 확장 + +- Docker API 또는 자체 FastAPI 래퍼 +- job queue +- streaming result +- Redis/task state +- webhook +- monitor dashboard +- Prometheus metrics +- browser pool +- rate limit +- memory adaptive dispatcher + +### 24.4 고급 확장 + +- adaptive crawler +- best-first scoring +- embedding strategy +- C4A script 기반 브라우저 자동화 +- custom domain crawler +- anti-bot retry/fallback +- MHTML/PDF archive + +## 25. 결론 + +Crawl4AI는 단순 페이지 다운로드 도구가 아니라 “웹을 LLM 친화 데이터로 변환하는 비동기 크롤링 프레임워크”에 가깝다. +앞으로 우리 프로젝트의 기본 소스로 삼는다면 `AsyncWebCrawler + CrawlerRunConfig + ExtractionStrategy + MarkdownGenerator + DeepCrawlStrategy` +조합을 중심으로 래핑하고, Docker 서버 코드는 운영형 API 설계 참고 또는 별도 배포 모듈로 분리해 사용하는 것이 가장 현실적이다. + +가장 중요한 설계 결정은 다음 세 가지다. + +1. 기본 추출 방식은 Crawl4AI의 주요 설계 방향에 맞춰 LLM 기반 의미 추출을 우선한다. +2. CSS/XPath/Regex 같은 결정적 전략은 크롤링 옵션으로 제공해 비용, 속도, 반복 DOM 안정성이 중요한 경우 선택하게 한다. +3. 브라우저 크롤링은 비용이 크므로 URL seeding, cache, dispatcher, profile 재사용으로 호출량을 통제한다. diff --git a/오픈소스분석자료/Firecrawl_분석_및_기능명세.md b/오픈소스분석자료/Firecrawl_분석_및_기능명세.md new file mode 100644 index 0000000..c820cb9 --- /dev/null +++ b/오픈소스분석자료/Firecrawl_분석_및_기능명세.md @@ -0,0 +1,892 @@ +# Firecrawl 프로젝트 분석 및 기능명세 + +분석 대상: `C:\Users\lasta\MyProject\AI\참고\firecrawl-main` +분석일: 2026-05-13 +목적: 범용 온톨로지 구축 플랫폼의 웹 수집/정제/구조화 기반 소스로 Firecrawl을 거의 원형에 가깝게 재사용할 수 있는지 판단하고, 향후 구현 기준이 될 기능 명세를 정리한다. + +## 1. 결론 요약 + +Firecrawl은 단순 크롤러가 아니라 `검색 -> URL 발견 -> 페이지 수집 -> 동적 브라우저 실행 -> Markdown/HTML/JSON/스크린샷/파일 파싱 -> 비동기 작업 관리 -> SDK 제공`까지 포함하는 웹 데이터 수집 API 플랫폼이다. 현재 프로젝트의 범용 온톨로지 구축 플랫폼에는 다음 영역이 특히 직접 재사용 가치가 높다. + +- `apps/api/src/scraper/scrapeURL`: 단일 URL 수집의 핵심. Fetch, Playwright, PDF, 문서, 인덱스, Fire-engine 계열 엔진을 fallback 방식으로 선택한다. +- `apps/api/src/scraper/WebScraper/crawler.ts`: 사이트 내부 URL 탐색, sitemap, robots.txt, include/exclude path, depth, subdomain/external link 제어. +- `apps/api/src/controllers/v2/types.ts`: API 입력/출력 스키마. 특히 scrape/crawl/map/search 옵션 체계가 잘 정리되어 있다. +- `apps/api/src/controllers/v2/*`: 외부 API 기능 명세의 실제 기준. `scrape`, `crawl`, `map`, `search`, `batch scrape`, `parse`, `monitor`, `browser`, `agent`로 분리되어 있다. +- `apps/api/src/services/worker/scrape-worker.ts`, `queue-*`: 대량 수집, 크롤 작업, billing/logging/webhook/상태 관리의 운영 흐름. +- `apps/python-sdk`, `apps/js-sdk/firecrawl`: 우리 플랫폼 API 클라이언트 설계 시 참고할 수 있는 SDK 표면. + +다만 Firecrawl은 Node.js/TypeScript 기반의 API 서버, Redis/BullMQ 또는 NuQ/RabbitMQ/Postgres, Playwright microservice, Supabase/Autumn/Stripe/GCS/Sentry 등 SaaS 운영 요소가 섞여 있다. 현재 Python/FastAPI/SQLAlchemy 기반 프로젝트에 그대로 병합하기보다는, Firecrawl을 별도 수집 서비스로 두고 Python 온톨로지 파이프라인이 Firecrawl API를 호출하는 구조가 가장 안전하다. + +## 2. 프로젝트 성격 + +Firecrawl의 제품 목표는 웹 페이지를 LLM/RAG/Agent가 바로 사용할 수 있는 깨끗한 데이터로 변환하는 것이다. 제공 기능은 다음 세 가지 축으로 요약된다. + +- 단일 페이지 변환: URL을 Markdown, HTML, raw HTML, link/image 목록, screenshot, structured JSON 등으로 변환한다. +- 사이트 단위 수집: seed URL에서 sitemap과 링크 그래프를 따라가며 여러 페이지를 비동기로 수집한다. +- 지능형 데이터 추출: JSON Schema, LLM prompt, browser action, search result scraping, file parsing을 결합한다. + +온톨로지 구축 플랫폼 관점에서는 Firecrawl이 `웹 수집 계층`과 `텍스트/문서 정제 계층`을 맡고, 현재 프로젝트의 Python 코드는 `도메인 어댑터`, `엔티티/관계 추출`, `Claim/Evidence 저장`, `추천/검증 UI`를 맡는 분업이 적합하다. + +## 3. 최상위 구조 + +```text +firecrawl-main/ + apps/ + api/ # 핵심 API 서버, 스크래퍼, 크롤러, 큐/워커 + playwright-service-ts/ # Playwright 브라우저 마이크로서비스 + python-sdk/ # Python SDK + js-sdk/firecrawl/ # JS/TS SDK + php-sdk, ruby-sdk, + rust-sdk, elixir-sdk # 다언어 SDK + ui/ingestion-ui/ # ingestion UI + test-suite/ # API/load 테스트 + test-site/ # 테스트용 사이트 + go-html-to-md-service/ # HTML -> Markdown 변환 보조 서비스 + nuq-postgres/ # NuQ 큐용 Postgres 구성 + examples/ # LLM/agent/추출 예제 다수 + docker-compose.yaml # self-host 전체 구성 + SELF_HOST.md # 자체 호스팅 안내 + README.md # 제품/SDK/API 개요 +``` + +핵심은 `apps/api`이다. 나머지는 SDK, 배포, 예제, 테스트, UI 보조 레이어다. + +## 4. 기술 스택 + +- 언어: TypeScript/Node.js, 일부 Rust native package, 일부 Go service +- API: Express, express-ws, Zod validation, multer multipart +- 브라우저: 별도 `playwright-service-ts`, Fire-engine CDP/TLS client 연동 가능 +- HTML 처리: Cheerio, JSDOM, Turndown, joplin-turndown-plugin-gfm, Rust 기반 link filtering/extraction +- 문서 처리: PDF, DOC/DOCX/ODT/RTF/XLS/XLSX 계열 파일 처리 모듈 +- 큐/상태: BullMQ, Redis, NuQ, RabbitMQ, Postgres +- 검색: Google 기본, SearXNG 대체, DuckDuckGo/v2 검색 코드 +- LLM: OpenAI, Anthropic, Google, Groq, xAI, OpenRouter, Ollama/OpenAI-compatible +- 운영: Docker Compose, Kubernetes/Helm 예제, Sentry, Prometheus, logging, billing + +## 5. 런타임 아키텍처 + +```mermaid +flowchart TD + A["Client / SDK / API"] --> B["Express v2 Router"] + B --> C["Auth / rate limit / credit / blocklist"] + C --> D{"Endpoint"} + D --> E["Scrape Controller"] + D --> F["Crawl Controller"] + D --> G["Map Controller"] + D --> H["Search Controller"] + E --> I["scrapeURL engine fallback"] + F --> J["WebCrawler URL discovery"] + J --> K["Queue scrape jobs"] + G --> J + H --> L["Search provider"] + H --> I + K --> M["Scrape Worker"] + M --> I + I --> N["Fetch / Playwright / PDF / Document / Index / Fire-engine"] + N --> O["Markdown / metadata / formats / actions"] + O --> P["Result / status / webhook / logs"] +``` + +### 핵심 흐름 + +1. API 요청은 `apps/api/src/routes/v2.ts`에서 endpoint별 controller로 라우팅된다. +2. Zod 스키마가 URL, 옵션, format, crawler option을 strict하게 검증한다. +3. 단일 scrape는 동기적으로 처리하되 내부적으로 semaphore와 worker 공통 함수를 사용한다. +4. crawl/batch scrape는 작업 ID를 반환하고 큐에 개별 scrape job을 넣는다. +5. worker는 URL별로 `scrapeURL`을 실행하고 성공/실패/robots 차단/비용/로그/웹훅을 기록한다. +6. 결과는 status endpoint, websocket, webhook, SDK polling을 통해 조회된다. + +## 6. 핵심 모듈 분석 + +### 6.1 API 라우터 + +파일: `apps/api/src/routes/v2.ts` + +주요 endpoint: + +- `POST /v2/search` +- `POST /v2/parse` +- `POST /v2/scrape` +- `GET /v2/scrape/:jobId` +- `POST /v2/scrape/:jobId/interact` +- `DELETE /v2/scrape/:jobId/interact` +- `POST /v2/batch/scrape` +- `GET /v2/batch/scrape/:jobId` +- `DELETE /v2/batch/scrape/:jobId` +- `GET /v2/batch/scrape/:jobId/errors` +- `POST /v2/map` +- `POST /v2/crawl` +- `POST /v2/crawl/params-preview` +- `GET /v2/crawl/ongoing` +- `GET /v2/crawl/:jobId` +- `DELETE /v2/crawl/:jobId` +- `WS /v2/crawl/:jobId` +- `GET /v2/crawl/:jobId/errors` +- `POST /v2/extract` +- `GET /v2/extract/:jobId` +- `POST /v2/agent` +- `GET /v2/agent/:jobId` +- `DELETE /v2/agent/:jobId` +- `POST /v2/monitor` +- `GET /v2/monitor` +- `GET /v2/monitor/:monitorId` +- `PATCH /v2/monitor/:monitorId` +- `DELETE /v2/monitor/:monitorId` +- `POST /v2/monitor/:monitorId/run` +- `GET /v2/monitor/:monitorId/checks` +- `GET /v2/monitor/:monitorId/checks/:checkId` +- `POST /v2/browser` +- `GET /v2/browser` +- `POST /v2/browser/:sessionId/execute` +- `DELETE /v2/browser/:sessionId` +- `GET /v2/team/credit-usage` +- `GET /v2/team/token-usage` +- `GET /v2/concurrency-check` +- `GET /v2/team/queue-status` +- `GET /v2/team/activity` + +우리 프로젝트에서 우선 필요한 것은 `scrape`, `crawl`, `map`, `batch scrape`, `parse`, `search`다. `billing`, `credit`, `team`, `x402`, `agent signup`, `support proxy`는 초기에는 제외 가능하다. + +### 6.2 스키마와 옵션 체계 + +파일: `apps/api/src/controllers/v2/types.ts` + +Firecrawl은 입력 옵션을 Zod로 strict validation한다. 알 수 없는 key는 거부하는 방식이라 API 안정성이 높다. + +#### 공통 Scrape 옵션 + +- `formats`: 기본 `markdown`. 지원 format은 `markdown`, `html`, `rawHtml`, `links`, `images`, `summary`, `json`, `changeTracking`, `screenshot`, `attributes`, `branding`, `question`, `highlights`, `query`, `audio`. +- `headers`: 요청 header. +- `includeTags`, `excludeTags`: 특정 selector 포함/제외. iframe selector 변환도 처리한다. +- `onlyMainContent`: 기본 true. 본문 중심 추출. +- `onlyCleanContent`: 기본 false. +- `timeout`: 최소 1000ms. +- `waitFor`: 기본 0, 최대 60000ms, timeout의 절반 이하. +- `mobile`: 모바일 viewport 사용. +- `parsers`: PDF/문서 parser 옵션. +- `actions`: wait/click/write/press/scroll/scrape/screenshot 등 브라우저 액션. +- `location`: country/languages. 기본 country는 `us-generic`. +- `skipTlsVerification`: TLS 검증 skip. +- `removeBase64Images`: 기본 true. +- `fastMode`: 빠른 수집 모드. +- `blockAds`: 기본 true. +- `proxy`: `basic`, `stealth`, `enhanced`, `auto`. 기본 `auto`. +- `maxAge`, `minAge`, `storeInCache`: 캐시 사용 기준. +- `lockdown`: 캐시/index 기반 제한 모드. +- `profile`: 브라우저 profile 이름과 저장 여부. + +중요 transform: + +- JSON format이 있고 기본 timeout 30000ms이면 60000ms로 늘린다. +- stealth/enhanced/auto proxy이며 기본 timeout이면 120000ms로 늘린다. +- changeTracking은 markdown format을 요구하고 waitFor/timeout을 늘린다. +- actions + waitFor 총 대기 시간은 60초를 넘지 못한다. + +#### Crawler 옵션 + +- `includePaths`: 포함할 path regex. +- `excludePaths`: 제외할 path regex. +- `maxDiscoveryDepth`: 발견 깊이 제한. +- `limit`: 기본 10000. +- `crawlEntireDomain`: 전체 domain 허용. +- `allowExternalLinks`: 기본 false. +- `allowSubdomains`: 기본 false. +- `ignoreRobotsTxt`: 기본 false. +- `robotsUserAgent`: robots.txt 확인 user-agent. +- `sitemap`: `skip`, `include`, `only`. 기본 `include`. +- `deduplicateSimilarURLs`: 기본 true. +- `ignoreQueryParameters`: 기본 false. +- `regexOnFullURL`: 기본 false. +- `delay`: URL 간 delay. + +#### Map 옵션 + +Map은 URL 목록 발견용이다. 기본 `includeSubdomains=true`, `ignoreQueryParameters=true`, `limit=5000`, 최대 `100000`이다. `search`, `sitemap`, `filterByPath`, `useIndex`, `ignoreCache`, `location`, `headers`를 지원한다. + +#### Search 옵션 + +- `query`: 검색어. +- `limit`: 기본 10, 최대 100. +- `sources`: `web`, `images`, `news`. +- `categories`: `github`, `research`, `pdf`. +- `includeDomains`, `excludeDomains`: 동시에 지정 불가. +- `lang`: 기본 en. +- `country`/`location`. +- `timeout`: 기본 60000ms. +- `asyncScraping`: 검색 결과 scraping을 비동기 job으로 반환 가능. +- `scrapeOptions`: 검색 결과 페이지를 바로 scrape할 때 사용하는 제한된 scrape 옵션. + +### 6.3 단일 URL 수집 엔진 + +파일: `apps/api/src/scraper/scrapeURL/index.ts`, `apps/api/src/scraper/scrapeURL/engines/index.ts` + +Firecrawl의 핵심은 URL과 요청 feature를 보고 엔진 후보를 만든 뒤 fallback 순서로 시도하는 구조다. + +지원 엔진: + +- `index`: 기존 index/cache에서 문서 조회. +- `index;documents`: 문서 index 조회. +- `fire-engine;chrome-cdp`: 고급 브라우저 엔진. +- `fire-engine;chrome-cdp;stealth`: stealth proxy 브라우저 엔진. +- `fire-engine;tlsclient`: TLS client 기반 수집. +- `fire-engine;tlsclient;stealth`: stealth TLS client. +- `playwright`: 자체 Playwright microservice. +- `fetch`: HTTP fetch 기반 빠른 수집. +- `pdf`: PDF 전용 처리. +- `document`: DOCX/ODT/RTF/XLS/XLSX 등 문서 처리. +- `wikipedia`: Wikimedia 전용 엔진. +- `x-twitter`: X/Twitter 전용 엔진. + +Feature flag: + +- `actions`, `waitFor`, `screenshot`, `screenshot@fullScreen`, `pdf`, `document`, `audio`, `atsv`, `location`, `mobile`, `skipTlsVerification`, `useFastMode`, `stealthProxy`, `branding`, `disableAdblock`. + +선택 방식: + +1. URL 확장자, option, format을 보고 필요한 feature flag를 계산한다. +2. 각 엔진이 feature를 지원하는지 확인한다. +3. quality와 feature priority를 기준으로 fallback list를 만든다. +4. 엔진별 max reasonable time을 계산하고 timeout/abort manager와 함께 실행한다. +5. HTML을 Markdown으로 변환하고 metadata, links, images, screenshot, extract 결과 등을 조립한다. +6. 실패 시 `NoEnginesLeftError`, `DNSResolutionError`, `SSLError`, `PDFOCRRequiredError`, `ActionError`, `CrawlDenialError` 등 typed error로 전달한다. + +온톨로지 플랫폼에는 이 구조가 매우 유용하다. 특정 쇼핑몰/공식몰/문서/PDF마다 직접 fetcher를 분기하지 않고, Firecrawl이 feature 기반 fallback을 담당하게 할 수 있다. + +### 6.4 URL 발견과 사이트 크롤링 + +파일: `apps/api/src/scraper/WebScraper/crawler.ts` + +`WebCrawler`는 seed URL에서 사이트 내부 링크를 발견하고 filtering한다. + +주요 기능: + +- sitemap 로드와 sitemap 링크 제한. +- robots.txt 로드와 robots parser. +- max depth, max discovery depth 적용. +- include/exclude regex path filtering. +- backward crawling 차단. +- external link/subdomain 허용 여부 판단. +- query parameter 무시/중복 제거. +- 비웹 프로토콜, social/mailto, section anchor, 비문서 file type 제거. +- URL별 denial reason 생성. + +현재 프로젝트의 `crawler_platform.app.core.crawler.discovery`, `site_crawler`, `content_zone`, `fetchers`를 Firecrawl 방식으로 강화할 수 있다. 단, Python 코드에 직접 포팅하기보다는 `POST /v2/map` 또는 `POST /v2/crawl`을 호출해 URL discovery를 위임하는 것이 빠르다. + +### 6.5 Crawl 작업 처리 + +파일: `apps/api/src/controllers/v2/crawl.ts`, `apps/api/src/services/worker/scrape-worker.ts` + +Crawl은 단일 요청에서 모든 페이지를 즉시 반환하지 않는다. + +처리 절차: + +1. `crawlRequestSchema`로 URL, crawler option, scrape option 검증. +2. 자연어 `prompt`가 있으면 site structure를 일부 map한 뒤 LLM으로 crawler option 생성. +3. include/exclude regex 유효성 확인. +4. credit 또는 self-host 설정에 따라 limit 조정. +5. Redis/queue에 `StoredCrawl` 저장. +6. kickoff job을 큐에 넣고 `id`, status URL을 반환. +7. worker가 URL을 발견하고 개별 scrape job으로 확장한다. +8. `GET /v2/crawl/:jobId` 또는 websocket으로 진행률과 결과를 조회한다. + +응답 status: + +- `scraping` +- `completed` +- `failed` +- `cancelled` + +status 응답은 `completed`, `total`, `creditsUsed`, `expiresAt`, `next`, `data: Document[]`를 포함한다. + +### 6.6 Search + +파일: `apps/api/src/controllers/v2/search.ts`, `apps/api/src/search/*` + +Search는 검색 결과를 반환하고, 옵션에 따라 각 결과 페이지를 scrape해서 markdown까지 포함한다. + +온톨로지 플랫폼 활용: + +- 브랜드/상품/성분/카테고리 후보 URL 발견. +- 공식 문서, PDF, research 자료 검색. +- seed URL이 부족한 신규 도메인 bootstrap. +- `includeDomains`/`excludeDomains`로 신뢰 출처 제한. + +초기 MVP에서는 외부 검색 품질보다 `검색 결과 -> 후보 Source/Page -> 검토 큐` 흐름을 만드는 것이 중요하다. + +### 6.7 Parse + +`POST /v2/parse`는 multipart file upload를 받아 HTML/PDF/document를 scrape-like document로 변환한다. 파일 크기 제한은 50MB다. + +온톨로지 플랫폼 활용: + +- 로컬 PDF catalog, 제품 설명서, 성분표 문서 ingest. +- HTML fixture나 저장된 페이지 snapshot ingest. +- URL이 아닌 파일 기반 evidence 확보. + +### 6.8 Browser / Interact / Actions + +Firecrawl은 두 종류의 상호작용을 제공한다. + +- scrape request의 `actions`: scrape 전에 wait, click, write, press, scroll, screenshot, scrape 등을 수행한다. +- `POST /v2/scrape/:jobId/interact`: scrape job에 연결된 browser session에 code 또는 prompt 기반 조작을 수행한다. + +온톨로지 플랫폼 활용: + +- 쿠키 배너 닫기. +- 검색/필터/더보기 버튼 클릭. +- pagination 또는 lazy-loaded product list 수집. +- 특정 selector 대기 후 수집. + +주의: action 기반 수집은 재현성과 비용이 낮아질 수 있으므로, Source 단위 설정으로 제한하고 audit log를 남기는 것이 좋다. + +### 6.9 Monitor + +Monitor는 특정 URL/옵션을 주기적으로 실행하고 check 결과/diff를 관리하는 기능이다. + +온톨로지 플랫폼 활용: + +- 공식 상품 페이지 변경 감지. +- 성분/가격/품절/리뉴얼 페이지 모니터링. +- Claim evidence의 stale 여부 판단. + +초기 버전에서는 Firecrawl Monitor 전체를 들여오기보다, 현재 프로젝트의 `scheduler/update_policy.py`에서 Firecrawl scrape를 주기 호출하고 content hash/changeTracking을 저장하는 방식이 단순하다. + +### 6.10 SDK + +Python SDK는 `apps/python-sdk/firecrawl` 아래에 있으며, v2 메서드가 `/v2/scrape`, `/v2/crawl`, `/v2/map`, `/v2/search`, `/v2/batch/scrape`, `/v2/parse`, browser interaction을 감싼다. + +우리 프로젝트가 Firecrawl을 별도 서비스로 사용할 경우 Python SDK를 직접 사용하거나, 현재 FastAPI 서비스 내부에 얇은 adapter를 만드는 방식이 적합하다. + +## 7. 기능명세 + +### 7.1 Document 모델 + +Firecrawl의 핵심 결과 단위는 `Document`다. + +필드: + +- `title`, `description`, `url` +- `markdown`, `html`, `rawHtml` +- `links`, `images` +- `screenshot`, `audio` +- `json`, `extract`, `summary`, `answer`, `highlights`, `branding` +- `attributes`: selector/attribute/value 목록 +- `actions`: action 중 생성된 screenshot/scrape/javascript return/pdf +- `changeTracking`: 이전 scrape 대비 상태와 diff +- `metadata`: title, description, language, keywords, robots, OpenGraph, favicon, sourceURL, statusCode, scrapeId, contentType, proxyUsed, cacheState, cachedAt, creditsUsed 등 +- `serpResults`: search result title/description/url + +온톨로지 플랫폼 매핑: + +- `Document.url` -> `Page.url` +- `Document.markdown` -> `Page.cleaned_text` 또는 `Page.markdown` +- `Document.html/rawHtml` -> 저장 여부 선택. 기본은 저장하지 않고 hash만 저장 권장. +- `Document.metadata.sourceURL/statusCode/contentType` -> `Page.fetch_status`, `Page.metadata` +- `Document.links/images` -> discovery 후보와 media evidence +- `Document.json/extract` -> 도메인 extractor 입력 또는 사전 추출값 +- `Document.changeTracking` -> claim freshness/update scheduling + +### 7.2 Scrape API 명세 + +Endpoint: `POST /v2/scrape` + +목적: 단일 URL을 LLM-ready document로 변환한다. + +필수 입력: + +- `url`: HTTP/HTTPS URL. protocol이 없으면 `http://`를 보정한다. + +선택 입력: + +- 공통 Scrape 옵션 전체. +- `origin`: 요청 출처 tag. 기본 `api`. +- `integration`: 외부 통합 정보. +- `zeroDataRetention`: 데이터 보존 제한. + +정상 응답: + +```json +{ + "success": true, + "data": { + "markdown": "...", + "metadata": { + "sourceURL": "https://example.com", + "statusCode": 200, + "proxyUsed": "basic" + } + } +} +``` + +실패 응답: + +```json +{ + "success": false, + "code": "ERROR_CODE", + "error": "message" +} +``` + +우리 플랫폼 수용 기준: + +- URL 단위 수집의 기본 provider는 Firecrawl scrape로 한다. +- 기본 format은 `markdown`, 필요 시 `html`, `links`, `images`, `screenshot`, `json`을 Source config에서 켠다. +- extractor는 Firecrawl JSON을 그대로 신뢰하기보다, `markdown + metadata + sourceURL`을 현재 ontology extractor에 넣어 Claim/Evidence를 만든다. + +### 7.3 Crawl API 명세 + +Endpoint: `POST /v2/crawl` + +목적: seed URL에서 여러 URL을 발견하고 각 페이지를 scrape한다. + +필수 입력: + +- `url` + +선택 입력: + +- Crawler 옵션: `includePaths`, `excludePaths`, `limit`, `maxDiscoveryDepth`, `allowExternalLinks`, `allowSubdomains`, `ignoreRobotsTxt`, `sitemap`, `delay` 등. +- `scrapeOptions`: 각 페이지에 적용할 scrape 옵션. +- `webhook`: 상태 통지. +- `maxConcurrency` +- `prompt`: 자연어로 crawler option 생성. + +정상 응답: + +```json +{ + "success": true, + "id": "job-id", + "url": "http://host/v2/crawl/job-id" +} +``` + +Status 조회: + +```json +{ + "success": true, + "status": "scraping", + "completed": 12, + "total": 100, + "creditsUsed": 12, + "expiresAt": "...", + "data": [] +} +``` + +우리 플랫폼 수용 기준: + +- 사이트 전체 수집은 `crawl-site` 내부 구현을 Firecrawl crawl 호출로 대체 또는 선택 가능하게 한다. +- 결과 Document[]는 Page 단위로 upsert하고, 각 Page를 ontology extraction queue로 넘긴다. +- `includePaths/excludePaths`는 Source config의 `url_patterns`로 매핑한다. + +### 7.4 Map API 명세 + +Endpoint: `POST /v2/map` + +목적: scrape 없이 URL 후보 목록만 빠르게 발견한다. + +입력: + +- `url` +- `search`: path/title 검색 조건. +- `sitemap`: `only`, `include`, `skip` +- `limit`: 기본 5000, 최대 100000 +- `includeSubdomains`, `allowExternalLinks`, `ignoreQueryParameters`, `filterByPath` +- `useIndex`, `ignoreCache` + +응답: + +```json +{ + "success": true, + "links": [ + { "url": "https://example.com/a", "title": "...", "description": "..." } + ] +} +``` + +우리 플랫폼 수용 기준: + +- 신규 Source 등록 시 먼저 Map을 실행해 수집 범위 preview를 보여준다. +- 사용자가 선택한 URL 패턴을 config로 저장한다. +- 대규모 크롤 전에 Map 결과로 예상 page 수와 domain/path 분포를 산출한다. + +### 7.5 Batch Scrape API 명세 + +Endpoint: `POST /v2/batch/scrape` + +목적: URL 배열을 비동기 scrape job으로 처리한다. + +입력: + +- `urls`: 1개 이상 URL 배열. +- 공통 Scrape 옵션. +- `webhook`, `appendToId`, `ignoreInvalidURLs`, `maxConcurrency`, `zeroDataRetention`. + +응답: + +```json +{ + "success": true, + "id": "job-id", + "url": "http://host/v2/batch/scrape/job-id", + "invalidURLs": [] +} +``` + +우리 플랫폼 수용 기준: + +- Map으로 발견한 URL 중 우선순위가 높은 URL 묶음을 batch scrape로 실행한다. +- 실패 URL은 `crawl_errors` 또는 Page status로 저장하고 재시도 정책에 연결한다. + +### 7.6 Search API 명세 + +Endpoint: `POST /v2/search` + +목적: query 기반으로 web/news/images 결과를 얻고, 필요하면 결과 페이지 내용까지 scrape한다. + +입력: + +- `query` +- `limit`, `sources`, `categories`, `includeDomains`, `excludeDomains` +- `lang`, `country`, `location` +- `timeout` +- `asyncScraping` +- `scrapeOptions` + +응답: + +```json +{ + "success": true, + "id": "job-id", + "creditsUsed": 3, + "data": { + "web": [ + { + "url": "https://example.com", + "title": "Example", + "description": "...", + "markdown": "..." + } + ] + } +} +``` + +우리 플랫폼 수용 기준: + +- 자동 Source 후보 발굴 기능에 사용한다. +- `includeDomains`를 우선 사용해 공식몰/공식 문서/신뢰 출처 탐색을 제한한다. +- 검색 결과는 즉시 Claim으로 넣지 않고 Source/Page 후보 검토 큐로 넣는다. + +### 7.7 Parse API 명세 + +Endpoint: `POST /v2/parse` + +목적: 업로드 파일을 Document로 변환한다. + +입력: + +- multipart field `file` +- 공통 Scrape 옵션. +- 파일 최대 50MB. +- kind: `html`, `pdf`, `document`. + +우리 플랫폼 수용 기준: + +- 로컬 catalog/PDF/manual ingest에 사용한다. +- parse 결과는 URL source가 없을 수 있으므로 `Source.type=file`, `Page.url=file://...` 또는 별도 `documents` 테이블 정책을 정한다. + +### 7.8 Monitor API 명세 + +Endpoint 집합: + +- `POST /v2/monitor` +- `GET /v2/monitor` +- `GET /v2/monitor/:monitorId` +- `PATCH /v2/monitor/:monitorId` +- `DELETE /v2/monitor/:monitorId` +- `POST /v2/monitor/:monitorId/run` +- `GET /v2/monitor/:monitorId/checks` +- `GET /v2/monitor/:monitorId/checks/:checkId` + +목적: 특정 수집 대상의 변경을 주기적으로 감시한다. + +우리 플랫폼 수용 기준: + +- MVP에서는 직접 도입하지 않고, Firecrawl `changeTracking` 또는 주기 scrape 결과의 content hash 비교로 대체한다. +- 장기적으로 Claim freshness, 상품 리뉴얼, 가격/품절 변경에 연결한다. + +### 7.9 Browser Session API 명세 + +Endpoint: + +- `POST /v2/browser` +- `GET /v2/browser` +- `POST /v2/browser/:sessionId/execute` +- `DELETE /v2/browser/:sessionId` +- `POST /v2/scrape/:jobId/interact` +- `DELETE /v2/scrape/:jobId/interact` + +목적: 브라우저 세션을 생성하고 code/prompt 기반으로 조작한다. + +우리 플랫폼 수용 기준: + +- Source config에 `actions`를 저장하는 형태를 우선한다. +- 자유로운 browser execute는 보안/재현성 위험이 있으므로 관리자 전용 디버깅 기능으로 제한한다. + +## 8. Self-host 구성 + +`docker-compose.yaml` 기준 서비스: + +- `api`: Express API와 worker harness. +- `playwright-service`: 브라우저 수집 microservice. +- `redis`: queue/rate limit/cache. +- `rabbitmq`: NuQ worker messaging. +- `nuq-postgres`: NuQ 상태 저장. + +필수/주요 환경 변수: + +- `PORT`, `HOST` +- `USE_DB_AUTHENTICATION=false`로 self-host API key 없이 사용 가능 +- `REDIS_URL`, `REDIS_RATE_LIMIT_URL` +- `PLAYWRIGHT_MICROSERVICE_URL` +- `NUQ_RABBITMQ_URL` +- `POSTGRES_USER`, `POSTGRES_PASSWORD`, `POSTGRES_DB`, `POSTGRES_HOST`, `POSTGRES_PORT` +- `OPENAI_API_KEY`, `OPENAI_BASE_URL`, `MODEL_NAME`, `OLLAMA_BASE_URL` 등 AI 기능용 +- `PROXY_SERVER`, `PROXY_USERNAME`, `PROXY_PASSWORD` +- `SEARXNG_ENDPOINT` +- `BULL_AUTH_KEY` +- `MAX_CPU`, `MAX_RAM` +- `ALLOW_LOCAL_WEBHOOKS` + +Self-host 제한: + +- Fire-engine 고급 기능은 cloud/internal 의존성이 있어 자체 호스팅에서 제한될 수 있다. +- 기본적으로 fetch + Playwright + PDF/document 처리 중심으로 보는 것이 현실적이다. +- Supabase/Stripe/Autumn/GCS/Sentry 의존 영역은 자체 플랫폼에서는 제거하거나 stub 처리해야 한다. + +## 9. 현재 프로젝트와의 통합 설계 + +현재 프로젝트는 Python/FastAPI 기반이며 핵심 구조는 다음과 같다. + +- `crawler_platform/app/core/crawler`: fetch/discovery/clean/pipeline. +- `crawler_platform/app/core/extractor`: rule-based/AI extractor. +- `crawler_platform/app/core/ontology`: entity, relation, claim, triple store. +- `crawler_platform/app/core/database`: SQLAlchemy 저장소. +- `crawler_platform/app/core/research`: graph research loop. +- `crawler_platform/app/api/routes.py`: 관리 API. +- `configs/perfume_subscription.yaml`: Source/domain config. + +권장 통합 방식: + +```mermaid +flowchart LR + A["Crawler Platform FastAPI"] --> B["Firecrawl Adapter"] + B --> C["Firecrawl Self-host API"] + C --> D["Scrape / Map / Crawl / Search"] + D --> E["Document"] + E --> F["Page Upsert"] + F --> G["Ontology Extractor"] + G --> H["Entity / Claim / Evidence"] +``` + +### 단계별 채택 계획 + +1. `FirecrawlClient` adapter 추가 + - Python SDK 또는 HTTP client 사용. + - `scrape_url`, `map_site`, `crawl_site`, `batch_scrape`, `search_sources`, `parse_file` 메서드 제공. + +2. Source config 확장 + - `fetcher: firecrawl` + - `firecrawl.formats` + - `firecrawl.actions` + - `firecrawl.crawler_options` + - `firecrawl.scrape_options` + - `firecrawl.search_options` + +3. Page 저장 모델 확장 + - `markdown` + - `raw_html_hash` + - `metadata_json` + - `scrape_id` + - `content_type` + - `cache_state` + - `change_status` + +4. 기존 extractor 연결 + - Firecrawl Document의 `markdown`을 표준 입력으로 사용. + - `metadata.sourceURL`을 evidence source로 유지. + - `links/images`를 다음 discovery 후보 또는 evidence attachment로 저장. + +5. UI 기능 추가 + - Map preview. + - Crawl job status. + - Page markdown preview. + - 실패 URL과 denial reason 표시. + +## 10. 재사용 우선순위 + +### 즉시 재사용 권장 + +- v2 API 명세와 옵션 체계. +- `scrape`, `map`, `crawl`, `batch scrape`. +- Document 결과 모델. +- Python SDK 또는 HTTP adapter. +- Docker Compose self-host 실행 방식. + +### 부분 재사용 권장 + +- `actions`: Source별 필요한 경우에만. +- `search`: Source 후보 발굴 단계. +- `parse`: 파일 ingest 단계. +- `monitor/changeTracking`: 업데이트 감지 단계. +- `crawler.ts`의 denial reason/URL filtering 정책: Python config validation에 반영. + +### 초기 제외 권장 + +- billing/credit/team/account 기능. +- x402 micropayment. +- support proxy. +- agent signup. +- Supabase/Stripe/Autumn/GCS/Sentry SaaS 운영 코드. +- Fire-engine cloud 의존 기능. + +## 11. 온톨로지 플랫폼 기능명세 초안 + +Firecrawl을 기반 수집기로 사용할 때 범용 온톨로지 구축 플랫폼은 다음 기능을 가져야 한다. + +### 11.1 Source 등록 + +입력: + +- source name +- base URL 또는 seed URL 목록 +- source type: official, marketplace, review, document, search +- fetcher: `requests`, `playwright`, `firecrawl` +- Firecrawl crawler/scrape/search options +- 신뢰도 기본값 +- robots 준수 정책 +- update schedule + +출력: + +- Source record +- Map preview 결과 +- 예상 page count + +### 11.2 URL 발견 + +기능: + +- Firecrawl Map 호출. +- sitemap only/include/skip 선택. +- include/exclude path regex 적용. +- subdomain/external link 정책 적용. +- query parameter 무시 여부. +- URL 후보를 Page 상태 `discovered`로 저장. + +검증: + +- URL 중복 제거. +- domain/path scope 위반 차단. +- robots 차단 URL 표시. + +### 11.3 페이지 수집 + +기능: + +- 단일 URL scrape. +- 다중 URL batch scrape. +- site crawl job 실행. +- Markdown, metadata, links, images 저장. +- HTML 원문 저장 여부 선택. +- screenshot 선택 저장. + +상태: + +- discovered +- queued +- fetching +- fetched +- failed +- blocked_by_robots +- skipped + +### 11.4 문서/파일 수집 + +기능: + +- PDF/DOCX/HTML 파일 업로드. +- Firecrawl Parse 호출. +- 문서 metadata와 markdown 저장. +- 파일 기반 evidence source 생성. + +### 11.5 구조화 추출 + +기능: + +- Firecrawl JSON format 또는 현재 Python extractor 선택. +- 기본 경로는 `Document.markdown -> ontology extractor`. +- JSON Schema 기반 추출은 도메인별 schema를 사용. +- 추출 결과는 바로 확정하지 않고 Claim/Evidence로 저장. + +### 11.6 Claim/Evidence 생성 + +기능: + +- Entity 후보 생성. +- subject-predicate-object Claim 생성. +- Evidence text와 source URL, page id, selector 또는 markdown span 저장. +- confidence score 산출. +- Source 신뢰도와 추출 방식에 따른 confidence 조정. + +### 11.7 변경 감지 + +기능: + +- Page content hash 비교. +- Firecrawl changeTracking format 사용 가능. +- 변경된 페이지만 재추출. +- 삭제/숨김/동일/변경 상태 기록. + +### 11.8 검토 UI + +기능: + +- Source별 Map preview. +- Crawl job 진행률. +- Page markdown 미리보기. +- Claim 목록과 evidence 확인. +- confidence 수동 조정. +- entity merge. +- 실패 URL과 denial reason 확인. + +## 12. 리스크와 주의점 + +- 라이선스: Firecrawl root LICENSE는 AGPL 계열로 보인다. 소스 자체를 서비스에 내장/수정 배포할 경우 공개 의무가 발생할 수 있으므로 별도 확인이 필요하다. +- 언어/스택 차이: 현재 프로젝트는 Python, Firecrawl 핵심은 TypeScript다. 직접 코드 병합보다 서비스 분리가 적합하다. +- 운영 복잡도: Redis, RabbitMQ, Postgres, Playwright service가 필요하다. +- Cloud 의존 기능: Fire-engine, 일부 index/search/branding/agent 기능은 self-host에서 제한될 수 있다. +- 비용/속도: Playwright/action/screenshot/stealth는 비용이 크다. Source별 정책이 필요하다. +- 데이터 보존: raw HTML/screenshot/audio 저장은 개인정보/저작권/용량 이슈가 있으므로 기본 off 권장. +- 검색 결과 신뢰도: Search는 후보 발굴용이며 Claim 근거로 바로 쓰면 안 된다. + +## 13. 구현 권장안 + +최초 구현은 다음 범위가 좋다. + +1. Firecrawl self-host를 별도 Docker Compose로 실행한다. +2. Python 프로젝트에 `FirecrawlAdapter`를 만든다. +3. `POST /crawl` 또는 CLI `crawl-url`에 `fetcher=firecrawl` 옵션을 추가한다. +4. 단일 URL scrape 결과의 markdown을 기존 extractor로 넘긴다. +5. Map preview와 batch scrape는 두 번째 단계에서 붙인다. +6. Monitor/changeTracking/search/parse는 세 번째 단계에서 붙인다. + +이 방식이면 Firecrawl의 강한 수집 능력을 거의 변형 없이 사용하면서, 현재 프로젝트의 핵심 가치인 범용 온톨로지/Claim/Evidence/추천 구조는 Python 코드에 유지할 수 있다. + diff --git a/오픈소스분석자료/Guardrails_분석_및_기능명세.md b/오픈소스분석자료/Guardrails_분석_및_기능명세.md new file mode 100644 index 0000000..1a642ee --- /dev/null +++ b/오픈소스분석자료/Guardrails_분석_및_기능명세.md @@ -0,0 +1,694 @@ +# Guardrails 프로젝트 분석 및 기능명세 + +분석 대상: `C:\Users\lasta\MyProject\AI\참고\guardrails-main` +분석일: 2026-05-13 +목적: 범용 온톨로지 구축 플랫폼에서 LLM 생성 결과의 구조화, 검증, 실패 복구, 재질문, 운영형 검증 API의 기준 소스로 활용하기 위한 상세 분석 + +## 1. 프로젝트 개요 + +Guardrails는 Python 기반 LLM 신뢰성/구조화 출력 프레임워크이다. 핵심 목표는 LLM 입출력에 검증 규칙을 적용하고, 실패 시 정해진 정책에 따라 수정, 필터링, 예외, 재질문을 수행하며, 최종적으로 애플리케이션이 신뢰할 수 있는 구조화 데이터를 받도록 하는 것이다. + +이 프로젝트는 온톨로지 구축 플랫폼에서 특히 유용하다. 온톨로지 생성 과정은 개념, 클래스, 속성, 관계, 제약조건, 근거 문장, provenance 같은 구조화 산출물이 필요하고, LLM 응답이 스키마를 어기거나 부정확한 관계를 만들면 이후 그래프 저장소와 추론 엔진까지 오염된다. Guardrails는 이 지점에서 “LLM 출력 검증 게이트” 역할을 그대로 수행할 수 있다. + +주요 특징은 다음과 같다. + +- `Guard` 중심의 LLM 호출 래퍼 및 검증 실행 +- RAIL XML, Pydantic 모델, JSON Schema, 문자열 기반 출력 스키마 지원 +- validator 기반 입력/출력 검증 +- 실패 시 `reask`, `fix`, `filter`, `refrain`, `noop`, `exception`, `fix_reask`, `custom` 정책 적용 +- JSON 출력 파싱, 타입 보정, 추가 키 제거, JSON Schema 검증 +- 검증 실패 영역만 재질문하는 reask 루프 +- 동기/비동기/스트리밍 검증 실행 +- Guardrails Hub validator 설치/등록 체계 +- CLI 및 독립 서버 실행 지원 +- LangChain, LlamaIndex, LiteLLM, OpenAI, HuggingFace 등 LLM/프레임워크 연동 +- telemetry, history, validator log 기반 실행 추적 +- Text2SQL, document store, vector DB 같은 예시 애플리케이션 제공 + +## 2. 기술 스택 + +- 언어: Python 3.10 이상 +- 데이터 모델: Pydantic v2, dataclass +- 스키마/파싱: JSON Schema 2020-12, lxml, jsonschema, jsonref +- LLM 연동: OpenAI SDK, LiteLLM, HuggingFace, Manifest optional +- CLI: Typer, Click, Rich +- 재시도/실행 보조: tenacity, contextvars +- 벡터 검색 optional: FAISS, numpy +- SQL optional: SQLAlchemy, sqlvalidator, sqlglot +- 관측성: OpenTelemetry, Guardrails Hub telemetry +- 프레임워크 통합: LangChain Core Runnable, LlamaIndex +- 서버 optional: `guardrails-api` +- 라이선스: Apache License 2.0 + +## 3. 최상위 구조 + +```text +guardrails-main/ + guardrails/ # SDK 본체 + guard.py # Guard 메인 엔트리포인트 + async_guard.py # AsyncGuard + validator_base.py # Validator 베이스 및 registry + validator_service/ # 동기/비동기 validator 실행 엔진 + run/ # Runner, StreamRunner, AsyncRunner + schema/ # RAIL/Pydantic/primitive schema 처리 + actions/ # reask/filter/refrain 실패 처리 객체 + classes/ # history, validation outcome/logs, execution models + llm_providers.py # LLM 호출 어댑터 + formatters/ # JSON formatter, JSONFormer 어댑터 + cli/ # configure/create/start/validate/hub/db/watch + hub/ # Hub validator install/registry + integrations/ # LangChain, LlamaIndex, Databricks + applications/ # Text2SQL 예시 + document_store.py # 문서/페이지 저장 및 vector DB 검색 추상화 + vectordb/ # VectorDBBase, Faiss + docs/ # 공식 문서, 예제, API reference + tests/ # 단위/통합 테스트 + server_ci/ # 서버 Docker/CI 검증 구성 +``` + +패키지의 실질적인 공개 API는 `Guard`, `AsyncGuard`, `Validator`, `OnFailAction`, `ValidationOutcome`, validator registry, schema 변환 함수군이다. +우리 프로젝트에서 기준 소스로 삼을 우선순위는 `guard.py`, `validator_base.py`, `validator_service/`, `run/`, `schema/`, `actions/`, `classes/validation*`, `classes/history/` 순서가 적절하다. + +## 4. 핵심 런타임 아키텍처 + +### 4.1 기본 실행 흐름 + +```mermaid +flowchart TD + A["사용자: Guard 생성"] --> B["스키마 로드: RAIL/Pydantic/String/JSON Schema"] + B --> C["validator map 구성"] + C --> D["Guard.__call__ 또는 Guard.parse"] + D --> E["Runner 생성"] + E --> F["입력 메시지 검증"] + F --> G["LLM 호출 또는 기존 llm_output 사용"] + G --> H["출력 파싱: JSON/string"] + H --> I["JSON Schema 검증 및 타입 보정"] + I --> J["validator_service.validate"] + J --> K{"실패 발생?"} + K -- no --> L["ValidationOutcome 반환"] + K -- yes --> M["OnFailAction 적용"] + M --> N{"reask 필요?"} + N -- yes --> O["reask 메시지/부분 스키마 생성"] + O --> G + N -- no --> L +``` + +### 4.2 핵심 객체 관계 + +- `Guard`: 사용자 진입점. 스키마, validator, 실행 옵션, history, server/client 여부를 가진다. +- `Runner`: 한 번의 Guard 호출을 실제로 실행한다. LLM 호출, 파싱, 스키마 검증, validator 실행, reask 반복을 담당한다. +- `Validator`: 값 하나를 검증하는 규칙 단위이다. `_validate()`를 구현하고 `PassResult` 또는 `FailResult`를 반환한다. +- `ValidatorServiceBase`: validator 실행 전후 로그, 실패 정책 적용, 여러 validator 결과 병합을 담당한다. +- `SequentialValidatorService`: 동기 Guard에서 validator를 순차 실행한다. +- `AsyncValidatorService`: async validator를 병렬 실행하고 결과를 병합한다. +- `ValidationOutcome`: 최종 결과 DTO. 원본 LLM 출력, 검증된 출력, reask, 통과 여부, 오류, 검증 요약을 포함한다. +- `Call`, `Iteration`, `Inputs`, `Outputs`: 실행 history와 단계별 로그 모델이다. +- `ProcessedSchema`: RAIL/Pydantic/primitive 입력을 JSON Schema, validator 목록, validator map, execution options로 변환한 결과이다. + +## 5. Guard 기능명세 + +### 5.1 Guard 생성 방식 + +`Guard`는 네 가지 생성 경로를 제공한다. + +| 생성 방식 | 함수 | 입력 | 용도 | +| --- | --- | --- | --- | +| 기본 생성 후 validator 추가 | `Guard().use(...)` | validator 인스턴스 | 단순 문자열/출력 검증 | +| RAIL 파일 | `Guard.for_rail(path)` | `.rail` 파일 경로 | XML 기반 스키마/프롬프트/validator 정의 | +| RAIL 문자열 | `Guard.for_rail_string(xml)` | RAIL XML 문자열 | DB/설정에서 동적 로드 | +| Pydantic 모델 | `Guard.for_pydantic(model)` | Pydantic BaseModel 또는 모델 리스트 | Python 타입 기반 구조화 출력 | +| 문자열 출력 | `Guard.for_string(validators)` | validator 목록 | 일반 텍스트 응답 검증 | + +온톨로지 플랫폼에서는 `Guard.for_pydantic()`을 우선 기준으로 삼는 것이 좋다. 개념 추출, 관계 추출, 속성 정규화, 증거 문장 연결 등은 Pydantic 모델로 명확히 표현할 수 있고, 이 모델을 그대로 API contract와 테스트 fixture에 재사용할 수 있다. RAIL은 사용자 정의 DSL/설정 파일 기반 검증을 지원할 때 보조 수단으로 쓰는 편이 적절하다. + +### 5.2 Guard 실행 방식 + +| 실행 방식 | 함수 | 설명 | +| --- | --- | --- | +| LLM 호출 포함 | `guard(llm_api=..., messages=..., prompt_params=...)` | Guard가 LLM 호출부터 검증까지 수행 | +| 기존 출력 검증 | `guard.parse(llm_output=...)` | 이미 생성된 LLM 출력 문자열을 파싱/검증 | +| 별칭 | `guard.validate(llm_output)` | `parse()`와 동일 | +| 서버 모드 | `settings.use_server=True` 또는 `Guard(use_server=True)` | Guardrails API 서버에 검증 위임 | +| 스트리밍 | `stream=True` | `StreamRunner` 사용 | + +`__call__`은 기본적으로 `messages`가 필요하다. 이미 응답을 보유한 후처리 파이프라인에서는 `parse()`를 사용해야 한다. + +### 5.3 Guard 입력/출력 계약 + +입력 주요 필드: + +- `llm_api`: OpenAI/LiteLLM/HuggingFace/사용자 callable +- `messages`: chat message 목록 +- `prompt_params`: prompt template 치환 값 +- `metadata`: validator에 전달되는 외부 컨텍스트 +- `num_reasks`: 검증 실패 시 재질문 최대 횟수 +- `full_schema_reask`: 실패 필드만 물을지 전체 스키마를 다시 생성할지 결정 +- `llm_output`: LLM 호출 없이 검증할 기존 출력 + +출력 `ValidationOutcome`: + +- `rawLlmOutput`: 원본 LLM 문자열 +- `validatedOutput`: 검증과 보정이 반영된 최종 값 +- `validationPassed`: 최종 통과 여부 +- `reask`: 재질문이 필요한 실패 객체 +- `validationSummaries`: 실패 validator 요약 +- `error`: 실행 중 오류 +- `callId`: history 식별자 + +## 6. Schema 기능명세 + +### 6.1 RAIL 처리 + +`guardrails/schema/rail_schema.py`는 RAIL XML을 JSON Schema와 validator map으로 변환한다. + +지원되는 주요 RAIL 타입: + +- `string` +- `integer` +- `float` +- `bool` +- `date` +- `time` +- `datetime` +- `percentage` +- `enum` +- `list` +- `object` +- `choice` + +RAIL 요소의 `validators` 속성은 validator 문자열을 파싱하고, `on-fail-*` 속성은 validator별 실패 정책으로 변환된다. 객체 필드는 JSON path 형태의 validator map에 연결된다. 예를 들어 `$.entities.*.label` 같은 경로에 특정 validator를 붙일 수 있다. + +온톨로지 플랫폼 적용: + +- RAIL은 운영자가 UI에서 검증 정책을 XML/DSL로 저장하는 기능을 만들 때 유용하다. +- 초기 구현에서는 Pydantic 모델 기반 스키마를 우선하고, RAIL은 “고급 사용자/템플릿 import” 기능으로 뒤에 붙이는 것이 안정적이다. + +### 6.2 Pydantic 처리 + +`guardrails/schema/pydantic_schema.py`는 Pydantic 모델을 JSON Schema로 변환하고, 필드별 validator metadata를 추출한다. + +적용 가능한 온톨로지 모델 예: + +```python +class OntologyEntity(BaseModel): + id: str + label: str + type: Literal["class", "individual", "property"] + description: str + evidence: list[str] + +class OntologyRelation(BaseModel): + source_id: str + predicate: str + target_id: str + confidence: float + evidence: list[str] +``` + +이런 모델을 `Guard.for_pydantic()`에 넣으면 LLM 응답을 JSON 구조로 강제하고, 누락 필드/타입 오류/추가 키/validator 실패를 한 실행 흐름 안에서 처리할 수 있다. + +### 6.3 Primitive/String 처리 + +`primitive_to_schema()`는 단순 문자열 또는 기본 타입 검증용 schema를 만든다. 문서 요약, label 후보, relation predicate 후보처럼 단일 문자열 출력을 검증할 때 적합하다. + +## 7. Validator 기능명세 + +### 7.1 Validator 기본 구조 + +`Validator`는 모든 검증 규칙의 베이스 클래스이다. + +필수 구현: + +- `_validate(value, metadata) -> ValidationResult` + +선택 구현: + +- `_inference_local(model_input)` +- `_inference_remote(model_input)` +- `async_validate(value, metadata)` +- `validate_stream(...)` +- `async_validate_stream(...)` + +반환 타입: + +- `PassResult`: 검증 성공. 선택적으로 `value_override`, `validated_chunk`, `metadata` 포함 +- `FailResult`: 검증 실패. `error_message`, `fix_value`, `metadata` 포함 + +Validator 인스턴스는 `rail_alias`로 registry에 등록되어야 한다. Guard는 validator reference를 `id`, `on`, `on_fail`, `kwargs` 형태로 직렬화한다. + +### 7.2 Validator 실행 위치 + +Validator는 다음 위치에 붙을 수 있다. + +- `output` 또는 `$`: 전체 출력 +- `messages`: 입력 메시지 +- JSON path: `$.field`, `$.items.*.name` 등 구조화 출력의 특정 필드 + +온톨로지 플랫폼에서는 다음 경로 매핑이 중요하다. + +| 경로 | 검증 예 | +| --- | --- | +| `$` | 전체 ontology extraction result가 최소 엔티티/관계를 포함하는지 | +| `$.entities.*.id` | ID 형식, 중복 여부 | +| `$.entities.*.label` | 빈 문자열 금지, 길이 제한, 금칙어 | +| `$.relations.*.source_id` | 존재하는 entity ID인지 | +| `$.relations.*.predicate` | 허용 ontology predicate인지 | +| `$.relations.*.confidence` | 0.0~1.0 범위 | +| `$.relations.*.evidence.*` | 원문에 존재하는 근거 문장인지 | + +### 7.3 OnFailAction 명세 + +| 액션 | 동작 | 온톨로지 적용 | +| --- | --- | --- | +| `exception` | 즉시 예외 발생 | 저장 전 엄격 검증, 배치 실패 처리 | +| `noop` | 실패해도 원본 유지 | soft warning만 남길 때 | +| `fix` | `FailResult.fix_value`로 교체 | label trim, confidence clipping | +| `fix_reask` | fix 후 재검증, 실패하면 reask | 자동 보정 가능하지만 위험한 필드 | +| `reask` | 실패 위치를 ReAsk 객체로 표시 | 관계/근거/타입 오류 재생성 | +| `filter` | 실패 값을 제거 | 부적합 entity/relation 삭제 | +| `refrain` | 전체 응답을 비움 | 안전성/정책 위반 시 결과 폐기 | +| `custom` | 사용자 함수 호출 | 그래프 DB 조회 기반 보정 | + +온톨로지 구축에서는 `exception`보다 `reask`, `filter`, `fix`, `custom`의 조합이 실용적이다. 예를 들어 relation의 source/target ID가 존재하지 않으면 `reask`, confidence 범위 오류는 `fix`, evidence가 원문에 없으면 `filter` 또는 `reask`가 적합하다. + +## 8. Runner 및 ReAsk 기능명세 + +### 8.1 Runner 단계 + +`Runner.step()`은 다음 순서로 실행된다. + +1. `Inputs`, `Outputs`, `Iteration` 생성 +2. 입력 메시지 준비 및 입력 validator 실행 +3. LLM API 호출 또는 전달받은 `llm_output` 사용 +4. 원본 출력 파싱 +5. JSON Schema 검증 +6. validator map 기반 검증 +7. 실패 정책 후처리 +8. reask 객체 수집 +9. reask가 있고 예산이 남으면 다음 loop 준비 + +### 8.2 ReAsk 처리 + +`guardrails/actions/reask.py`는 실패 유형을 다음 객체로 표현한다. + +- `FieldReAsk`: 특정 필드 값 검증 실패 +- `SkeletonReAsk`: 전체 구조/schema 검증 실패 +- `NonParseableReAsk`: LLM 출력 파싱 실패 + +`get_reask_setup()`은 실패 객체, 기존 출력, 스키마, validator map을 바탕으로 다음 LLM 호출에 사용할 메시지와 스키마를 만든다. 부분 reask가 가능하면 실패 필드만 다시 요청하고, `full_schema_reask=True`이면 전체 구조를 다시 요청한다. + +온톨로지 플랫폼 적용: + +- entity/relation 한두 개 필드 오류는 부분 reask가 비용과 품질 면에서 유리하다. +- Pydantic 모델 기반 전체 ontology extraction은 `full_schema_reask=True`가 안정적인 경우가 많다. +- production에서는 reask 횟수를 1~2회로 제한하고, 실패한 relation만 “검토 필요” 큐로 보내는 정책이 좋다. + +## 9. ValidatorService 기능명세 + +### 9.1 동기 실행 + +`SequentialValidatorService`는 validator를 순차 실행한다. 동기 Guard에서 async validator를 사용하면 명시적으로 오류를 낸다. 스트리밍 검증에서는 chunk 누적, validator별 partial accumulator, fix 결과 병합을 수행한다. + +### 9.2 비동기 실행 + +`AsyncValidatorService`는 같은 경로에 붙은 validator들을 `asyncio.gather()`로 병렬 실행한다. 결과 처리 규칙은 다음과 같다. + +- `Filter` 또는 `Refrain`이 나오면 즉시 해당 값 반환 +- `FieldReAsk`가 여러 개면 fail result를 병합 +- `fix`, `fix_reask`, `custom` 결과가 여러 개면 diff/merge 로직으로 병합 +- child object/list는 재귀적으로 검증 + +온톨로지 플랫폼에서 원문 근거 확인, 외부 사전 조회, 그래프 DB 중복 조회, embedding similarity 검증처럼 I/O가 많은 validator는 async 기반으로 구현하는 것이 좋다. + +## 10. LLM Provider 및 Formatter 명세 + +### 10.1 LLM 호출 어댑터 + +`llm_providers.py`는 여러 호출 방식을 `PromptCallableBase` 형태로 감싼다. + +지원 범주: + +- OpenAI 호환 callable +- LiteLLM +- Manifest +- HuggingFace model/pipeline +- 임의 Python callable +- async callable + +`get_llm_ask()`와 `get_async_llm_ask()`는 전달된 `llm_api`, `model`, kwargs를 보고 적절한 callable wrapper를 선택한다. + +### 10.2 구조화 출력 Formatter + +`formatters/json_formatter.py`는 JSON Schema를 기반으로 구조화 생성을 보조한다. Pydantic 기반 Guard에서 `output_formatter="jsonformer"` 같은 방식으로 formatter를 붙일 수 있다. + +온톨로지 플랫폼에서는 모델별 structured output 기능이 다르므로 다음 순서로 적용하는 것이 좋다. + +1. 모델이 native JSON Schema/function calling을 지원하면 provider native 기능 사용 +2. 그렇지 않으면 Guardrails prompt suffix와 JSON 파싱/검증 사용 +3. 로컬 HuggingFace 모델에는 JSONFormer 같은 formatter 검토 + +## 11. CLI 및 서버 기능명세 + +### 11.1 CLI 명령 + +`guardrails.cli`는 다음 명령군을 제공한다. + +- `guardrails configure`: `.guardrailsrc` 설정 및 Hub token/telemetry 설정 +- `guardrails create`: validator 목록으로 config 템플릿 생성 +- `guardrails start`: Guardrails API 서버 실행 +- `guardrails validate`: RAIL과 LLM 출력 파일 기반 검증 +- `guardrails hub install/list/uninstall/submit`: Hub validator 관리 +- `guardrails db upgrade/downgrade`: DB migration +- `guardrails watch`: 개발 보조 + +온톨로지 플랫폼에서는 CLI를 직접 노출하기보다 내부 관리 명령 또는 admin API로 래핑하는 것이 좋다. + +### 11.2 서버 모드 + +README와 `guardrails/cli/start.py` 기준으로 Guardrails는 `guardrails-api` 패키지가 설치되어 있으면 독립 서버로 실행될 수 있다. 서버는 Guard 설정을 로드하고 REST API 또는 OpenAI 호환 endpoint로 검증을 제공한다. + +적용 방안: + +- 단일 애플리케이션 초기 단계: 라이브러리 내장 방식 권장 +- 여러 서비스가 공통 검증 정책을 공유하는 단계: Guardrails 서버를 별도 배포 +- SaaS형 온톨로지 플랫폼: tenant별 guard config를 서버에 등록하고, extraction worker가 검증 API를 호출 + +## 12. History, Logging, Telemetry 명세 + +Guardrails는 각 호출을 `Call`로 기록하고, reask를 포함한 각 시도를 `Iteration`으로 남긴다. 각 validator 실행은 `ValidatorLogs`에 기록된다. + +기록되는 주요 정보: + +- 입력 메시지 +- prompt params +- 원본 LLM 출력 +- 파싱 결과 +- schema 검증 결과 +- validator별 시작/종료 시간 +- validator별 검증 전/후 값 +- 실패 메시지 +- 최종 guarded output +- call status + +온톨로지 구축에서는 이 history가 매우 중요하다. 엔티티/관계가 왜 생성되었고, 어떤 검증을 통과/실패했으며, 어떤 값이 자동 보정되었는지 감사 로그로 남길 수 있다. 단, 기본 history는 메모리 `Stack`이므로 production에서는 DB sink를 별도로 구현해야 한다. + +## 13. DocumentStore, VectorDB, Text2SQL 분석 + +### 13.1 DocumentStore + +`document_store.py`는 문서와 페이지를 저장하고 vector DB로 유사 페이지를 검색하는 추상화이다. + +핵심 객체: + +- `Document`: `id`, `pages`, `metadata` +- `Page`: `PageCoordinates`, `text`, `metadata` +- `DocumentStoreBase`: `add_document`, `search`, `add_text`, `add_texts`, `flush` +- `EphemeralDocumentStore`: SQLAlchemy metadata store + vector DB 조합 + +온톨로지 플랫폼에서는 이미 별도의 크롤링/문서 저장 구조가 있다면 이 모듈을 그대로 핵심 저장소로 쓰기보다는 “validator나 few-shot example retrieval용 경량 참고 구현”으로 쓰는 것이 적절하다. + +### 13.2 Text2SQL + +`applications/text2sql.py`는 Guardrails를 이용한 응용 예시이다. SQL 스키마와 예시 질의를 prompt에 넣고, 생성된 SQL을 RAIL validator로 검증한다. + +온톨로지 플랫폼에 주는 시사점: + +- LLM 생성 결과를 도메인별 validator로 감싸는 패턴이 잘 드러난다. +- 예시 검색 + Guard 검증 + reask 루프 구조는 “문서 기반 온톨로지 추출”에도 동일하게 적용할 수 있다. +- SQL 대신 ontology schema, SHACL shape, OWL/RDF vocabulary를 context로 넣으면 같은 패턴을 재사용할 수 있다. + +## 14. 테스트 기반 기능 범위 + +테스트 폴더는 다음 기능을 검증한다. + +- Guard 기본 호출, parse, validate +- AsyncGuard 및 async streaming +- RAIL 파싱, Python/Pydantic schema 변환 +- JSON parsing, structured data, formatter +- on_fail action: reask, fix, filter, refrain, noop, exception +- multi reask +- validator base 및 validator service +- CLI 동작 +- Guardrails server +- OpenAI/LiteLLM embedding/provider 연동 +- document store +- LangChain/LlamaIndex integration +- telemetry +- Hub install/registry + +즉, Guardrails의 주요 기능은 테스트로 비교적 넓게 커버되어 있다. 우리 프로젝트에서 소스 일부를 거의 그대로 가져온다면, 관련 테스트도 함께 가져와서 “원본 호환성 테스트”로 유지하는 것이 좋다. + +## 15. 범용 온톨로지 구축 플랫폼 적용 설계 + +### 15.1 Guardrails의 역할 + +Guardrails는 온톨로지 플랫폼에서 다음 레이어로 배치한다. + +```mermaid +flowchart LR + A["문서 수집/Crawl4AI/Firecrawl"] --> B["청킹 및 전처리"] + B --> C["LLM Ontology Extraction"] + C --> D["Guardrails 검증 게이트"] + D --> E["정규화/중복 병합"] + E --> F["Graph DB / RDF Store"] + D --> G["검토 큐 / ReAsk / 실패 로그"] +``` + +핵심 책임: + +- LLM 응답을 지정된 ontology extraction schema로 강제 +- 스키마 위반, 타입 오류, 누락 필드 차단 +- entity/relation 단위 validator 실행 +- 자동 수정 가능한 값 보정 +- 잘못된 relation 또는 근거 없는 triple 제거 +- 재질문으로 복구 가능한 오류 복구 +- 검증 로그와 provenance 저장 + +### 15.2 그대로 사용 권장 모듈 + +다음 모듈은 변형 없이 또는 import 경로만 조정해서 기본 소스로 사용해도 좋다. + +| 모듈 | 사용 이유 | +| --- | --- | +| `guardrails/classes/validation_outcome.py` | 결과 DTO가 잘 정리되어 있음 | +| `guardrails/classes/validation/*` | Pass/Fail/log/summary 구조 재사용 가치 높음 | +| `guardrails/actions/*` | reask/filter/refrain 표현이 범용적 | +| `guardrails/types/on_fail.py` | 실패 정책 enum 그대로 사용 가능 | +| `guardrails/utils/parsing_utils.py` | LLM JSON 파싱/타입 보정 유용 | +| `guardrails/schema/validator.py` | JSON Schema 검증 재사용 가능 | +| `guardrails/schema/pydantic_schema.py` | Pydantic 기반 schema 변환 핵심 | +| `guardrails/validator_service/*` | validator 실행/병합/실패 처리 엔진 | +| `guardrails/run/runner.py` | reask loop 기준 구현 | + +### 15.3 래핑 또는 수정 권장 모듈 + +| 모듈 | 이유 | 권장 방식 | +| --- | --- | --- | +| `guardrails/guard.py` | OpenAI/서버/telemetry/rc 의존이 섞여 있음 | `OntologyGuard` facade로 감싸기 | +| `guardrails/validator_base.py` | Hub/remote inference/rc 의존 있음 | 온톨로지 전용 `BaseOntologyValidator` 추가 | +| `guardrails/llm_providers.py` | provider별 변화가 잦음 | 현재 프로젝트 LLM gateway에 맞춘 adapter 작성 | +| `guardrails/hub/*` | 외부 Hub 의존 | 초기에는 제외 또는 optional | +| `guardrails/telemetry/*` | 외부 OTEL 설정 필요 | 내부 audit log로 대체 가능 | +| `guardrails/cli/*` | 제품 CLI와 책임 중복 | admin command로 필요한 기능만 이식 | +| `document_store.py` | 저장소 모델이 단순함 | 기존 crawler_platform 저장소와 통합 | + +### 15.4 온톨로지 전용 Validator 목록 + +초기 구축에 필요한 validator 명세는 다음과 같다. + +| Validator명 | 대상 경로 | 기능 | 실패 정책 | +| --- | --- | --- | --- | +| `EntityIdFormatValidator` | `$.entities.*.id` | ID prefix/slug/UUID 규칙 검증 | `fix` 또는 `exception` | +| `UniqueEntityIdValidator` | `$.entities` | 엔티티 ID 중복 검증 | `reask` | +| `EntityTypeValidator` | `$.entities.*.type` | class/individual/property 등 허용 타입 검증 | `reask` | +| `LabelRequiredValidator` | `$.entities.*.label` | 빈 label, 너무 긴 label 차단 | `fix_reask` | +| `RelationEndpointExistsValidator` | `$.relations.*` | source_id/target_id가 entities에 존재하는지 검증 | `reask` | +| `PredicateVocabularyValidator` | `$.relations.*.predicate` | 허용 predicate 또는 ontology vocabulary 매핑 | `custom` 또는 `reask` | +| `NoSelfRelationValidator` | `$.relations.*` | 금지된 self-loop relation 차단 | `filter` | +| `ConfidenceRangeValidator` | `$.relations.*.confidence` | 0~1 범위 보정 | `fix` | +| `EvidenceExistsValidator` | `$.relations.*.evidence.*` | evidence가 source document chunk에 존재하는지 | `filter` 또는 `reask` | +| `NoHallucinatedClassValidator` | `$.entities.*` | 원문 근거 없는 class 생성 차단 | `reask` | +| `OntologyAcyclicValidator` | `$` | subclass hierarchy cycle 탐지 | `custom` | +| `SHACLShapeValidator` | `$` | SHACL/OWL 제약 검증 | `exception` 또는 `reask` | + +### 15.5 온톨로지 추출 Guard 명세 + +권장 Pydantic 출력 모델: + +```python +class OntologyEvidence(BaseModel): + text: str + source_id: str + start_offset: int | None = None + end_offset: int | None = None + +class OntologyEntity(BaseModel): + id: str + label: str + type: Literal["class", "individual", "object_property", "data_property"] + description: str | None = None + aliases: list[str] = [] + evidence: list[OntologyEvidence] = [] + confidence: float + +class OntologyRelation(BaseModel): + id: str + source_id: str + predicate: str + target_id: str + evidence: list[OntologyEvidence] = [] + confidence: float + +class OntologyExtractionResult(BaseModel): + entities: list[OntologyEntity] + relations: list[OntologyRelation] + warnings: list[str] = [] +``` + +Guard 생성 정책: + +- `Guard.for_pydantic(OntologyExtractionResult)` +- `num_reasks=1` 기본, 고가치 문서만 2 +- schema/parsing 오류는 `full_schema_reask=True` +- field validator 오류는 부분 reask 우선 +- 최종 실패 결과는 graph store 저장 금지, 검토 큐로 이동 + +## 16. 정확한 기능명세 + +### 16.1 기능: 구조화 출력 생성 검증 + +- 입력: LLM chat messages, ontology schema, source chunk metadata +- 처리: + - LLM 호출 + - JSON 또는 문자열 파싱 + - JSON Schema 검증 + - 추가 키 제거 + - 타입 보정 + - field validator 실행 +- 출력: `ValidationOutcome[OntologyExtractionResult]` +- 예외: + - 파싱 불가: `NonParseableReAsk` + - schema 불일치: `SkeletonReAsk` + - validator 실패: `FieldReAsk` 또는 on_fail 정책 결과 + +### 16.2 기능: 기존 LLM 출력 사후 검증 + +- 입력: `llm_output` 문자열 +- 처리: `Guard.parse()` 경로로 LLM 호출 없이 검증 +- 출력: `ValidationOutcome` +- 사용처: 비동기 worker가 이미 받은 LLM 결과를 저장 전 검증 + +### 16.3 기능: 입력 메시지 검증 + +- 입력: `messages` +- 처리: validator map의 `messages` 경로 validator 실행 +- 출력: 검증된 messages +- 실패: 입력 prompt가 정책/길이/금칙어를 위반하면 LLM 호출 전 차단 +- 사용처: 사용자 정의 ontology extraction prompt 안전성 검증 + +### 16.4 기능: Field-level 검증 + +- 입력: 구조화 출력의 특정 JSON path +- 처리: path에 등록된 validator 실행 +- 출력: 통과 값, 수정 값, 제거 값, reask 값 중 하나 +- 사용처: entity label, relation endpoint, predicate, evidence 검증 + +### 16.5 기능: 실패 자동 보정 + +- 입력: `FailResult.fix_value` +- 처리: on_fail=`fix` 또는 `fix_reask` +- 출력: 보정된 값 +- 사용처: 공백 제거, 소문자화, confidence clipping, ID slug 변환 + +### 16.6 기능: 실패 재질문 + +- 입력: `FieldReAsk`, `SkeletonReAsk`, `NonParseableReAsk` +- 처리: + - 실패 위치와 오류 메시지 기반 reask prompt 생성 + - 부분 schema 또는 전체 schema 생성 + - LLM 재호출 + - 기존 출력과 새 출력 병합 +- 출력: 재검증된 `ValidationOutcome` +- 제한: `num_reasks` 초과 시 실패 상태 반환 + +### 16.7 기능: 실패 필터링 + +- 입력: validator 실패 값 +- 처리: on_fail=`filter` +- 출력: 해당 값 제거 +- 사용처: hallucinated relation, evidence 없는 triple 제거 + +### 16.8 기능: 응답 보류 + +- 입력: validator 실패 값 +- 처리: on_fail=`refrain` +- 출력: 빈 응답 또는 None +- 사용처: 보안/정책상 온톨로지 생성을 중단해야 하는 문서 + +### 16.9 기능: 검증 로그 저장 + +- 입력: validator 실행 결과 +- 처리: `ValidatorLogs` 생성 +- 출력: + - validator name + - registered name + - property path + - value before/after + - validation result + - start/end time +- 사용처: ontology triple audit, 품질 대시보드, 사용자 검토 UI + +### 16.10 기능: 서버형 검증 API + +- 입력: guard config, validation request +- 처리: Guardrails API 서버에서 검증 수행 +- 출력: serialized `ValidationOutcome` +- 사용처: extraction worker와 검증 정책 서버 분리 + +## 17. 통합 로드맵 + +### Phase 1: 내장 검증 라이브러리로 사용 + +- Guardrails 원본을 `참고`로 유지 +- 현재 프로젝트에 `ontology_guard/` 또는 `crawler_platform/validation/` 패키지 생성 +- Pydantic ontology schema 정의 +- 최소 validator 5개 구현 +- `Guard.for_pydantic()` 기반 extraction 검증 PoC 작성 + +### Phase 2: 원본 핵심 모듈 이식 + +- `actions`, `validation classes`, `on_fail`, `parsing_utils`, `schema validator` 이식 +- Hub/telemetry/CLI 의존 제거 +- 내부 LLM gateway adapter 작성 +- 테스트 fixture와 원본 unit test 일부 이식 + +### Phase 3: 온톨로지 품질 게이트 확장 + +- SHACL/OWL/RDF validator 추가 +- graph DB lookup validator 추가 +- evidence alignment validator 추가 +- reask 실패 결과 검토 큐 구현 +- validator log persistence 구현 + +### Phase 4: 서버형 정책 엔진 + +- Guard config 저장소 구현 +- tenant/project별 guard policy 관리 +- extraction worker가 validation service 호출 +- 품질 지표 dashboard 구축 + +## 18. 리스크 및 주의사항 + +- Guardrails는 외부 Hub, telemetry, `.guardrailsrc` 의존이 코드 곳곳에 있다. 그대로 제품 본체에 넣기 전 이 의존을 명확히 비활성화해야 한다. +- `Validator` 생성 시 rc 파일이 없으면 오류가 나는 경로가 있으므로, 독립 플랫폼에서는 설정 로더를 대체하거나 기본 rc를 생성해야 한다. +- 서버 모드는 별도 `guardrails-api` optional dependency에 의존한다. +- LLM provider wrapper는 외부 SDK 변화에 민감하다. 우리 프로젝트에서는 provider adapter를 별도로 두는 것이 안전하다. +- 기본 history는 메모리 기반이다. 운영 감사 로그로 쓰려면 DB 저장 계층이 필요하다. +- reask는 비용과 지연을 증가시킨다. 문서 중요도와 실패 유형별로 횟수를 다르게 설정해야 한다. +- 자동 `fix`는 편하지만 ontology 의미를 바꿀 위험이 있다. 의미적 필드는 `reask` 또는 `custom` 검증이 더 안전하다. + +## 19. 결론 + +Guardrails는 범용 온톨로지 구축 플랫폼의 “LLM 출력 신뢰성 계층”으로 매우 적합하다. 특히 Pydantic schema 기반 구조화 출력, JSON Schema 검증, field-level validator, on_fail 정책, reask loop, ValidationOutcome/history/log 구조는 거의 그대로 기본 소스로 삼을 수 있다. + +다만 원본 전체를 무비판적으로 복사하기보다는, Hub/telemetry/CLI/provider 의존이 강한 부분은 얇은 adapter로 감싸고, 온톨로지 전용 validator와 audit persistence를 추가하는 방식이 좋다. 초기 기준 구현은 `Guard.for_pydantic(OntologyExtractionResult)`와 custom ontology validators 조합으로 시작하는 것이 가장 빠르고 안정적이다. diff --git a/오픈소스분석자료/Knowledge_Agent_분석_및_기능명세.md b/오픈소스분석자료/Knowledge_Agent_분석_및_기능명세.md new file mode 100644 index 0000000..642ae8a --- /dev/null +++ b/오픈소스분석자료/Knowledge_Agent_분석_및_기능명세.md @@ -0,0 +1,943 @@ +# Knowledge Agent 분석 및 기능명세 + +분석 대상: `C:\Users\lasta\MyProject\AI\참고\knowledge_agent-main` + +작성 목적: 오픈 프로젝트 `knowledge_agent-main`을 범용 온톨로지 구축 플랫폼의 기본 소스로 활용하기 위해, 아키텍처와 기능을 상세히 분석하고 재사용 가능 범위와 보완 필요 사항을 명세한다. + +## 1. 프로젝트 개요 + +`Knowledge Agent`는 LightRAG 지식베이스를 자동으로 분석, 확장, 정제, 감사, 개선 제안하는 멀티 에이전트형 지식 관리 시스템이다. + +핵심 목표는 정적인 RAG/지식그래프 저장소를 다음과 같은 “살아있는 지식 관리 루프”로 전환하는 것이다. + +1. 기존 지식베이스를 분석하여 지식 공백을 찾는다. +2. 지식 공백별 연구 주제를 생성한다. +3. 검색 계획을 세우고 외부 웹/PDF 자료를 수집한다. +4. 수집한 원문을 마크다운과 요약으로 정제하여 DB에 저장한다. +5. 적합한 URL을 선별하여 LightRAG에 적재한다. +6. 그래프 품질 문제를 감사한다. +7. 중복, 명칭 불일치, 관계 오류 등을 수정한다. +8. 반복되는 문제를 분석하여 시스템 개선안을 제시한다. + +범용 온톨로지 구축 플랫폼 관점에서는 “도메인 문서 수집 → 문서 정제 → 엔티티/관계 추출 기반 지식그래프 구축 → 품질 감사 → 정제 → 운영 개선”의 기본 골격으로 활용할 수 있다. + +## 2. 기술 스택 및 실행 환경 + +### 2.1 주요 의존성 + +`pyproject.toml` 기준 의존성은 다음과 같다. + +| 분류 | 패키지 | 용도 | +|---|---|---| +| 에이전트 프레임워크 | `langchain`, `langgraph` | 에이전트 실행 및 상태 그래프 구성 | +| LLM 연동 | `langchain-openai` | OpenAI 호환 Chat 모델 호출 | +| MCP 연동 | `langchain-mcp-adapters` | MCP 서버의 도구를 LangChain 도구로 연결 | +| DB | `psycopg2-binary` | PostgreSQL 연결 | +| 설정 | `python-dotenv`, `pydantic` | 환경 변수 및 데이터 검증 | +| JSON 복구 | `json-repair` | LLM 출력 JSON 파싱 안정화 | +| 웹 수집 | `requests`, `trafilatura`, `playwright`, `beautifulsoup4`, `html2text` | HTML/PDF 수집 및 본문 추출 | +| PDF 처리 | `pdfplumber` | PDF 텍스트 추출 | +| 토큰 제어 | `tiktoken` | 요약 전 입력 토큰 제한 | + +### 2.2 환경 변수 + +`.env.example`과 코드 기준으로 다음 환경 변수가 필요하다. + +| 변수 | 설명 | +|---|---| +| `DATABASE_URL` | PostgreSQL 연결 문자열. `db_utils.py`에서 필수로 사용 | +| `OPENAI_MODEL_NAME` | 사용할 OpenAI 호환 모델명. 기본값은 `chat` | +| `OPENAI_BASE_URL` | OpenAI 호환 API 서버 URL. 기본값은 `http://localhost:8001/v1` | + +### 2.3 MCP 서버 설정 + +`mcp.json`은 다음 MCP 서버를 전제로 한다. + +| 서버 | 역할 | +|---|---| +| `google_search` | 외부 검색 | +| `lightrag` | LightRAG 질의, 그래프 조회, 문서 적재, 엔티티/관계 수정 | +| `fetch` | URL fetch 보조 도구 | +| `file_tools` | 파일 시스템 접근 | +| `deepwiki` | 외부 지식 검색 보조 | + +이 프로젝트는 MCP 도구 이름에 강하게 의존한다. 예를 들어 `analyst`는 `query`, `graphs_get`, `graph_labels`, `google_search`, `fetch` 도구를 찾고, `fixer`는 `graph_update_entity`, `documents_delete_entity`, `graph_update_relation`, `documents_delete_relation`, `graph_entity_exists` 도구를 기대한다. + +## 3. 전체 아키텍처 + +### 3.1 구조 + +```text +run.py + └─ knowledge_agent.py + └─ LangGraph StateGraph + ├─ Analyst + ├─ Researcher + ├─ Curator + ├─ Auditor + ├─ Fixer + └─ Advisor + +db_utils.py + ├─ 보고서 저장 테이블 관리 + └─ 수집 문서 저장/조회 + +tools.py + ├─ URL 다운로드 + ├─ HTML/PDF 본문 추출 + ├─ 마크다운 생성 + └─ 사람 승인 도구 + +prompts/ + ├─ analyst_prompt.txt + ├─ planner_prompt.txt + ├─ refiner_prompt.txt + ├─ summarizer_prompt.txt + ├─ search_ranker_prompt.txt + └─ ingester_prompt.txt +``` + +### 3.2 상태 모델 + +`state.py`의 `AgentState`는 LangGraph 전체 상태를 정의한다. + +주요 상태 필드: + +| 필드 | 설명 | +|---|---| +| `messages` | LangChain 메시지 목록 | +| `task` | 실행 워크플로우명 | +| `status` | 현재 상태 메시지 | +| `timestamp` | 실행 시각 | +| `mcp_tools` | MCP 서버에서 로드한 도구 목록 | +| `model` | ChatOpenAI 모델 객체 | +| `logger` | 실행 로거 | +| `analyst_report_id`, `analyst_report` | Analyst 산출물 | +| `researcher_report_id`, `researcher_gaps_todo`, `researcher_gaps_complete`, `researcher_report` | Researcher 진행 상태 | +| `curator_report_id`, `curator_urls_for_ingestion`, `curator_url_ingestion_status`, `curator_report` | Curator 진행 상태 | +| `auditor_report_id`, `auditor_report` | Auditor 산출물 | +| `fixer_report_id`, `fixer_report` | Fixer 산출물 | +| `advisor_report_id`, `advisor_report` | Advisor 산출물 | + +## 4. 실행 흐름 + +### 4.1 진입점 + +`run.py`가 실행 진입점이다. + +처리 순서: + +1. `.env`를 로드한다. +2. `create_tables()`로 PostgreSQL 테이블을 생성한다. +3. CLI 인자를 파싱하여 실행 태스크를 결정한다. +4. `get_mcp_tools()`로 MCP 도구를 로드한다. +5. `ChatOpenAI` 모델 객체를 생성한다. +6. `create_knowledge_agent_graph(task, mcp_tools)`로 LangGraph 워크플로우를 만든다. +7. 초기 상태를 넣고 `app.ainvoke(initial_state)`로 실행한다. + +지원 CLI: + +| 옵션 | 실행 태스크 | +|---|---| +| `--maintenance` | 전체 유지보수 루프 | +| `--analyze` | 지식 공백 분석 | +| `--research` | 외부 조사 및 문서 수집 | +| `--curate` | URL 선별 및 LightRAG 적재 | +| `--audit` | 그래프 품질 감사 | +| `--fix` | 품질 문제 수정 | +| `--advise` | 시스템 개선 제안 | + +### 4.2 LangGraph 워크플로우 + +`knowledge_agent.py`가 태스크별 그래프를 구성한다. + +전체 유지보수 흐름: + +```text +analyst + → save_analyst_report + → researcher + → curator + → auditor + → save_auditor_report + → fixer + → save_fixer_report + → advisor + → save_advisor_report + → END +``` + +개별 태스크는 해당 노드와 저장 노드만 실행한다. + +## 5. 데이터베이스 명세 + +`db_utils.py`는 PostgreSQL을 사용하며, 실행 시 다음 테이블을 생성한다. + +### 5.1 보고서 테이블 + +공통 구조: + +```sql +id SERIAL PRIMARY KEY +report_id VARCHAR(255) UNIQUE NOT NULL +report JSONB +created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +``` + +테이블: + +| 테이블 | 저장 대상 | +|---|---| +| `analyst_reports` | 지식베이스 요약 및 지식 공백 | +| `researcher_reports` | 지식 공백별 검색 계획 및 검색 결과 | +| `curator_reports` | 선별 URL 및 적재 상태 | +| `auditor_reports` | 그래프 품질 문제 | +| `fixer_reports` | 수정 실행 결과 | +| `advisor_reports` | 시스템 개선 제안 | + +### 5.2 문서 테이블 + +`documents` 테이블: + +```sql +id SERIAL PRIMARY KEY +url TEXT UNIQUE NOT NULL +raw_document BYTEA +markdown_content TEXT +summary TEXT +created_at TIMESTAMP WITH TIME ZONE DEFAULT CURRENT_TIMESTAMP +``` + +역할: + +| 컬럼 | 설명 | +|---|---| +| `url` | 원본 URL. 중복 방지 기준 | +| `raw_document` | HTML/PDF 원문 바이너리 | +| `markdown_content` | 본문 추출 결과 | +| `summary` | LLM 요약 | + +범용 온톨로지 플랫폼에서는 이 테이블을 `source_documents` 또는 `collected_documents`로 확장하고, `domain`, `source_type`, `crawl_status`, `content_hash`, `license`, `language`, `published_at`, `ontology_project_id` 같은 컬럼을 추가하는 것이 좋다. + +## 6. 에이전트별 기능명세 + +### 6.1 Analyst + +파일: `sub_agents/analyst.py` +프롬프트: `prompts/analyst_prompt.txt` + +목적: LightRAG 지식베이스의 현재 상태를 분석하고, 지식 공백을 구조화된 연구 주제로 변환한다. + +입력: + +| 입력 | 설명 | +|---|---| +| `state.messages[0].content` | 분석 지시문 | +| `mcp_tools` | `query`, `graphs_get`, `graph_labels`, `google_search`, `fetch` | +| `analyst_report_id` | 실행 시각 기반 ID | + +처리: + +1. LightRAG 질의 및 그래프 조회 도구로 기존 지식베이스를 탐색한다. +2. 5-10개 수준의 주제 테마를 만든다. +3. 외부 검색으로 주제 지형을 보완한다. +4. 시간적/논리적 지식 공백을 식별한다. +5. 각 공백을 Researcher가 사용할 수 있는 `research_topic` 객체로 만든다. +6. JSON 보고서를 반환한다. +7. `save_analyst_report_node`가 JSON을 복구/파싱한 후 DB에 저장한다. + +출력 JSON 핵심 스키마: + +```json +{ + "report_id": "ana_...", + "knowledge_base_summary": { + "summary": "...", + "themes": [ + { + "theme_id": "T1", + "description": "..." + } + ] + }, + "identified_gaps": [ + { + "gap_id": "G1", + "description": "...", + "research_topic": { + "title": "...", + "summary": "...", + "key_questions": [], + "keywords": [], + "sources_to_consult": [], + "sources_to avoid": [] + } + } + ] +} +``` + +재사용 판단: + +| 항목 | 판단 | +|---|---| +| 지식 공백 탐지 패턴 | 거의 그대로 재사용 가능 | +| 출력 스키마 | 온톨로지 구축용으로 확장 필요 | +| 도구 의존성 | LightRAG 도구명에 의존하므로 어댑터 필요 | + +온톨로지 플랫폼 확장안: + +`research_topic`에 다음 필드를 추가하는 것이 좋다. + +| 필드 | 설명 | +|---|---| +| `target_ontology_scope` | 구축 대상 온톨로지 범위 | +| `candidate_entity_types` | 예상 엔티티 유형 | +| `candidate_relation_types` | 예상 관계 유형 | +| `competency_questions` | 온톨로지가 답해야 하는 역량 질문 | +| `source_priority_policy` | 공식 문서, 논문, 웹문서 등 우선순위 | + +### 6.2 Researcher + +파일: `sub_agents/researcher.py` +프롬프트: `planner_prompt.txt`, `refiner_prompt.txt`, `summarizer_prompt.txt` + +목적: Analyst가 만든 지식 공백별 연구 주제를 바탕으로 검색 계획을 세우고, URL을 검색하고, 원문을 수집/정제/요약하여 DB에 저장한다. + +입력: + +| 입력 | 설명 | +|---|---| +| 최신 `analyst_reports` | `initialize_researcher()`가 DB에서 로드 | +| `google_search` MCP 도구 | 검색 실행 | +| `process_url()` | URL 수집 및 문서화 | +| `summarizer_executor` | 문서 요약 | + +처리 단계: + +1. `initialize_researcher()`가 최신 Analyst 보고서를 읽는다. +2. `identified_gaps`를 `researcher_gaps_todo`로 변환한다. +3. Planner가 각 `research_topic`에 대해 5개 검색 계획을 만든다. +4. 각 검색 계획을 `google_search`로 실행한다. +5. 검색 결과 URL마다 `process_url()`을 호출한다. +6. `process_url()`은 `documents` 테이블에 URL을 추가하고 원문/마크다운을 저장한다. +7. Refiner가 검색 결과의 충분성을 평가한다. +8. 부족하면 최대 2개의 추가 검색을 수행한다. +9. 저장된 마크다운 문서를 요약한다. +10. `researcher_reports`에 공백별 검색 결과를 업데이트한다. + +검색 계획 스키마: + +```json +{ + "searches": [ + { + "search_id": "S_P1", + "query": "...", + "rationale": "...", + "parameters": { + "dateRestrict": "y1", + "sort": "date", + "num": 10 + } + } + ] +} +``` + +Refiner 출력 스키마: + +```json +{ + "status": "sufficient", + "rationale": "..." +} +``` + +또는: + +```json +{ + "status": "insufficient", + "rationale": "...", + "searches": [ + { + "search_id": "S_R1", + "query": "...", + "rationale": "...", + "parameters": {} + } + ] +} +``` + +요약 출력 스키마: + +```json +{ + "summary": "2-4 sentence summary" +} +``` + +재사용 판단: + +| 항목 | 판단 | +|---|---| +| Planner/Refiner/Summarizer 구조 | 거의 그대로 재사용 가능 | +| URL 중복 저장 | 그대로 재사용 가능 | +| HTML/PDF 수집 | 보완 후 재사용 권장 | +| 도메인별 검색 전략 | 프롬프트만 교체/확장 | + +범용 온톨로지 플랫폼에서 가장 가치가 높은 모듈이다. 특히 “지식 공백 → 검색 계획 → 검색 결과 → 문서 저장 → 요약” 흐름은 도메인별 온톨로지 구축의 자료 수집 레이어로 그대로 사용할 수 있다. + +### 6.3 Content Processor + +파일: `tools.py` + +목적: URL을 원문 문서와 마크다운 콘텐츠로 변환한다. + +함수: + +| 함수 | 설명 | +|---|---| +| `fetch_and_generate_markdown(url, logger)` | URL의 Content-Type을 확인하고 HTML/PDF를 처리 | +| `process_url(url, logger)` | URL 중복 확인, 신규 URL이면 수집 후 DB 업데이트 | +| `human_approval(plan)` | 파괴적 작업 전 터미널 승인 요청 | + +HTML 처리: + +1. `requests.head()`로 Content-Type 확인 +2. `text/html`이면 `trafilatura.fetch_url()`로 HTML 다운로드 +3. `trafilatura.extract()`로 본문 추출 +4. 추출 결과가 없거나 200자 미만이면 Playwright로 브라우저 렌더링 +5. `main`, `#main`, `#content`, `[role="main"]`, `body` 순으로 텍스트 추출 + +PDF 처리: + +1. `requests.get()`으로 PDF 다운로드 +2. `pdfplumber`로 페이지별 텍스트 추출 +3. 줄 단위로 연결하여 `markdown_content`에 저장 + +실패 처리: + +지원하지 않는 Content-Type 또는 예외 발생 시: + +```text +[MARKDOWN_GENERATION_FAILED: ...] +``` + +재사용 판단: + +| 항목 | 판단 | +|---|---| +| URL 중복 등록 | 그대로 사용 가능 | +| Trafilatura 우선 + Playwright fallback | 그대로 사용 가능 | +| PDF 텍스트 추출 | 그대로 사용 가능 | +| 403/SSL/JS 복잡 사이트 대응 | 개선 필요 | +| Content-Type이 부정확한 서버 대응 | 개선 필요 | +| robots.txt/저작권/라이선스 정책 | 추가 필요 | + +### 6.4 Curator + +파일: `sub_agents/curator.py` +프롬프트: `search_ranker_prompt.txt`, `ingester_prompt.txt` + +목적: Researcher의 검색 결과를 평가해 실제 LightRAG에 넣을 URL을 선별하고 적재한다. + +처리: + +1. `initialize_curator()`가 최신 Researcher 보고서를 읽는다. +2. 검색 결과별로 Search Ranker 에이전트를 실행한다. +3. 각 URL을 `approved` 또는 `denied`로 분류한다. +4. 승인 URL 목록을 `curator_reports.urls_for_ingestion`에 저장한다. +5. Ingester 에이전트가 LightRAG 문서 적재 도구를 호출한다. +6. URL별 적재 상태를 저장한다. + +Search Ranker 출력 스키마: + +```json +{ + "ranked_urls": [ + { + "url": "https://example.com", + "status": "approved", + "rationale": "..." + } + ] +} +``` + +Ingester 출력 스키마: + +```json +{ + "url_ingestion_status": [ + { + "url": "https://example.com", + "status": "ingested" + } + ] +} +``` + +현재 코드상 주의점: + +`curator.py`에서는 다음 형태로 호출한다. + +```python +update_curator_report(tool_input) +``` + +하지만 `db_utils.py`의 실제 함수 시그니처는 다음과 같다. + +```python +update_curator_report(report_id: str, job: str, results: list) +``` + +따라서 현재 상태로는 Curator 실행 중 타입 오류가 발생할 가능성이 높다. 다음처럼 수정해야 한다. + +```python +update_curator_report(report_id, "urls_for_ingestion", approved_urls) +update_curator_report(report_id, "url_ingestion_status", curator_url_ingestion_status) +``` + +재사용 판단: + +| 항목 | 판단 | +|---|---| +| URL 평가 기준 | 거의 그대로 재사용 가능 | +| URL 승인/거부 JSON 계약 | 그대로 사용 가능 | +| LightRAG 적재 흐름 | MCP 도구명 확인 후 사용 | +| 현재 구현 안정성 | 수정 후 사용 필요 | + +### 6.5 Auditor + +파일: `sub_agents/auditor.py` +프롬프트 파일: `prompts/auditor_prompt.txt`는 비어 있음 +실제 프롬프트: 코드 내 문자열 + +목적: LightRAG 그래프를 조회하여 중복 엔티티, 정규화 오류, 관계 품질 문제를 찾는다. + +사용 도구: + +| 도구 | 설명 | +|---|---| +| `graphs_get` | 그래프 조회 | +| `query` | 지식베이스 질의 | + +현재 구현상 문제: + +1. `save_auditor_report_node()`에서 `save_auditor_report()`를 호출하지만 import하지 않았다. +2. `create_openai_tools_agent()` 결과를 `AgentExecutor`로 감싸지 않고 직접 `ainvoke()`한다. +3. 저장 시 `save_auditor_report({"auditor_report": json.dumps(report_json)})` 형태로 넘기는데, `_save_report()`는 최상위 `report_id`를 요구한다. 이 형태는 `report_id` 누락 오류를 만들 수 있다. +4. `auditor_prompt.txt`가 비어 있어 프롬프트 관리 체계와 코드가 불일치한다. + +재사용 판단: + +| 항목 | 판단 | +|---|---| +| 감사 에이전트 개념 | 그대로 재사용 가능 | +| 현재 코드 | 수정 필요 | +| 프롬프트 파일화 | 필요 | +| 감사 결과 스키마 | 새로 명확화 필요 | + +온톨로지 플랫폼용 Auditor 권장 스키마: + +```json +{ + "report_id": "aud_...", + "ontology_project_id": "...", + "issues": [ + { + "issue_id": "Q1", + "issue_type": "duplicate_entity | relation_conflict | weak_evidence | naming_inconsistency | schema_violation", + "severity": "low | medium | high | critical", + "entities": [], + "relations": [], + "evidence": [], + "recommended_action": "..." + } + ] +} +``` + +### 6.6 Fixer + +파일: `sub_agents/fixer.py` +프롬프트 파일: `prompts/fixer_prompt.txt`는 비어 있음 +실제 프롬프트: 코드 내 문자열 + +목적: Auditor가 찾은 그래프 품질 문제를 수정한다. + +사용 도구: + +| 도구 | 설명 | +|---|---| +| `graph_update_entity` | 엔티티 수정 | +| `documents_delete_entity` | 엔티티 삭제 | +| `graph_update_relation` | 관계 수정 | +| `documents_delete_relation` | 관계 삭제 | +| `graph_entity_exists` | 엔티티 존재 확인 | +| `human_approval` | 수정 계획 승인 | +| `load_latest_report` | 최신 보고서 로드 | + +현재 구현상 문제: + +1. `save_fixer_report()`를 import하지 않았다. +2. `load_latest_report`는 LangChain `@tool`로 감싸져 있지 않은 일반 함수다. 도구 목록에 직접 넣으면 LangChain 도구로 인식되지 않을 수 있다. +3. `create_openai_tools_agent()` 결과를 `AgentExecutor`로 감싸지 않는다. +4. 저장 보고서 구조가 `_save_report()`의 요구 조건과 맞지 않을 수 있다. +5. CLI/자동 실행 환경에서 `input()` 기반 `human_approval`은 중단 위험이 있다. + +재사용 판단: + +| 항목 | 판단 | +|---|---| +| 사람 승인 후 수정 패턴 | 매우 중요, 재사용 권장 | +| 현재 코드 | 수정 필요 | +| 파괴적 작업 정책 | 플랫폼 핵심 기능으로 확장 필요 | + +온톨로지 플랫폼에서는 수정 작업을 다음 세 단계로 분리하는 것이 좋다. + +1. `FixPlanGenerator`: 수정 계획 생성 +2. `ApprovalGate`: 사람 승인 또는 정책 기반 자동 승인 +3. `FixExecutor`: 승인된 작업만 실행 + +### 6.7 Advisor + +파일: `sub_agents/advisor.py` +프롬프트 파일: `prompts/advisor_prompt.txt`는 비어 있음 +실제 프롬프트: 코드 내 문자열 + +목적: 감사/수정 보고서를 분석하여 반복 문제와 시스템 개선안을 제시한다. + +사용 도구: + +| 도구 | 설명 | +|---|---| +| `list_allowed_directories` | 접근 가능한 디렉터리 조회 | +| `list_directory` | 디렉터리 조회 | +| `search_files` | 파일 검색 | +| `read_text_file` | 파일 읽기 | +| `load_latest_report` | 최신 보고서 로드 | + +현재 구현상 문제: + +1. `save_advisor_report()`를 import하지 않았다. +2. `load_latest_report` 도구화 문제가 있다. +3. `create_openai_tools_agent()` 직접 호출 문제가 있다. +4. 프롬프트 파일이 비어 있다. + +재사용 판단: + +| 항목 | 판단 | +|---|---| +| 운영 개선 에이전트 개념 | 그대로 재사용 가능 | +| 코드 안정성 | 수정 필요 | +| 플랫폼 확장 가치 | 높음 | + +온톨로지 플랫폼에서는 Advisor가 다음 개선안을 만들도록 확장할 수 있다. + +| 개선 대상 | 예시 | +|---|---| +| 엔티티 타입 체계 | 특정 타입 누락, 과도한 `concept/idea` 사용 | +| 관계 타입 체계 | 관계명이 너무 일반적이거나 중복됨 | +| 수집 정책 | 특정 도메인 실패율, 저품질 출처 비율 | +| 프롬프트 | 추출 누락, 명칭 정규화 실패 | +| 스키마 | 필수 속성 누락, 식별자 정책 부족 | + +## 7. LightRAG 프롬프트 분석 + +파일: `lightrag/prompt.py` + +이 파일은 LightRAG의 엔티티/관계 추출 프롬프트를 JSON 기반으로 재정의한다. 범용 온톨로지 구축 플랫폼에서 매우 중요한 자산이다. + +### 7.1 엔티티 타입 + +정의된 엔티티 타입: + +| 타입 | 설명 | +|---|---| +| `organization/institution` | 기관, 기업, 정부, 비영리 조직 | +| `person` | 인물 | +| `location/geo` | 지리적 장소 | +| `event` | 사건 | +| `policy/proposal` | 정책, 제안, 공식 계획 | +| `law/regulation` | 법률, 규정 | +| `tax/fiscal_instrument` | 조세, 수수료, 재정 메커니즘 | +| `narrative` | 사회적 서사 | +| `misinformation/disinformation` | 허위정보, 조작정보 | +| `digital_asset` | 디지털 자산 또는 플랫폼 | +| `concept/idea` | 추상 개념 | +| `metric/score` | 수치 지표 | +| `publication/article` | 보고서, 책, 기사 | +| `political_group` | 정치적 집단 | +| `scenario/situation` | 상황/맥락 | +| `demographic/population` | 인구 집단 | +| `publisher/outlet` | 출판사/매체 | +| `time_period/era` | 시기/기간 | + +### 7.2 관계 타입 + +정의된 관계 타입: + +```text +TARGETS, EVALUATES, PRODUCES, CAUSES, IS_A, IS_PART_OF, +IS_LOCATED_IN, INFLUENCES, PUBLISHED_BY, LED_BY, +CRITICIZES, SUPPORTS, USES, INVOLVES, ESTIMATES, +AFFIRMED_BY, PAYS_INTO, REIMBURSES +``` + +### 7.3 출력 스키마 + +```json +{ + "entities": [ + { + "name": "...", + "type": "...", + "description": "..." + } + ], + "relationships": [ + { + "source": "...", + "target": "...", + "description": "...", + "type": "...", + "strength": 8 + } + ] +} +``` + +### 7.4 재사용 가치 + +이 파일은 범용 온톨로지 구축 플랫폼의 “기본 온톨로지 추출 프롬프트”로 활용 가치가 높다. 특히 다음 원칙이 좋다. + +1. 엔티티 타입을 JSON 사전으로 명시한다. +2. 관계 타입을 고정 리스트로 제한한다. +3. `UNKNOWN` 타입을 금지하고 애매한 경우 `concept/idea`로 보낸다. +4. 관계 강도 `strength`를 함께 출력한다. +5. 결과를 반드시 JSON으로 강제한다. + +다만 현재 타입 체계는 정치/사회정책 도메인에 치우쳐 있다. 범용 온톨로지 플랫폼에서는 프로젝트별 타입 팩을 주입할 수 있어야 한다. + +## 8. 재사용 가능 모듈 평가 + +| 모듈 | 재사용 등급 | 사유 | +|---|---:|---| +| `knowledge_agent.py` LangGraph 구성 | 높음 | 워크플로우 분기와 노드 연결 구조가 명확 | +| `run.py` 실행 진입점 | 중간 | 기본 실행 구조는 좋지만 설정/모델 기본값 정리 필요 | +| `state.py` | 높음 | 멀티 에이전트 상태 전달 모델로 활용 가능 | +| `db_utils.py` 보고서 저장 | 중간 | 기본 구조는 좋지만 스키마 확장과 일부 저장 구조 수정 필요 | +| `db_utils.py` 문서 저장 | 높음 | URL 중복 방지와 원문/마크다운/요약 저장이 유용 | +| `tools.py` URL 처리 | 높음 | HTML/PDF 수집 파이프라인이 실용적 | +| `researcher.py` | 높음 | 자료 수집 자동화 핵심 모듈 | +| `analyst.py` | 높음 | 지식 공백 기반 조사 설계에 적합 | +| `curator.py` | 중간 | 개념은 좋지만 코드 수정 필요 | +| `auditor.py` | 낮음-중간 | 개념은 좋지만 구현 완성도가 낮음 | +| `fixer.py` | 낮음-중간 | 사람 승인 패턴은 좋지만 코드 수정 필요 | +| `advisor.py` | 중간 | 운영 개선 아이디어는 좋지만 구현 정리 필요 | +| `prompts/*.txt` | 높음 | JSON 계약과 역할 분리가 명확 | +| `lightrag/prompt.py` | 매우 높음 | 온톨로지 추출 프롬프트 기반으로 직접 활용 가능 | + +## 9. 현재 코드의 주요 결함 및 수정 필요 사항 + +### 9.1 실행 오류 가능성이 높은 부분 + +| 위치 | 문제 | 영향 | 수정 방향 | +|---|---|---|---| +| `curator.py` | `update_curator_report()` 호출 인자 불일치 | Curator 실행 실패 | `update_curator_report(report_id, job, results)`로 수정 | +| `auditor.py` | `save_auditor_report` import 누락 | 저장 실패 | `from db_utils import save_auditor_report` 추가 | +| `fixer.py` | `save_fixer_report` import 누락 | 저장 실패 | import 추가 | +| `advisor.py` | `save_advisor_report` import 누락 | 저장 실패 | import 추가 | +| `auditor.py`, `fixer.py`, `advisor.py` | `create_openai_tools_agent()`를 `AgentExecutor`로 감싸지 않음 | 정상 실행 불확실 | Researcher/Analyst 방식으로 통일 | +| `auditor.py`, `fixer.py`, `advisor.py` | 저장 데이터에 최상위 `report_id`가 없을 수 있음 | `_save_report()` 오류 | 보고서 스키마 통일 | +| `prompts/auditor_prompt.txt` 등 | 파일은 있으나 비어 있고 코드에 프롬프트 하드코딩 | 유지보수성 저하 | 프롬프트 파일로 이동 | +| `load_latest_report` | 일반 함수를 도구 목록에 직접 삽입 | LangChain 도구 인식 실패 가능 | `@tool` 래핑 또는 에이전트 외부에서 로드 | + +### 9.2 설계상 보완점 + +| 영역 | 보완 필요 | +|---|---| +| 도메인 독립성 | LightRAG 도구명, 정치/정책형 엔티티 타입에 의존 | +| 수집 정책 | robots.txt, 라이선스, 출처 신뢰도, 차단 도메인 정책 부족 | +| 실패 복구 | 403, SSL 오류, JS 렌더링 실패, 빈 본문 처리 강화 필요 | +| 중복 문서 | URL 기준 중복만 처리. `content_hash` 기반 중복 제거 필요 | +| 보고서 버전 | 프로젝트/도메인/실행 단위 식별자 부족 | +| 승인 흐름 | CLI `input()` 기반 승인만 제공. 웹 UI/API 승인 필요 | +| 감사 스키마 | 품질 이슈 타입, 심각도, 수정안 스키마가 불명확 | +| 테스트 | 단위 테스트/통합 테스트 부재 | + +## 10. 범용 온톨로지 구축 플랫폼 적용 설계 + +### 10.1 추천 플랫폼 아키텍처 + +```text +Ontology Project + ├─ Source Discovery + │ ├─ Analyst + │ └─ Research Planner + ├─ Source Collection + │ ├─ Search Executor + │ ├─ URL Processor + │ └─ Document Store + ├─ Ontology Extraction + │ ├─ Entity Extractor + │ ├─ Relation Extractor + │ └─ Schema Mapper + ├─ Curation + │ ├─ Source Ranker + │ ├─ Evidence Scorer + │ └─ Ingestion Manager + ├─ Quality Control + │ ├─ Auditor + │ ├─ Fix Planner + │ └─ Approval Gate + └─ Continuous Improvement + └─ Advisor +``` + +### 10.2 기존 소스와 매핑 + +| 플랫폼 기능 | 기존 소스 | +|---|---| +| 프로젝트 실행 워크플로우 | `knowledge_agent.py`, `run.py` | +| 상태 전달 | `state.py` | +| 지식 공백 탐지 | `sub_agents/analyst.py`, `analyst_prompt.txt` | +| 검색 전략 생성 | `sub_agents/researcher.py`, `planner_prompt.txt` | +| 검색 결과 보완 판단 | `refiner_prompt.txt` | +| 문서 수집/정제 | `tools.py` | +| 문서 저장 | `db_utils.py`의 `documents` | +| 요약 | `summarizer_prompt.txt` | +| URL 선별 | `curator.py`, `search_ranker_prompt.txt` | +| LightRAG 적재 | `curator.py`, `ingester_prompt.txt` | +| 그래프 감사 | `auditor.py` | +| 수정 승인/실행 | `fixer.py`, `human_approval()` | +| 시스템 개선 | `advisor.py` | +| 엔티티/관계 추출 프롬프트 | `lightrag/prompt.py` | + +### 10.3 거의 변형 없이 가져갈 수 있는 기능 + +1. LangGraph 기반 워크플로우 분기 구조 +2. `AgentState` 중심 상태 전달 방식 +3. Analyst의 지식 공백 탐지 프롬프트 구조 +4. Researcher의 Planner/Refiner/Summarizer 단계 구조 +5. URL 중복 저장 후 원문/마크다운/요약을 관리하는 문서 저장소 구조 +6. Trafilatura 우선, Playwright fallback 수집 전략 +7. JSON 출력 강제 프롬프트 패턴 +8. LightRAG 엔티티/관계 추출 프롬프트의 JSON 스키마 방식 +9. Human approval을 거친 그래프 수정 개념 + +### 10.4 반드시 수정 후 가져갈 기능 + +1. Curator의 DB 업데이트 호출 오류 +2. Auditor/Fixer/Advisor의 누락 import +3. Auditor/Fixer/Advisor의 AgentExecutor 사용 방식 +4. 보고서 저장 스키마 불일치 +5. 빈 프롬프트 파일과 하드코딩 프롬프트 분리 +6. `load_latest_report` 도구화 방식 +7. 수집 실패 및 차단 도메인 처리 +8. 프로젝트/도메인 단위 멀티테넌시 스키마 + +## 11. 기능명세서 + +### 11.1 프로젝트 관리 + +| 기능 ID | 기능명 | 설명 | 입력 | 출력 | +|---|---|---|---|---| +| ONT-PROJ-001 | 온톨로지 프로젝트 생성 | 도메인, 목표, 기본 타입 체계를 가진 프로젝트 생성 | 프로젝트명, 도메인, 설명 | `ontology_project_id` | +| ONT-PROJ-002 | 프로젝트별 실행 설정 | 모델, MCP 도구, 수집 정책, 승인 정책 설정 | 설정 JSON | 저장된 설정 | +| ONT-PROJ-003 | 프로젝트별 실행 이력 조회 | 분석/수집/적재/감사 이력 확인 | 프로젝트 ID | 실행 목록 | + +### 11.2 지식베이스 분석 + +| 기능 ID | 기능명 | 설명 | 입력 | 출력 | +|---|---|---|---|---| +| ONT-ANA-001 | 기존 지식베이스 요약 | 그래프와 문서를 조회하여 현재 지식 범위 요약 | 프로젝트 ID | 주제 요약 | +| ONT-ANA-002 | 지식 공백 탐지 | 시간적/논리적/출처상 공백 식별 | 지식베이스 요약 | 공백 목록 | +| ONT-ANA-003 | 연구 주제 생성 | 공백을 검색 가능한 조사 브리프로 변환 | 공백 목록 | `research_topic` 목록 | +| ONT-ANA-004 | 역량 질문 생성 | 온톨로지가 답해야 할 질문 생성 | 도메인 설명 | `competency_questions` | + +### 11.3 자료 검색 및 수집 + +| 기능 ID | 기능명 | 설명 | 입력 | 출력 | +|---|---|---|---|---| +| ONT-RES-001 | 검색 계획 생성 | 연구 주제별 검색 쿼리와 파라미터 생성 | `research_topic` | 검색 계획 | +| ONT-RES-002 | 검색 실행 | MCP 검색 도구로 검색 수행 | 검색 계획 | 검색 결과 | +| ONT-RES-003 | URL 중복 확인 | URL이 이미 저장되어 있는지 확인 | URL | 문서 ID, 신규/기존 상태 | +| ONT-RES-004 | HTML 본문 추출 | Trafilatura/Playwright로 본문 추출 | URL | 원문, 마크다운 | +| ONT-RES-005 | PDF 텍스트 추출 | PDF를 다운로드하고 텍스트 추출 | URL | 원문, 텍스트 | +| ONT-RES-006 | 문서 요약 | 마크다운을 16k 토큰 이하로 제한 후 요약 | 문서 ID | 요약 | +| ONT-RES-007 | 검색 결과 충분성 평가 | 초기 검색 결과가 연구 질문을 충족하는지 판단 | 검색 결과 | 충분/부족, 보완 검색 | + +### 11.4 출처 큐레이션 + +| 기능 ID | 기능명 | 설명 | 입력 | 출력 | +|---|---|---|---|---| +| ONT-CUR-001 | URL 품질 평가 | 관련성, 권위성, 품질, 신규성 기준 평가 | 검색 결과 | 승인/거부 URL | +| ONT-CUR-002 | 적재 대상 선정 | 승인 URL을 적재 목록에 추가 | 승인 URL | 적재 대기 목록 | +| ONT-CUR-003 | 지식베이스 적재 | LightRAG 또는 내부 그래프 저장소에 문서 적재 | URL/문서 ID | 적재 상태 | +| ONT-CUR-004 | 적재 상태 추적 | URL별 적재 성공/실패 기록 | 적재 작업 ID | 상태 목록 | + +### 11.5 온톨로지 추출 + +| 기능 ID | 기능명 | 설명 | 입력 | 출력 | +|---|---|---|---|---| +| ONT-EXT-001 | 엔티티 타입 사전 관리 | 프로젝트별 엔티티 타입 정의 | 타입 정의 JSON | 타입 사전 | +| ONT-EXT-002 | 관계 타입 사전 관리 | 프로젝트별 관계 타입 정의 | 관계 정의 JSON | 관계 사전 | +| ONT-EXT-003 | 엔티티/관계 추출 | 문서 청크에서 엔티티와 관계 추출 | 문서 청크, 타입 사전 | 엔티티/관계 JSON | +| ONT-EXT-004 | 명칭 정규화 | 약어/별칭을 표준명으로 통합 | 엔티티 후보 | 표준 엔티티 | +| ONT-EXT-005 | 증거 연결 | 엔티티/관계에 원문 근거 연결 | 추출 결과 | evidence 링크 | +| ONT-EXT-006 | 관계 강도 산정 | 관계의 명시성/확실성 점수 산정 | 관계 후보 | `strength` | + +### 11.6 품질 감사 + +| 기능 ID | 기능명 | 설명 | 입력 | 출력 | +|---|---|---|---|---| +| ONT-AUD-001 | 중복 엔티티 탐지 | 이름/별칭/설명 기반 중복 탐지 | 그래프 | 중복 후보 | +| ONT-AUD-002 | 명칭 불일치 탐지 | 동일 개념의 표기 차이 탐지 | 그래프 | 정규화 이슈 | +| ONT-AUD-003 | 관계 충돌 탐지 | 상충 관계나 잘못된 방향 탐지 | 그래프 | 관계 이슈 | +| ONT-AUD-004 | 스키마 위반 탐지 | 허용되지 않은 타입/관계 탐지 | 그래프, 스키마 | 위반 목록 | +| ONT-AUD-005 | 약한 근거 탐지 | evidence가 부족한 엔티티/관계 탐지 | 그래프 | 저신뢰 항목 | + +### 11.7 수정 및 승인 + +| 기능 ID | 기능명 | 설명 | 입력 | 출력 | +|---|---|---|---|---| +| ONT-FIX-001 | 수정 계획 생성 | 감사 이슈를 실행 가능한 수정 계획으로 변환 | 감사 보고서 | 수정 계획 | +| ONT-FIX-002 | 사람 승인 요청 | 삭제/병합/관계 변경 전 승인 요청 | 수정 계획 | 승인/거부 | +| ONT-FIX-003 | 엔티티 수정 | 이름, 타입, 설명 수정 | 승인된 계획 | 수정 결과 | +| ONT-FIX-004 | 관계 수정 | 관계 타입, 방향, 설명, 강도 수정 | 승인된 계획 | 수정 결과 | +| ONT-FIX-005 | 엔티티/관계 삭제 | 승인된 파괴적 변경 실행 | 승인된 계획 | 삭제 결과 | +| ONT-FIX-006 | 수정 이력 저장 | 누가/언제/무엇을 변경했는지 저장 | 수정 결과 | 이력 레코드 | + +### 11.8 운영 개선 + +| 기능 ID | 기능명 | 설명 | 입력 | 출력 | +|---|---|---|---|---| +| ONT-ADV-001 | 실패 패턴 분석 | 수집/적재/추출/감사 실패 로그 분석 | 실행 로그 | 실패 패턴 | +| ONT-ADV-002 | 프롬프트 개선 제안 | 반복 오류를 줄이기 위한 프롬프트 수정안 제시 | 감사/수정 보고서 | 개선안 | +| ONT-ADV-003 | 타입/관계 체계 개선 제안 | 누락/중복 타입 및 관계 개선 | 추출 결과 | 스키마 제안 | +| ONT-ADV-004 | 수집 정책 개선 제안 | 차단 도메인, 신뢰 출처, 우선순위 개선 | 수집 로그 | 정책 제안 | +| ONT-ADV-005 | Top N 개선 리포트 | 가장 영향도 높은 개선안을 정리 | 전체 보고서 | 개선 보고서 | + +## 12. 권장 리팩터링 순서 + +1. Curator/Auditor/Fixer/Advisor 실행 오류를 먼저 수정한다. +2. 보고서 저장 스키마를 모든 에이전트에서 통일한다. +3. 하드코딩 프롬프트를 `prompts/*.txt`로 이동한다. +4. `load_latest_report`를 에이전트 도구로 쓸지, 노드 내부 로직으로 쓸지 분리한다. +5. DB 스키마에 `ontology_project_id`와 실행 ID를 추가한다. +6. 문서 테이블에 `content_hash`, `source_status`, `source_type`, `language`, `license`, `last_checked_at`을 추가한다. +7. LightRAG 프롬프트의 엔티티/관계 타입을 프로젝트별 설정으로 분리한다. +8. Auditor/Fixer 스키마를 명확히 정의하고 승인 UI/API를 설계한다. +9. 수집 실패 도메인 blocklist를 DB화한다. +10. 주요 기능별 테스트를 추가한다. + +## 13. 결론 + +`knowledge_agent-main`은 범용 온톨로지 구축 플랫폼의 초기 골격으로 활용 가치가 높다. 특히 Analyst-Researcher-Curator로 이어지는 “지식 공백 기반 자료 수집 루프”와 `lightrag/prompt.py`의 JSON 기반 엔티티/관계 추출 프롬프트는 거의 그대로 가져와도 된다. + +다만 현재 프로젝트는 연구/프로토타입 성격이 강하며, 전체 유지보수 워크플로우를 바로 운영 환경에 넣기에는 Curator 이후 단계의 코드 안정성이 부족하다. 따라서 기본 소스로 채택하되, 먼저 실행 오류와 보고서 스키마를 정리하고, 이후 범용 온톨로지 플랫폼에 맞게 프로젝트 단위 설정, 도메인별 타입 체계, 품질 감사/승인 체계를 확장하는 방식이 적합하다. diff --git a/오픈소스분석자료/Neo4j_GraphRAG_분석_및_기능명세.md b/오픈소스분석자료/Neo4j_GraphRAG_분석_및_기능명세.md new file mode 100644 index 0000000..bfce54a --- /dev/null +++ b/오픈소스분석자료/Neo4j_GraphRAG_분석_및_기능명세.md @@ -0,0 +1,1078 @@ +# Neo4j GraphRAG Python 분석 및 기능명세 + +분석 대상: `C:\Users\lasta\MyProject\AI\참고\neo4j-graphrag-python-main` +분석일: 2026-05-13 +프로젝트 버전: `neo4j-graphrag 1.16.0` +라이선스: Apache License 2.0 계열(`LICENSE.APACHE2.txt`, `LICENSE.txt`) +목표: 범용 온톨로지 구축 플랫폼의 기본 소스로 거의 변형 없이 재사용 가능한 기능과, 플랫폼 레이어에서 감싸야 할 기능을 식별한다. + +## 1. 총평 + +`neo4j-graphrag-python`은 Neo4j가 공식 제공하는 GraphRAG Python 패키지이며, 단순 질의응답 RAG보다 “비정형 문서 → 스키마/온톨로지 후보 → 엔티티/관계 추출 → 그래프 정제 → Neo4j 저장 → 검색/질의응답” 흐름에 더 강하다. + +우리 프로젝트가 지향하는 “범용 온톨로지 구축 플랫폼” 관점에서는 다음 모듈이 가장 중요하다. + +| 영역 | 재사용 판단 | 핵심 소스 | +|---|---:|---| +| KG 구축 파이프라인 | 높음 | `experimental/pipeline/kg_builder.py`, `config/template_pipeline/simple_kg_builder.py` | +| 컴포넌트형 파이프라인 엔진 | 높음 | `experimental/pipeline/pipeline.py`, `component.py`, `orchestrator.py` | +| 온톨로지/그래프 스키마 모델 | 매우 높음 | `experimental/components/schema.py` | +| 스키마 자동 추출 | 높음 | `experimental/components/schema.py`, `graph_schema_extraction.py` | +| 엔티티/관계 추출 | 높음 | `experimental/components/entity_relation_extractor.py` | +| 그래프 정제/스키마 준수 | 높음 | `experimental/components/graph_pruning.py` | +| Neo4j 저장 | 높음 | `experimental/components/kg_writer.py`, `neo4j_queries.py` | +| 엔티티 중복 해소 | 중간~높음 | `experimental/components/resolver.py` | +| 검색/RAG | 높음 | `retrievers/*`, `generation/graphrag.py` | +| LLM/임베딩 어댑터 | 높음 | `llm/*`, `embeddings/*` | +| 문서 로더 | 중간 | `experimental/components/data_loader.py` | +| 외부 벡터DB 연동 | 선택 | `retrievers/external/*` | + +단, `experimental` 네임스페이스의 KG 구축 기능은 공식 문서상 API 변경 가능성이 있는 실험 기능이다. “그대로 복사”보다는 패키지 의존성으로 고정 버전을 사용하고, 우리 플랫폼의 안정 API를 별도 래퍼로 제공하는 방식이 안전하다. + +## 2. 프로젝트 구조 + +```text +src/neo4j_graphrag/ + embeddings/ # OpenAI, Azure OpenAI, Ollama, VertexAI, Cohere, Bedrock, Mistral, SentenceTransformer 임베딩 + llm/ # OpenAI, Azure, Ollama, VertexAI, Anthropic, Cohere, Bedrock, Mistral LLM + retrievers/ # Vector, Hybrid, Text2Cypher, Tools, Vector+Cypher 검색기 + generation/ # GraphRAG, 프롬프트 템플릿, RAG 결과 타입 + experimental/ + components/ # KG 구축 컴포넌트: loader, splitter, schema, extractor, pruner, writer, resolver + pipeline/ # 비동기 DAG 파이프라인 엔진, 설정 파일 실행기 + indexes.py # Neo4j 벡터/풀텍스트 인덱스 생성, 벡터 upsert + schema.py # 기존 Neo4j DB 스키마 조회/포맷팅/Text2Cypher용 스키마 생성 + filters.py # 검색 필터 DSL → Cypher 변환 + message_history.py # InMemory/Neo4j 대화 기록 + tool.py # LLM tool schema 추상화 +``` + +테스트 구조는 `tests/unit`, `tests/e2e`가 분리되어 있고, Neo4j/Weaviate/Pinecone/Qdrant 연동 E2E가 있다. 핵심 동작은 테스트가 비교적 넓게 잡혀 있어 베이스 소스로 신뢰도가 높다. + +## 3. 의존성 및 실행 조건 + +### 3.1 기본 요구사항 + +- Python: `>=3.10,<3.15` +- Neo4j Python driver: `neo4j>=5.17,<7` +- Pydantic v2 +- `pypdf`, `fsspec`, `json-repair`, `pyyaml`, `numpy`, `scipy`, `tenacity` + +### 3.2 Neo4j 요구사항 + +- Neo4j `>=5.18.1` +- Neo4j Aura `>=5.18.0` +- Neo4j `2026.01+`: `SEARCH` clause 기반 in-index filtering 지원 +- KG writer 및 entity resolver 일부 기능은 APOC 필요 + - README의 KG construction 예시는 APOC core 설치를 요구한다. + - `SinglePropertyExactMatchResolver`, similarity resolver는 `apoc.refactor.mergeNodes`를 사용한다. + +### 3.3 선택 의존성 + +| Extra | 용도 | +|---|---| +| `openai` | OpenAI/Azure OpenAI LLM 및 임베딩 | +| `ollama` | 로컬 Ollama LLM/임베딩 | +| `google` | Vertex AI | +| `cohere` | Cohere | +| `anthropic` | Anthropic | +| `mistralai` | Mistral AI | +| `bedrock` | AWS Bedrock | +| `sentence-transformers` | 로컬 임베딩 | +| `experimental` | KG 구축 파이프라인, LlamaIndex/LangChain splitter, Parquet | +| `nlp` | spaCy resolver. Python 3.14에서는 미지원 | +| `fuzzy-matching` | RapidFuzz 기반 중복 해소 | +| `weaviate`, `pinecone`, `qdrant` | 외부 벡터DB retriever | + +## 4. 핵심 아키텍처 + +### 4.1 전체 흐름 + +```mermaid +flowchart LR + A["문서/텍스트 입력"] --> B["DataLoader"] + B --> C["TextSplitter"] + C --> D["TextChunkEmbedder"] + C --> E["SchemaBuilder 또는 SchemaFromTextExtractor"] + E --> F["LLMEntityRelationExtractor"] + C --> F + F --> G["GraphPruning"] + G --> H["KGWriter: Neo4jWriter 또는 ParquetWriter"] + H --> I["EntityResolver"] + I --> J["Neo4j Knowledge Graph"] + J --> K["Retriever: Vector/Hybrid/Text2Cypher"] + K --> L["GraphRAG"] +``` + +### 4.2 설계 철학 + +이 프로젝트는 “하나의 거대한 KG builder”가 아니라 비동기 컴포넌트 DAG를 조립하는 방식이다. + +- 각 컴포넌트는 `Component`를 상속한다. +- 컴포넌트의 `run`은 Pydantic `DataModel`을 반환한다. +- `Pipeline.connect()`로 이전 컴포넌트 출력 필드를 다음 컴포넌트 입력으로 매핑한다. +- `PipelineRunner`는 JSON/YAML 설정 파일로 파이프라인을 복원하고 실행한다. +- `SimpleKGPipeline`은 대표적인 템플릿 파이프라인이다. + +우리 플랫폼에서는 이 구조를 그대로 “워크플로우 엔진”으로 활용할 수 있다. 별도 GUI나 API 서버에서는 `SimpleKGPipeline` 설정을 생성하고 실행 결과를 추적하는 레이어를 만들면 된다. + +## 5. 데이터 모델 명세 + +### 5.1 문서 모델 + +`DocumentInfo` + +| 필드 | 타입 | 설명 | +|---|---|---| +| `path` | `str` | 파일 경로 또는 inline text 식별자 | +| `metadata` | `dict[str,str] \| None` | 문서 메타데이터. Document 노드 property로 저장 | +| `uid` | `str` | UUID 기본 생성. Document id | +| `document_type` | `pdf`, `markdown`, `inline_text` | 문서 유형 | + +`LoadedDocument` + +| 필드 | 타입 | 설명 | +|---|---|---| +| `text` | `str` | 추출된 원문 | +| `document_info` | `DocumentInfo` | 문서 식별/메타데이터 | + +### 5.2 텍스트 청크 모델 + +`TextChunk` + +| 필드 | 타입 | 설명 | +|---|---|---| +| `text` | `str` | 청크 텍스트 | +| `index` | `int` | 원문 내 순서 | +| `metadata` | `dict[str,Any] \| None` | 청크 메타데이터. `embedding`이 있으면 별도 embedding property로 분리 | +| `uid` | `str` | UUID 기본 생성 | + +`TextChunks` + +| 필드 | 타입 | 설명 | +|---|---|---| +| `chunks` | `list[TextChunk]` | 청크 목록 | + +### 5.3 그래프 모델 + +`Neo4jNode` + +| 필드 | 타입 | 설명 | +|---|---|---| +| `id` | `str` | 내부 관계 연결용 id | +| `label` | `str` | Neo4j label | +| `properties` | `dict[str, PropertyValue]` | Neo4j property | +| `embedding_properties` | `dict[str, list[float]]` | 벡터 property | + +`Neo4jRelationship` + +| 필드 | 타입 | 설명 | +|---|---|---| +| `start_node_id` | `str` | 시작 노드 id | +| `end_node_id` | `str` | 끝 노드 id | +| `type` | `str` | relationship type | +| `properties` | `dict[str, PropertyValue]` | relationship property | +| `embedding_properties` | `dict[str, list[float]]` | relationship vector property | + +`Neo4jGraph` + +| 필드 | 타입 | 설명 | +|---|---|---| +| `nodes` | `list[Neo4jNode]` | 노드 목록 | +| `relationships` | `list[Neo4jRelationship]` | 관계 목록 | + +### 5.4 Lexical graph 설정 + +`LexicalGraphConfig` 기본값: + +| 설정 | 기본값 | 의미 | +|---|---|---| +| `document_node_label` | `Document` | 문서 노드 label | +| `chunk_node_label` | `Chunk` | 청크 노드 label | +| `chunk_to_document_relationship_type` | `FROM_DOCUMENT` | Chunk → Document | +| `next_chunk_relationship_type` | `NEXT_CHUNK` | Chunk → 다음 Chunk | +| `node_to_chunk_relationship_type` | `FROM_CHUNK` | Entity → Chunk | +| `chunk_id_property` | `id` | chunk id property | +| `chunk_index_property` | `index` | chunk 순서 property | +| `chunk_text_property` | `text` | chunk 본문 property | +| `chunk_embedding_property` | `embedding` | chunk embedding property | + +온톨로지 플랫폼에서는 이 설정을 테넌트/프로젝트 단위로 고정하거나, 사용자가 “문서 그래프 모델”을 커스터마이즈할 수 있게 노출하면 된다. + +## 6. 온톨로지/스키마 모델 명세 + +핵심 파일: `src/neo4j_graphrag/experimental/components/schema.py` + +### 6.1 PropertyType + +| 필드 | 타입 | 설명 | +|---|---|---| +| `name` | `str` | property 이름 | +| `type` | Neo4j property type literal | `STRING`, `INTEGER`, `FLOAT`, `BOOLEAN`, `DATE`, `LOCAL_DATETIME`, `POINT`, `LIST` 등 | +| `description` | `str` | LLM 추출 가이드 | +| `required` | `bool` | deprecated. 존재 제약은 `ConstraintType(EXISTENCE)` 권장 | + +### 6.2 NodeType + +| 필드 | 타입 | 설명 | +|---|---|---| +| `label` | `str` | 노드 label | +| `description` | `str` | 의미 설명 | +| `properties` | `list[PropertyType]` | 허용 property. 최소 1개 | +| `additional_properties` | `bool` | 스키마 외 property 허용 여부 | + +특이 동작: + +- 문자열 `"Person"`으로 입력하면 `{label:"Person", properties:[{name:"name", type:"STRING"}], additional_properties:true}`로 자동 변환된다. +- label이 `__`로 시작하거나 끝나면 내부 예약 label로 보고 거부한다. + +### 6.3 RelationshipType + +| 필드 | 타입 | 설명 | +|---|---|---| +| `label` | `str` | relationship type | +| `description` | `str` | 의미 설명 | +| `properties` | `list[PropertyType]` | 관계 property | +| `additional_properties` | `bool` | 스키마 외 property 허용 여부 | + +문자열 `"WORKS_AT"` 입력도 허용된다. + +### 6.4 ConstraintType + +지원 제약: + +| 타입 | 의미 | +|---|---| +| `UNIQUENESS` | 노드 property unique. 복합 가능 | +| `EXISTENCE` | 노드/관계 property 필수. 단일 property | +| `KEY` | Neo4j node key/relationship key. 필수 + unique. 복합 가능 | + +범용 온톨로지 플랫폼에서는 이 모델을 “온톨로지 제약조건”의 기본 표현으로 재사용할 수 있다. 다만 OWL/RDFS의 class hierarchy, domain/range, cardinality, inverse property, equivalent class 같은 의미론적 제약은 별도 확장이 필요하다. + +### 6.5 Pattern + +`Pattern`은 `(source node label, relationship label, target node label)` 구조를 표현한다. 예: + +```python +("Person", "WORKS_AT", "Organization") +``` + +이것은 ontology의 relationship domain/range 후보로 직접 매핑 가능하다. + +### 6.6 GraphSchema + +`GraphSchema`는 다음 정보를 묶는다. + +- `node_types` +- `relationship_types` +- `patterns` +- `constraints` +- `additional_node_types` +- `additional_relationship_types` +- `additional_patterns` + +스키마가 제공되면 LLM 추출 프롬프트의 grounding 정보가 되고, 이후 `GraphPruning`이 이 스키마에 맞지 않는 노드/관계/property를 제거한다. + +## 7. KG 구축 기능명세 + +### 7.1 SimpleKGPipeline + +파일: `experimental/pipeline/kg_builder.py` + +`SimpleKGPipeline`은 비정형 텍스트나 PDF/Markdown 파일에서 KG를 만들기 위한 고수준 API이다. + +#### 생성자 입력 + +| 파라미터 | 필수 | 설명 | +|---|---:|---| +| `llm` | 예 | 엔티티/관계 추출용 LLM | +| `driver` | 예 | Neo4j driver | +| `embedder` | 예 | chunk embedding 생성기 | +| `schema` | 아니오 | `GraphSchema`, dict, `"FREE"`, `"EXTRACTED"`, `None` | +| `from_file` | 아니오 | `True`: file path 입력. `False`: text 입력 | +| `text_splitter` | 아니오 | 기본 `FixedSizeSplitter` | +| `file_loader` | 아니오 | 기본 extension 기반 PDF/Markdown loader | +| `kg_writer` | 아니오 | 기본 `Neo4jWriter` | +| `on_error` | 아니오 | `"IGNORE"` 또는 `"RAISE"` | +| `perform_entity_resolution` | 아니오 | 기본 `True` | +| `prompt_template` | 아니오 | 추출 프롬프트 템플릿 | +| `lexical_graph_config` | 아니오 | Document/Chunk 그래프 label/relationship 커스터마이즈 | +| `neo4j_database` | 아니오 | Neo4j database 이름 | + +#### 실행 입력 + +`run_async(file_path=None, text=None, document_metadata=None)` + +| 입력 | 조건 | 설명 | +|---|---|---| +| `file_path` | `from_file=True`일 때 필요 | PDF/Markdown 파일 경로 | +| `text` | `from_file=False`일 때 필요 | 직접 입력 텍스트 | +| `document_metadata` | 선택 | Document node property로 저장 | + +#### 스키마 모드 + +| 모드 | 설정 | 동작 | +|---|---|---| +| 자동 추출 | `schema=None` 또는 `"EXTRACTED"` | 입력 텍스트에서 LLM으로 스키마를 한 번 추출한 뒤 전체 chunk 추출에 사용 | +| 자유 추출 | `schema="FREE"` 또는 empty schema | 스키마 없이 엔티티/관계 추출 | +| 고정 스키마 | dict 또는 `GraphSchema` | 사용자가 정의한 온톨로지 구조에 맞춰 추출 | + +범용 온톨로지 플랫폼에서는 세 모드를 다음 UI/API로 제공하는 것이 적합하다. + +- “자동 온톨로지 초안 생성” +- “스키마 없이 자유 그래프 생성” +- “승인된 온톨로지에 맞춰 인스턴스 추출” + +### 7.2 DataLoader + +파일: `experimental/components/data_loader.py` + +제공 구현: + +- `PdfLoader`: PDF 텍스트 추출 +- `MarkdownLoader`: Markdown 텍스트 로드 +- 내부 extension 기반 loader: `.pdf`, `.md`, `.markdown` + +기능명세: + +| 기능 | 입력 | 출력 | 비고 | +|---|---|---|---| +| PDF 로드 | `filepath`, `metadata` | `LoadedDocument` | `pypdf` 사용 | +| Markdown 로드 | `filepath`, `metadata` | `LoadedDocument` | plain text로 처리 | +| 문서 메타데이터 생성 | path, metadata | `DocumentInfo` | Document node에 연결 | + +확장 필요: + +- HTML, DOCX, PPTX, XLSX, CSV, 웹 크롤링 결과, API 문서 등 우리 프로젝트 입력 소스에 맞춘 loader 추가 +- 이미 우리 프로젝트에 crawler가 있으므로 crawler output을 `LoadedDocument`로 변환하는 adapter 필요 + +### 7.3 TextSplitter + +제공 구현: + +- `FixedSizeSplitter` +- `LangChainTextSplitterAdapter` +- `LlamaIndexTextSplitterAdapter` + +기능명세: + +| 기능 | 입력 | 출력 | 비고 | +|---|---|---|---| +| 고정 길이 chunking | text, chunk_size, overlap | `TextChunks` | `approximate=True`이면 단어 중간 절단 회피 | +| LangChain splitter 감싸기 | LangChain splitter | `TextChunks` | 기존 생태계 활용 | +| LlamaIndex splitter 감싸기 | LlamaIndex splitter | `TextChunks` | 기존 생태계 활용 | + +온톨로지 플랫폼에서는 도메인별 chunking 전략이 중요하다. + +- 법령/규정: 조문 단위 +- 논문: section/paragraph 단위 +- 사내 문서: heading hierarchy 유지 +- 웹 문서: URL, heading, DOM 경로 메타데이터 유지 + +따라서 기본 splitter는 재사용하되, “구조 보존 splitter”를 별도 컴포넌트로 추가하는 것이 좋다. + +### 7.4 TextChunkEmbedder + +파일: `experimental/components/embedder.py` + +기능: + +- `TextChunks`의 각 chunk text를 embedder로 임베딩한다. +- 임베딩을 chunk metadata의 `embedding`에 저장한다. +- `LexicalGraphBuilder`가 `embedding` metadata를 chunk node의 `embedding_properties`로 분리한다. + +재사용 판단: 높음. 단, 대량 문서 처리에서는 batch embedding, rate limit, retry, cache가 플랫폼 레이어에 필요하다. + +### 7.5 SchemaBuilder / SchemaFromTextExtractor + +파일: `experimental/components/schema.py`, `graph_schema_extraction.py` + +기능: + +- 수동 schema dict 또는 `GraphSchema`를 검증한다. +- 텍스트에서 자동으로 node type, relationship type, pattern, constraint 후보를 추출한다. +- OpenAI/VertexAI 등 structured output 지원 LLM에서는 JSON schema 기반 구조화 출력을 사용한다. +- 기존 Neo4j graph에서 schema를 읽어 schema 후보로 만들 수 있다. + +기능명세: + +| 기능 | 입력 | 출력 | +|---|---|---| +| 수동 스키마 검증 | node types, relationship types, patterns, constraints | `GraphSchema` | +| 자동 스키마 추출 | text/chunks, LLM, prompt | `GraphSchema` | +| 기존 graph 스키마 추출 | Neo4j driver | `GraphSchema` 후보 | +| schema visualization | `GraphSchema` | 시각화 graph | + +우리 플랫폼 확장 포인트: + +- 스키마 버전 관리 +- 자동 추출 schema의 승인/반려 workflow +- label/property 표준화 규칙 +- 한국어 label/영문 label alias 관리 +- 온톨로지 class hierarchy 확장 + +### 7.6 LLMEntityRelationExtractor + +파일: `experimental/components/entity_relation_extractor.py` + +기능: + +- 각 chunk에 대해 LLM으로 `Neo4jGraph(nodes, relationships)`를 추출한다. +- chunk별 node id에 chunk UUID prefix를 붙여 충돌을 방지한다. +- `create_lexical_graph=True`이면 Document/Chunk graph와 Entity → Chunk provenance 관계를 함께 생성한다. +- JSON repair를 사용해 깨진 JSON 응답을 복구한다. +- `on_error=IGNORE`이면 실패 chunk는 빈 graph로 처리한다. +- `on_error=RAISE`이면 LLM/JSON 오류를 예외로 올린다. +- `max_concurrency`로 LLM 호출 동시성을 제한한다. +- structured output 지원 LLM이면 `Neo4jGraph` Pydantic 모델을 response schema로 사용할 수 있다. + +기능명세: + +| 기능 | 입력 | 출력 | 중요 옵션 | +|---|---|---|---| +| chunk별 엔티티/관계 추출 | `TextChunks`, `GraphSchema`, examples | `Neo4jGraph` | `max_concurrency` | +| lexical graph 생성 | chunks, document_info | Document/Chunk 포함 graph | `create_lexical_graph` | +| 출처 연결 | extracted entity, chunk | `FROM_CHUNK` 관계 | provenance 핵심 | +| 오류 처리 | LLM JSON 오류 | 빈 graph 또는 예외 | `on_error` | + +온톨로지 플랫폼에서 매우 중요한 특성: + +- 모든 추출 엔티티가 chunk와 연결되므로 근거 추적이 가능하다. +- 추출 결과를 바로 DB에 쓰기 전에 `GraphPruning`으로 스키마 위반을 제거할 수 있다. +- 추출 결과를 사용자가 승인하는 “검수 큐”를 만들려면 `Neo4jWriter` 이전에 graph를 저장/표시하는 컴포넌트를 끼우면 된다. + +### 7.7 GraphPruning + +파일: `experimental/components/graph_pruning.py` + +기능: + +- 추출 graph가 `GraphSchema`를 준수하도록 노드/관계/property를 제거한다. +- lexical graph(Document/Chunk)는 별도로 보존한다. +- pruning 통계를 반환한다. + +제거 사유: + +| 사유 | 의미 | +|---|---| +| `NOT_IN_SCHEMA` | 스키마에 없는 node/relationship/property | +| `MISSING_REQUIRED_PROPERTY` | 필수 property 누락 | +| `NO_PROPERTY_LEFT` | 유효 property가 하나도 없음 | +| `INVALID_START_OR_END_NODE` | 관계의 양 끝 노드가 유효하지 않음 | +| `INVALID_PATTERN` | 허용 pattern이 아님 | +| `MISSING_LABEL` | label 없음 | + +기능명세: + +| 기능 | 입력 | 출력 | +|---|---|---| +| 노드 정제 | graph, schema | 유효 node 목록 | +| 관계 정제 | graph, schema, valid nodes | 유효 relationship 목록 | +| property 정제 | node/relationship properties | 스키마에 맞는 property만 유지 | +| 통계 생성 | pruning 결과 | `PruningStats` | + +우리 플랫폼에서는 pruning 결과를 “자동 폐기”만 하지 말고, 사용자에게 “추출됐지만 온톨로지에서 거부된 후보”로 보여주는 기능이 필요하다. 이것이 온톨로지 개선 루프의 핵심 데이터가 된다. + +### 7.8 LexicalGraphBuilder + +파일: `experimental/components/lexical_graph.py` + +생성 그래프: + +```mermaid +flowchart LR + C1["Chunk 0"] -->|FROM_DOCUMENT| D["Document"] + C2["Chunk 1"] -->|FROM_DOCUMENT| D + C1 -->|NEXT_CHUNK| C2 + E1["Entity"] -->|FROM_CHUNK| C1 +``` + +기능명세: + +| 기능 | 생성물 | 설명 | +|---|---|---| +| Document node | `Document` | path, createdAt, metadata, document_type | +| Chunk node | `Chunk` | text, index, metadata, embedding | +| Chunk → Document | `FROM_DOCUMENT` | 문서 소속 | +| Chunk → Chunk | `NEXT_CHUNK` | 원문 순서 | +| Entity → Chunk | `FROM_CHUNK` | 추출 근거 | + +온톨로지 플랫폼에서는 이 provenance 구조를 거의 그대로 사용하면 된다. 다만 문서 소스가 crawler/web이면 Document node에 `url`, `crawl_job_id`, `source_type`, `retrieved_at`, `content_hash` 같은 property를 추가하는 것이 좋다. + +### 7.9 KGWriter + +파일: `experimental/components/kg_writer.py` + +제공 구현: + +- `Neo4jWriter` +- `ParquetWriter` + +#### Neo4jWriter + +기능: + +- node batch upsert +- relationship batch upsert +- non-lexical node에 `__Entity__` label 추가 +- 임시 내부 id index 생성 +- write 이후 임시 label/property 정리 +- Neo4j 버전에 따라 dynamic label 및 variable scope clause 지원 여부 분기 + +중요 파라미터: + +| 파라미터 | 기본값 | 설명 | +|---|---:|---| +| `driver` | 필수 | Neo4j driver | +| `neo4j_database` | `None` | DB 이름 | +| `batch_size` | `1000` | batch write 크기 | +| `clean_db` | `True` | writer 내부 임시 데이터 정리 | + +출력: + +```json +{ + "status": "SUCCESS", + "metadata": { + "statistics": { + "node_count": 0, + "relationship_count": 0, + "nodes_per_label": {}, + "rel_per_type": {}, + "input_files_count": 0, + "input_files_total_size_bytes": 0 + }, + "files": [] + } +} +``` + +#### ParquetWriter + +기능: + +- node label별 Parquet 파일 생성 +- `(head_label, relationship_type, tail_label)`별 relationship Parquet 파일 생성 +- schema constraints를 metadata로 반영 +- Neo4j bulk import 또는 데이터 레이크 연계를 위한 중간 산출물 생성 + +재사용 판단: + +- 실시간/소규모 구축: `Neo4jWriter` +- 대량 batch/검수/승인 workflow: `ParquetWriter` 또는 custom writer 권장 + +### 7.10 EntityResolver + +파일: `experimental/components/resolver.py` + +제공 구현: + +| Resolver | 방식 | 의존성 | +|---|---|---| +| `SinglePropertyExactMatchResolver` | 같은 label + 같은 property 값이면 merge | APOC | +| `SpaCySemanticMatchResolver` | property text embedding cosine similarity | spaCy, numpy, APOC | +| `FuzzyMatchResolver` | RapidFuzz string similarity | rapidfuzz, APOC | + +기능명세: + +| 기능 | 설명 | +|---|---| +| 대상 선택 | 기본 `MATCH (entity:__Entity__)`, `filter_query`로 scope 축소 | +| label별 그룹화 | `__Entity__`, `__KGBuilder__` 제외 | +| property 비교 | 기본 `name`, 다중 property 가능 | +| merge | `apoc.refactor.mergeNodes(..., {properties:'discard', mergeRels:true})` | +| 통계 | resolve 대상 수, 생성/merge 결과 수 | + +주의: + +- 기본 merge 정책은 property 충돌 시 discard이다. +- 온톨로지 플랫폼에서는 자동 merge 전 “후보 그룹 검수” 기능이 필요하다. +- 한국어/영문 alias, 약어, 조직명 변형 처리에는 custom resolver가 필요하다. + +## 8. RAG 및 검색 기능명세 + +### 8.1 VectorRetriever + +파일: `retrievers/vector.py` + +기능: + +- Neo4j vector index 기반 ANN 검색 +- `query_text` 입력 시 embedder로 vector 생성 +- `query_vector` 직접 입력 가능 +- `top_k`, `effective_search_ratio`, metadata `filters` 지원 +- `return_properties` 또는 `result_formatter`로 반환 형식 커스터마이즈 +- Neo4j 2026.01+에서는 `SEARCH` clause와 filterable properties 활용 가능 + +### 8.2 VectorCypherRetriever + +기능: + +- vector 검색 결과를 시작점으로 custom Cypher traversal 수행 +- “유사 chunk 검색 후 주변 entity/관계 확장” 패턴에 적합 + +온톨로지 플랫폼에서 매우 유용한 검색: + +- 특정 문장과 유사한 chunk를 찾고, 해당 chunk에서 추출된 entity와 ontology class를 함께 반환 +- 사용자의 질의와 관련된 provenance, source document, neighbor graph를 함께 표시 + +### 8.3 HybridRetriever / HybridCypherRetriever + +기능: + +- vector search + fulltext search 결합 +- ranker 지원 +- Cypher 확장형 retriever 제공 + +사용처: + +- 고유명사/코드/제품명은 fulltext가 강하고, 의미 검색은 vector가 강하다. +- 온톨로지 탐색 UI에서는 hybrid 검색을 기본으로 두는 것이 좋다. + +### 8.4 Text2CypherRetriever + +파일: `retrievers/text2cypher.py` + +기능: + +- 자연어 질의를 LLM으로 Cypher로 변환 +- 기존 Neo4j schema를 자동 조회하거나 수동 schema 입력 +- few-shot examples 제공 가능 +- 생성 Cypher에서 코드블록 추출, 공백 포함 label/property/type backtick 보정 +- read-only query type만 허용하는 안전장치가 있다. + +주의: + +- 운영 환경에서는 사용자 권한별 schema 제한, allowlist, query timeout, result limit이 필요하다. +- ontology 관리 기능에 직접 연결할 경우 쓰기 쿼리는 별도 승인된 API로만 처리해야 한다. + +### 8.5 ToolsRetriever + +기능: + +- 여러 tool/retriever를 LLM tool calling 방식으로 선택하게 한다. +- 질의 유형별로 vector/hybrid/text2cypher/custom tool을 라우팅하는 데 적합하다. + +### 8.6 External Retriever + +지원: + +- Weaviate +- Pinecone +- Qdrant + +패턴: + +- 외부 vector DB에서 vector 검색 +- 검색 결과 id를 Neo4j graph와 join + +우리 프로젝트가 Neo4j 중심이면 초기에는 보류 가능하다. 다만 대규모 벡터 검색을 별도 인프라로 분리할 가능성이 있으면 adapter 구조는 참고 가치가 높다. + +### 8.7 GraphRAG + +파일: `generation/graphrag.py` + +기능: + +1. retriever로 context 검색 +2. prompt template에 query/context/examples 주입 +3. LLM 호출 +4. answer 반환 +5. 옵션으로 retriever context 포함 반환 +6. message history가 있으면 질의에 대화 요약을 결합 + +입력: + +| 파라미터 | 설명 | +|---|---| +| `query_text` | 사용자 질문 | +| `message_history` | 대화 기록 | +| `examples` | few-shot 예시 | +| `retriever_config` | retriever별 옵션. 예: `top_k` | +| `return_context` | 검색 결과 포함 여부 | +| `response_fallback` | 검색 결과가 없을 때 fallback 답변 | + +우리 플랫폼 적용: + +- “온톨로지 기반 질의응답” +- “이 entity가 어디서 나왔는가?” +- “이 class와 관련된 문서/근거/관계는?” +- “스키마에 맞지 않아 버려진 후보는?” + +## 9. LLM 및 임베딩 어댑터 + +### 9.1 LLM + +지원 구현: + +- `OpenAILLM`, `AzureOpenAILLM` +- `OllamaLLM` +- `VertexAILLM` +- `AnthropicLLM` +- `CohereLLM` +- `MistralAILLM` +- `BedrockLLM` + +공통 특성: + +- `LLMInterface`, `LLMInterfaceV2`, `LLMBase` +- sync/async invoke +- structured output 일부 지원 +- rate limit retry 기본 내장 + - max attempts: 3 + - min wait: 1s + - max wait: 60s + - multiplier: 2 + +### 9.2 임베딩 + +지원 구현: + +- `OpenAIEmbeddings`, `AzureOpenAIEmbeddings` +- `OllamaEmbeddings` +- `VertexAIEmbeddings` +- `CohereEmbeddings` +- `MistralAIEmbeddings` +- `BedrockEmbeddings` +- `SentenceTransformerEmbeddings` + +우리 프로젝트는 provider 독립성이 중요하므로 이 어댑터 계층은 거의 그대로 사용 가능하다. 플랫폼 설정에는 “LLM profile”, “Embedding profile” 개념을 두고, pipeline 실행 시 profile을 주입하는 구조가 좋다. + +## 10. 설정 파일 기반 실행 + +`PipelineRunner.from_config_file(file_path)`로 JSON/YAML 파이프라인 실행이 가능하다. + +기본 SimpleKGPipeline 설정 예: + +```yaml +version_: 1 +template_: SimpleKGPipeline +neo4j_config: + params_: + uri: bolt://localhost:7687 + user: neo4j + password: + resolver_: ENV + var_: NEO4J_PASSWORD +llm_config: + class_: OpenAILLM + params_: + model_name: gpt-5 + api_key: + resolver_: ENV + var_: OPENAI_API_KEY + model_params: + temperature: 0 + max_tokens: 2000 +embedder_config: + class_: OpenAIEmbeddings + params_: + model: text-embedding-3-large +schema: + node_types: + - Person + - label: Organization + properties: + - name: name + type: STRING + relationship_types: + - WORKS_AT + patterns: + - ["Person", "WORKS_AT", "Organization"] +from_file: true +perform_entity_resolution: true +on_error: IGNORE +``` + +우리 플랫폼에서는 이 설정을 DB에 저장하고, GUI/API에서 생성/수정/버전 관리하도록 만들면 된다. + +## 11. 범용 온톨로지 구축 플랫폼 적용 설계 + +### 11.1 그대로 재사용할 1차 기반 + +| 플랫폼 기능 | 사용할 소스 | +|---|---| +| 문서 → KG 실행 | `SimpleKGPipeline` | +| 커스텀 워크플로우 | `Pipeline`, `Component` | +| 온톨로지 스키마 표현 | `GraphSchema`, `NodeType`, `RelationshipType`, `Pattern`, `ConstraintType` | +| 자동 온톨로지 초안 | `SchemaFromTextExtractor` | +| 엔티티/관계 추출 | `LLMEntityRelationExtractor` | +| 스키마 정합성 검증 | `GraphPruning` | +| Neo4j 저장 | `Neo4jWriter` | +| 출처 그래프 | `LexicalGraphBuilder` | +| 중복 엔티티 병합 | `SinglePropertyExactMatchResolver`, `FuzzyMatchResolver` | +| 검색/QA | `VectorRetriever`, `HybridRetriever`, `Text2CypherRetriever`, `GraphRAG` | + +### 11.2 플랫폼에서 추가해야 할 레이어 + +#### A. 프로젝트/테넌트 관리 + +- 온톨로지 프로젝트 생성 +- 데이터소스 연결 +- Neo4j database 또는 namespace 매핑 +- LLM/embedding profile 매핑 + +#### B. 온톨로지 버전 관리 + +현재 `GraphSchema`는 schema 객체일 뿐 버전 관리 기능은 없다. + +필요 기능: + +- schema draft/published 상태 +- version number +- 변경 diff +- migration plan +- label/property rename 이력 +- 이전 버전 추출 결과와 새 버전 비교 + +#### C. 추출 결과 검수 + +현재 pipeline은 추출 후 pruning/write까지 자동으로 갈 수 있다. + +필요 기능: + +- 추출 graph 임시 저장 +- node/relationship/property 단위 승인/반려 +- pruning된 후보 복원/스키마 반영 +- confidence score 또는 LLM rationale 저장 +- 근거 chunk 하이라이트 + +#### D. 도메인별 document loader + +추가 대상: + +- crawler output loader +- HTML loader +- DOCX/PPTX/XLSX loader +- CSV/DB table loader +- API response loader +- code/documentation loader + +#### E. Ontology semantics 확장 + +`GraphSchema`는 property graph schema에 가깝다. 범용 온톨로지 플랫폼이면 다음 개념이 필요할 수 있다. + +| 온톨로지 개념 | 현재 지원 | 확장 필요 | +|---|---:|---| +| Class/Entity type | 지원 | class hierarchy 추가 | +| Object property | relationship type으로 지원 | inverse/symmetric/transitive 추가 | +| Data property | property type으로 지원 | domain/range 강화 | +| Domain/Range | pattern으로 부분 지원 | 다중 domain/range, inheritance 반영 | +| Cardinality | 미지원 | min/max/exact cardinality | +| Equivalent class/property | 미지원 | alias/equivalence 모델 | +| Disjoint class | 미지원 | validation rule | +| SKOS concept | 미지원 | concept scheme, broader/narrower | + +#### F. 운영 안정성 + +- LLM call budget 관리 +- chunk/embedding cache +- 재시도/중단/재개 +- job progress DB 저장 +- 대량 batch queue +- Neo4j transaction timeout 설정 +- Text2Cypher query sandbox + +### 11.3 권장 내부 모듈 구조 + +우리 프로젝트에 통합할 때는 원본 소스를 직접 수정하기보다 다음 형태가 좋다. + +```text +crawler_platform/ + ontology/ + schemas.py # GraphSchema 래퍼, 버전/상태/소유자 메타데이터 + pipeline_profiles.py # LLM/embedding/Neo4j profile + jobs.py # KG build job 상태 + adapters/ + documents.py # crawler output → LoadedDocument + schema.py # 우리 온톨로지 모델 ↔ GraphSchema + services/ + kg_builder.py # SimpleKGPipeline 실행 래퍼 + review.py # 추출 결과 검수 + search.py # GraphRAG/Text2Cypher 래퍼 + resolver.py # merge 후보/실행 +``` + +## 12. 상세 기능명세 + +### 12.1 온톨로지 프로젝트 관리 + +| ID | 기능 | 설명 | 우선순위 | +|---|---|---|---:| +| ONT-PROJ-001 | 프로젝트 생성 | 이름, 설명, Neo4j DB/profile, 기본 언어 설정 | P0 | +| ONT-PROJ-002 | 데이터소스 연결 | 파일, 크롤링 결과, URL, DB table 등록 | P0 | +| ONT-PROJ-003 | LLM profile 선택 | provider/model/key/params 선택 | P0 | +| ONT-PROJ-004 | Embedding profile 선택 | provider/model/dimension 설정 | P0 | +| ONT-PROJ-005 | 실행 이력 조회 | pipeline run 상태/시간/token/비용/오류 | P1 | + +### 12.2 온톨로지 스키마 관리 + +| ID | 기능 | 설명 | 우선순위 | +|---|---|---|---:| +| ONT-SCH-001 | 수동 schema 작성 | node type, relationship type, property, pattern 작성 | P0 | +| ONT-SCH-002 | 자동 schema 추출 | 문서 샘플에서 `SchemaFromTextExtractor` 실행 | P0 | +| ONT-SCH-003 | schema 검증 | `GraphSchema` Pydantic validation + custom rule | P0 | +| ONT-SCH-004 | schema 버전 발행 | draft → published | P0 | +| ONT-SCH-005 | schema diff | version 간 label/property/pattern 변경 비교 | P1 | +| ONT-SCH-006 | constraint 관리 | uniqueness/key/existence 제약 관리 | P1 | +| ONT-SCH-007 | class hierarchy | 상위/하위 class 정의 | P2 | + +### 12.3 KG 구축 + +| ID | 기능 | 설명 | 우선순위 | +|---|---|---|---:| +| ONT-KG-001 | 파일 기반 KG 구축 | PDF/Markdown → SimpleKGPipeline | P0 | +| ONT-KG-002 | 텍스트 기반 KG 구축 | crawler text 또는 inline text → SimpleKGPipeline | P0 | +| ONT-KG-003 | lexical graph 생성 | Document/Chunk/provenance graph 생성 | P0 | +| ONT-KG-004 | chunk embedding | Chunk embedding property 저장 | P0 | +| ONT-KG-005 | 스키마 기반 추출 | published GraphSchema로 LLM 추출 | P0 | +| ONT-KG-006 | 자동 스키마 추출 후 KG 구축 | `schema="EXTRACTED"` 사용 | P1 | +| ONT-KG-007 | 자유 추출 | `schema="FREE"` 사용 | P1 | +| ONT-KG-008 | pruning 통계 저장 | 제거 노드/관계/property 기록 | P0 | +| ONT-KG-009 | 추출 결과 임시 저장 | writer 전 검수용 graph 저장 | P1 | +| ONT-KG-010 | Parquet export | 대량 import 또는 검수 산출물 | P2 | + +### 12.4 검수 및 승인 + +| ID | 기능 | 설명 | 우선순위 | +|---|---|---|---:| +| ONT-REV-001 | 추출 후보 목록 | node/relationship/property 후보 조회 | P1 | +| ONT-REV-002 | 근거 chunk 표시 | `FROM_CHUNK` 관계 기반 원문 근거 표시 | P1 | +| ONT-REV-003 | 승인/반려 | 후보 단위 상태 변경 | P1 | +| ONT-REV-004 | pruning 후보 검토 | 스키마 위반으로 제거된 후보 검토 | P1 | +| ONT-REV-005 | schema 개선 제안 | 반복 pruning된 후보를 schema 후보로 제안 | P2 | + +### 12.5 엔티티 해소 + +| ID | 기능 | 설명 | 우선순위 | +|---|---|---|---:| +| ONT-RES-001 | exact match merge | label + name exact match | P0 | +| ONT-RES-002 | fuzzy merge 후보 | RapidFuzz로 후보 계산 | P1 | +| ONT-RES-003 | semantic merge 후보 | spaCy 또는 embedding similarity | P2 | +| ONT-RES-004 | merge 검수 | 후보 그룹 승인 후 APOC merge | P1 | +| ONT-RES-005 | merge 이력 | 병합 전후 node id, property 충돌 기록 | P1 | + +### 12.6 검색 및 질의응답 + +| ID | 기능 | 설명 | 우선순위 | +|---|---|---|---:| +| ONT-SEA-001 | vector 검색 | Chunk vector index 검색 | P0 | +| ONT-SEA-002 | hybrid 검색 | vector + fulltext | P1 | +| ONT-SEA-003 | graph 확장 검색 | VectorCypher/HybridCypher로 주변 graph 반환 | P0 | +| ONT-SEA-004 | Text2Cypher | 자연어 → read-only Cypher | P1 | +| ONT-SEA-005 | GraphRAG 답변 | 검색 context 기반 답변 생성 | P1 | +| ONT-SEA-006 | provenance 포함 답변 | 답변에 문서/chunk 근거 포함 | P1 | + +## 13. 통합 시 주의사항 + +### 13.1 원본 코드를 직접 수정하지 않는 것이 좋다 + +이 프로젝트의 KG builder는 `experimental`이다. 원본을 직접 수정하면 upstream 반영이 어려워진다. 권장 방식: + +1. `neo4j-graphrag==1.16.0`으로 버전 고정 +2. 우리 프로젝트에 adapter/wrapper 작성 +3. 필요한 custom component만 우리 namespace에 구현 +4. 원본 API 변경 시 wrapper만 수정 + +### 13.2 LLM JSON 품질 + +`json-repair`와 structured output이 있더라도 LLM 추출은 완전하지 않다. + +필수 보완: + +- 스키마 grounding prompt 강화 +- examples few-shot 관리 +- chunk별 실패율 기록 +- pruning 결과 분석 +- 사람이 승인하는 workflow + +### 13.3 Neo4j/APOC 의존성 + +KG writer와 resolver는 Neo4j 버전/APOC에 민감하다. + +운영 체크: + +- Neo4j 버전 확인 +- APOC core 설치 확인 +- `apoc.refactor.mergeNodes` 사용 가능 여부 확인 +- vector/fulltext index 생성 권한 확인 +- multi database 사용 시 `neo4j_database` 일관성 유지 + +### 13.4 Text2Cypher 보안 + +`Text2CypherRetriever`는 read-only 검사를 갖지만, 운영 서비스에서는 추가 안전장치가 필요하다. + +- 허용 schema 제한 +- query timeout +- result limit +- 금지 키워드 검사 +- 사용자별 권한 필터 +- 쓰기/삭제/관리 명령 차단 + +## 14. 초기 적용 로드맵 + +### Phase 1: 최소 KG 구축 + +목표: crawler 결과 또는 텍스트를 Neo4j KG로 저장. + +작업: + +1. `neo4j-graphrag[openai,experimental]` 의존성 추가 +2. Neo4j 연결 profile 모델 추가 +3. LLM/embedding profile 모델 추가 +4. crawler output → `LoadedDocument` adapter 작성 +5. `SimpleKGPipeline(from_file=False)` 실행 service 작성 +6. `GraphSchema` 수동 입력 API 작성 +7. `Neo4jWriter` 결과 통계 저장 + +### Phase 2: 온톨로지 초안/검수 + +목표: 자동 schema 추출과 추출 결과 검수. + +작업: + +1. `schema="EXTRACTED"` 실행 지원 +2. 추출 schema draft 저장 +3. `GraphPruning` 결과 저장 +4. pruned item 검토 화면/API +5. 승인된 schema version 발행 + +### Phase 3: 검색/질의응답 + +목표: 구축된 KG를 탐색하고 질의응답. + +작업: + +1. Chunk vector index 생성 자동화 +2. `VectorCypherRetriever`로 chunk → entity/provenance 검색 +3. `HybridRetriever` 도입 +4. `GraphRAG` service 작성 +5. Text2Cypher read-only 질의 API 추가 + +### Phase 4: 엔티티 해소/품질관리 + +목표: 중복 entity와 schema 품질 개선. + +작업: + +1. exact match resolver 실행 +2. fuzzy 후보 생성 +3. merge 후보 검수 +4. merge 이력 저장 +5. 반복 pruning 기반 schema 개선 추천 + +## 15. 결론 + +`neo4j-graphrag-python`은 우리 범용 온톨로지 구축 플랫폼의 “KG 생성 엔진”과 “GraphRAG 검색 엔진”으로 매우 적합하다. 특히 `GraphSchema`, `SimpleKGPipeline`, `LLMEntityRelationExtractor`, `GraphPruning`, `Neo4jWriter`, `VectorCypherRetriever`, `Text2CypherRetriever`는 거의 그대로 가져다 쓸 수 있다. + +다만 이 프로젝트가 제공하는 것은 “라이브러리/엔진”이지 “플랫폼”은 아니다. 우리 프로젝트의 핵심 차별점은 다음 레이어에서 만들어야 한다. + +- 온톨로지 프로젝트/버전 관리 +- schema draft/publish workflow +- 추출 결과 검수 및 provenance UI +- crawler 결과와의 자연스러운 연결 +- 대량 실행 job 관리 +- entity merge 후보 검수 +- Text2Cypher 보안/권한 레이어 + +따라서 권장 전략은 원본 소스를 복사해 수정하기보다, `neo4j-graphrag`를 고정 버전 의존성으로 두고 우리 플랫폼 서비스가 이 라이브러리를 orchestration하는 방식이다. 필요한 경우 custom loader, custom splitter, custom resolver, custom writer만 우리 코드베이스에 추가한다. diff --git a/오픈소스분석자료/OntoCast_분석_및_기능명세.md b/오픈소스분석자료/OntoCast_분석_및_기능명세.md new file mode 100644 index 0000000..96ba50e --- /dev/null +++ b/오픈소스분석자료/OntoCast_분석_및_기능명세.md @@ -0,0 +1,1219 @@ +# OntoCast 분석 및 기능명세 + +분석 대상: `C:\Users\lasta\MyProject\AI\참고\ontocast-main` +분석 기준일: 2026-05-13 +라이선스: Apache License 2.0 +프로젝트 성격: 문서에서 RDF 지식그래프를 생성하기 위한 에이전트형 온톨로지 보조 triple extraction 프레임워크 + +--- + +## 1. 결론 요약 +이 오픈소스코드는 사용자 직접 데이터를 입력해서 온톨로지 구축하는 기능을 추가하고 그 용도로 사용하는 것으로 한다. +OntoCast는 범용 온톨로지 구축 플랫폼의 기본 소스로 재사용 가치가 높다. 특히 다음 영역은 거의 원형 그대로 가져와도 된다. + +- `AgentState`, `UnitFactsState`, `UnitOntologyState` 기반 상태 모델 +- LangGraph 기반 문서 처리 워크플로우 +- 온톨로지 선택, 생성, 갱신, 비평, 재시도 루프 +- GraphUpdate 기반 SPARQL 증분 갱신 구조 +- RDFGraph, Ontology, ContentUnit 도메인 모델 +- LLM 응답 캐싱 및 budget tracking +- Fuseki, Neo4j, filesystem triple store 추상화 +- embedding 기반 entity aggregation, URI 정규화, owl:sameAs 생성 + +다만 OntoCast는 “완성된 범용 온톨로지 구축 플랫폼”이라기보다 “문서 입력 → 온톨로지/팩트 RDF 생성 → 저장”에 집중된 코어 엔진이다. 앞으로 만들 플랫폼에서는 사용자/프로젝트/작업 관리, 온톨로지 편집 UI, 검수 워크플로우, 권한, 배치 운영, 데이터셋 카탈로그, 품질 지표 대시보드가 별도 상위 계층으로 필요하다. + +--- + +## 2. 프로젝트 개요 + +### 2.1 목적 + +OntoCast는 비정형 문서 또는 JSON/text 입력을 받아 다음 산출물을 만든다. + +- 도메인 온톨로지 RDF/Turtle +- 문서에서 추출된 사실 triple RDF/Turtle +- 문서 단위 aggregated facts graph +- 온톨로지 버전/해시/lineage metadata +- LLM 사용량과 triple 생성량 budget metadata + +핵심 아이디어는 온톨로지를 먼저 선택하거나 새로 만들고, 그 온톨로지를 기준으로 문서의 사실 정보를 RDF triple로 추출하는 것이다. + +### 2.2 기술 스택 + +`pyproject.toml` 기준 주요 의존성은 다음과 같다. + +| 영역 | 사용 기술 | +|---|---| +| 언어 | Python 3.12 이상 | +| 워크플로우 | LangGraph | +| LLM 연동 | LangChain, OpenAI, Ollama | +| API 서버 | Robyn | +| RDF 처리 | rdflib, pyld, oxrdflib, owlready2 | +| 그래프 저장소 | Fuseki, Neo4j n10s, filesystem | +| 문서 처리 옵션 | docling, easyocr | +| 청킹/임베딩 | langchain-experimental, sentence-transformers, simsimd | +| 군집화 | hdbscan, umap-learn, rapidfuzz | +| 설정 | pydantic-settings | +| CLI | click | + +### 2.3 실행 형태 + +OntoCast는 두 가지 모드로 동작한다. + +1. API 서버 모드 + - 엔트리포인트: `ontocast.cli.serve:run` + - 명령: `ontocast --env-file .env` + - 주요 엔드포인트: `/health`, `/info`, `/process`, `/flush` + +2. 파일 배치 처리 모드 + - 명령: `ontocast --env-file .env --input-path ./docs` + - 입력 경로의 JSON/PDF/지원 문서를 순회 처리 + +--- + +## 3. 디렉터리 구조 분석 + +| 경로 | 역할 | +|---|---| +| `ontocast/config.py` | Pydantic 기반 전체 설정 모델 | +| `ontocast/cli/serve.py` | Robyn API 서버와 CLI 엔트리포인트 | +| `ontocast/stategraph/` | LangGraph 워크플로우 정의, 라우팅, 병렬 node factory | +| `ontocast/agent/` | 문서 변환, 청킹, 온톨로지 선택/생성/비평, facts 생성/비평 agent | +| `ontocast/onto/` | RDFGraph, Ontology, AgentState, SPARQL model 등 핵심 도메인 모델 | +| `ontocast/tool/` | LLM, converter, chunker, cache, SPARQL, triple store 등 도구 | +| `ontocast/tool/agg/` | entity disambiguation 및 graph aggregation | +| `ontocast/tool/triple_manager/` | Fuseki, Neo4j, filesystem 저장소 구현 | +| `ontocast/prompt/` | LLM prompt template | +| `docs/` | 사용자 가이드와 API reference | +| `test/` | 단위/통합 테스트 | +| `data/` | 예시 ontology, PDF, JSON, chunk 데이터 | +| `docker/` | Fuseki, Neo4j, Qdrant docker-compose | + +--- + +## 4. 전체 처리 흐름 + +코드 기준 핵심 워크플로우는 `ontocast/stategraph/create.py`의 `create_agent_graph()`에 정의되어 있다. + +```mermaid +flowchart TD + START([START]) --> CONVERT[CONVERT_TO_MD] + CONVERT --> CHUNK[CHUNK] + CHUNK --> SELECT[SELECT_ONTOLOGY] + SELECT -->|기존 온톨로지 없음| BOOTSTRAP[BOOTSTRAP_ONTOLOGY] + SELECT -->|온톨로지 생성/갱신 필요| RENDER_ONTO[RENDER_ONTOLOGY_UPDATE] + SELECT -->|facts만 생성| RENDER_FACTS[RENDER_FACTS] + BOOTSTRAP --> RENDER_ONTO + RENDER_ONTO --> NORMALIZE[NORMALIZE_ONTOLOGY_UPDATES] + NORMALIZE --> CONSOLIDATE[CONSOLIDATE_ONTOLOGY] + CONSOLIDATE -->|facts 생성 필요| RENDER_FACTS + CONSOLIDATE -->|ontology only| SERIALIZE[SERIALIZE] + RENDER_FACTS --> MERGE[MERGE_FACTS] + MERGE --> SERIALIZE + SERIALIZE --> END([END]) +``` + +### 4.1 단계별 설명 + +| 단계 | 구현 위치 | 설명 | +|---|---|---| +| 문서 변환 | `agent/convert_document.py` | PDF/지원 문서를 markdown/text로 변환하거나 JSON/text를 파싱 | +| 청킹 | `agent/chunk_text.py`, `tool/chunk/` | 긴 문서를 semantic chunk 또는 fallback chunk로 분할 | +| 온톨로지 선택 | `agent/select_ontology.py` | 기존 ontology 목록을 LLM에 제시하고 문서에 맞는 ontology를 선택 | +| 온톨로지 부트스트랩 | `stategraph/node_factories.py` | 기존 ontology가 없으면 문서 발췌문으로 seed ontology 생성 | +| 온톨로지 map | `stategraph/atomic.py`, `agent/render_ontology.py` | content unit별 ontology delta 또는 fresh ontology 생성 | +| 온톨로지 critic | `agent/criticise_ontology.py` | ontology 품질을 평가하고 개선 suggestions 생성 | +| 온톨로지 normalize | `agent/normalize_ontology.py` | per-unit ontology delta를 GraphUpdate로 병합, provenance 제거 | +| 선택적 consolidation | `stategraph/node_factories.py` | 중복 class/property 정리, hierarchy 일관성 개선 | +| facts map | `agent/render_facts.py` | ontology를 기준으로 content unit별 facts graph 생성 | +| facts critic | `agent/criticise_facts.py` | facts triple이 원문과 ontology에 맞는지 평가 | +| facts merge | `stategraph/node_factories.py`, `tool/agg/` | content unit별 graph를 entity aggregation 후 병합 | +| 저장 | `agent/serialize.py`, `toolbox.py` | filesystem/Fuseki/Neo4j에 ontology와 facts 저장 | + +--- + +## 5. 핵심 아키텍처 + +### 5.1 ToolBox + +구현 위치: `ontocast/toolbox.py` + +`ToolBox`는 시스템의 dependency container이다. 서버 시작 시 `Config`를 받아 다음 객체를 초기화한다. + +- `LLMTool` +- `Cacher` +- `AtomicToolBox` +- `FilesystemTripleStoreManager` +- `FusekiTripleStoreManager` +- `Neo4jTripleStoreManager` +- `OntologyManager` +- `ConverterTool` +- `ChunkerTool` +- `EmbeddingBasedAggregator` +- `SPARQLTool` +- `GraphVersionManager` +- `DiffTool` + +재사용 판단: 매우 높음. +범용 플랫폼에서는 ToolBox를 application service container로 삼되, 사용자별/프로젝트별 설정 주입이 가능하도록 확장하면 된다. + +### 5.2 Config + +구현 위치: `ontocast/config.py` + +설정은 다음 계층으로 나뉜다. + +- `LLMConfig`: provider, model, temperature, base_url, api_key +- `ChunkConfig`: semantic chunk threshold, min/max chunk size +- `ServerConfig`: port, recursion limit, render mode, parallel worker, retry 횟수 +- `Neo4jConfig` +- `FusekiConfig` +- `DomainConfig` +- `PathConfig` +- `WebSearchConfig` +- `AggregationConfig` +- `ToolConfig` +- `Config` + +재사용 판단: 높음. +다만 플랫폼형 서비스에서는 `.env` 중심 설정 외에 DB 저장형 프로젝트 설정, 사용자별 secret 관리가 필요하다. + +### 5.3 AgentState + +구현 위치: `ontocast/onto/state.py` + +`AgentState`는 전체 문서 처리 상태를 담는 중앙 모델이다. + +주요 필드: + +- `input_text` +- `files` +- `content_units` +- `current_content_unit` +- `current_ontology` +- `aggregated_facts` +- `ontology_user_instruction` +- `facts_user_instruction` +- `dataset` +- `source_url` +- `ontology_updates`, `ontology_updates_applied` +- `facts_updates`, `facts_updates_applied` +- `parallel_facts_units` +- `ontology_units` +- `ontology_provenance_artifact` +- `failure_stage`, `failure_reason` +- `status`, `statuses` +- `node_visits`, `max_visits` +- `render_mode` +- `ontology_max_triples` +- `context_manager` +- `suggestions` +- `budget_tracker` + +중요 메서드: + +- `render_updated_graph()`: GraphUpdate를 RDFGraph에 적용 +- `update_ontology()`: pending ontology update 적용 +- `update_facts()`: pending facts update 적용 +- `doc_iri`, `doc_namespace`, `graph_uri`: 문서 기반 IRI 생성 +- `get_context_for_agent()`, `update_context_for_agent()`: agent 간 context 관리 + +재사용 판단: 매우 높음. +범용 플랫폼에서는 이 모델이 “작업 실행 상태”의 기본 스키마가 될 수 있다. + +### 5.4 Unit State + +구현 위치: `ontocast/onto/unit_states.py` + +병렬 map 단계에서 content unit 하나를 독립 처리하기 위한 상태 모델이다. + +- `UnitState` +- `UnitFactsState` +- `UnitOntologyState` + +이 구조 덕분에 전체 문서 상태를 매번 공유하지 않고 unit별 renderer/critic loop를 병렬 실행할 수 있다. + +재사용 판단: 높음. +대규모 문서, 웹 크롤링 문서, 다중 파일 ingestion에 유리하다. + +--- + +## 6. GraphUpdate / SPARQL 증분 갱신 구조 + +구현 위치: + +- `ontocast/onto/sparql_models.py` +- `ontocast/onto/state.py` +- `agent/render_ontology.py` +- `agent/render_facts.py` + +OntoCast의 중요한 장점은 LLM이 매번 전체 Turtle graph를 다시 생성하지 않고, 변경분만 `GraphUpdate`로 출력하게 하는 구조이다. + +### 6.1 주요 모델 + +| 모델 | 역할 | +|---|---| +| `TripleOp` | insert/delete triple operation | +| `GenericSparqlQuery` | 직접 SPARQL update query | +| `GraphUpdate` | 여러 `TripleOp` 또는 custom query 묶음 | +| `GraphUpdateRenderReport` | LLM renderer의 구조화 출력 | + +### 6.2 동작 방식 + +1. 현재 ontology 또는 facts graph를 prompt에 포함한다. +2. LLM은 전체 TTL 대신 `GraphUpdate`를 반환한다. +3. `GraphUpdate.generate_sparql_queries()`가 SPARQL update query로 변환한다. +4. `AgentState.render_updated_graph()`가 rdflib graph에 update를 적용한다. +5. max triple 제한을 넘으면 ontology update를 건너뛴다. + +### 6.3 장점 + +- 출력 토큰 절감 +- 변경 이력 추적 용이 +- ontology versioning과 연결 가능 +- LLM이 기존 graph를 파괴하는 위험 감소 +- critic feedback을 update operation으로 재적용 가능 + +재사용 판단: 최상. +범용 온톨로지 구축 플랫폼의 핵심 기능으로 삼는 것이 좋다. + +--- + +## 7. 온톨로지 관리 기능 + +### 7.1 Ontology 모델 + +구현 위치: `ontocast/onto/ontology.py` + +Ontology는 RDF graph와 메타데이터를 함께 들고 있다. + +주요 속성: + +- `ontology_id` +- `title` +- `description` +- `version` +- `iri` +- `graph` +- `hash` +- `parent_hashes` +- `created_at` +- `updated_at` +- `initial_version` + +주요 기능: + +- RDF graph와 object property 동기화 +- version/hash lineage 관리 +- ontology 변경 시 updated version 파생 +- null ontology 판별 +- ontology 설명 문자열 생성 + +### 7.2 OntologyManager + +구현 위치: `ontocast/tool/ontology_manager.py` + +기능: + +- ontology 목록 보관 +- ontology 추가 +- 사용 가능한 ontology 존재 여부 확인 +- ontology selection agent에 목록 제공 + +현재는 단순 in-memory manager에 가깝다. 플랫폼에서는 DB 기반 registry로 확장해야 한다. + +### 7.3 Versioning + +구현 위치: `ontocast/tool/graph_version_manager.py` + +문서와 README 기준 제공 기능: + +- semantic version increment +- hash-based lineage +- parent hash 추적 +- ontology graph diff 분석 +- version statistics + +도입 시 유의점: + +- 플랫폼에서는 ontology version을 “초안, 검수중, 승인, 폐기” 같은 lifecycle과 연결해야 한다. +- hash lineage는 내부 무결성 관리에 유용하지만 사용자 UI에서는 semantic version과 변경 요약을 앞세우는 편이 좋다. + +--- + +## 8. 문서 입력 및 청킹 + +### 8.1 Document Conversion + +구현 위치: + +- `agent/convert_document.py` +- `tool/converter.py` + +지원 입력: + +- JSON +- TXT +- docling이 지원하는 문서 포맷 +- README 기준 PDF, Markdown 등 + +JSON 입력의 특수 필드: + +- `text`: 처리 대상 텍스트 +- `ontology_user_instruction`: ontology 생성/갱신 지시 +- `facts_user_instruction`: facts 추출 지시 +- `url`: provenance용 source URL + +주의점: + +- `convert_document()`는 주석상 “processing only one file”이라고 되어 있으며, 루프 구조도 마지막 파일 기준 상태 갱신에 가깝다. +- 플랫폼에서 다중 파일을 하나의 corpus로 처리하려면 이 부분은 확장해야 한다. + +### 8.2 Chunking + +구현 위치: + +- `agent/chunk_text.py` +- `tool/chunk/chunker.py` +- `tool/chunk/util.py` + +설정: + +- `CHUNK_BREAKPOINT_THRESHOLD_TYPE` +- `CHUNK_BREAKPOINT_THRESHOLD_AMOUNT` +- `CHUNK_MIN_SIZE` +- `CHUNK_MAX_SIZE` + +기능: + +- 입력 텍스트를 content unit으로 분할 +- 각 content unit에 index와 doc IRI 부여 +- `max_chunks`로 앞부분 일부만 처리 가능 + +도입 의견: + +- 웹 크롤러/문서 수집 플랫폼과 결합할 경우, 문서 단위 chunk metadata를 더 풍부하게 만들어야 한다. +- 예: section title, page number, source URL, crawl timestamp, MIME type, language. + +--- + +## 9. LLM Agent 상세 + +### 9.1 Ontology Selection Agent + +구현 위치: `agent/select_ontology.py` + +기능: + +- 현재 `OntologyManager`가 가진 ontology 목록을 번호 목록으로 구성 +- 문서의 first/middle/last chunk 일부를 발췌해 대표 excerpt 생성 +- LLM이 적절한 ontology index를 선택 +- 없으면 null ontology로 진행 + +명세: + +- 입력: `AgentState.content_units`, ontology 목록 +- 출력: `AgentState.current_ontology` +- 실패 처리: ontology가 없으면 `NULL_ONTOLOGY` + +개선 필요: + +- 코드 주석은 `answer_index == 0`을 None으로 설명하지만 dynamic model은 `1..num_ontologies+1` 범위를 사용한다. 실제 None 선택은 `num_ontologies + 1`이어야 자연스럽다. 이 부분은 가져오기 전에 테스트와 함께 수정하는 것이 좋다. + +### 9.2 Ontology Renderer + +구현 위치: `agent/render_ontology.py` + +기능: + +- fresh ontology 생성 +- 기존 ontology에 대한 GraphUpdate 생성 +- known prefix를 추출해 RDFGraph parser context에 제공 +- user instruction과 critic suggestions 반영 +- optional external evidence 반영 + +모드: + +| 상황 | 출력 | +|---|---| +| seed ontology 없음 | `OntologyRenderReport` 안의 fresh `Ontology` | +| seed ontology 있음 | `GraphUpdateRenderReport` 안의 `GraphUpdate` | + +### 9.3 Ontology Critic + +구현 위치: `agent/criticise_ontology.py` + +기능: + +- ontology가 문서 domain을 잘 표현하는지 평가 +- score와 success 반환 +- 실패 시 actionable fixes와 systemic critique summary를 `Suggestions`로 변환 +- score > 90이면 success로 간주 + +출력 모델: + +- `OntologyCritiqueReport` +- `TripleFix` +- `Suggestions` + +### 9.4 Facts Renderer + +구현 위치: `agent/render_facts.py` + +기능: + +- ontology graph를 기준으로 source text에서 facts RDF 생성 +- fresh facts graph가 비어 있으면 Turtle 생성 +- 기존 facts graph가 있으면 GraphUpdate 생성 +- ontology prefix와 namespace를 prompt에 포함 +- facts user instruction 반영 + +출력: + +- fresh: `FactsRenderReport` +- update: `GraphUpdateRenderReport` + +### 9.5 Facts Critic + +구현 위치: `agent/criticise_facts.py` + +기능: + +- facts graph가 원문을 충분히 표현하는지 평가 +- ontology와 facts graph의 정합성 확인 +- score > 90 또는 success면 통과 +- 실패 시 `Suggestions` 생성 + +--- + +## 10. Retry Loop 및 외부 근거 검색 + +구현 위치: + +- `stategraph/atomic.py` +- `agent/external_evidence.py` +- `tool/web_search.py` + +### 10.1 Atomic Loop + +`facts_loop()`와 `ontology_loop()`는 다음 패턴을 따른다. + +1. renderer 실행 +2. renderer 실패 시 external evidence request 확인 +3. 필요하면 검색 계획 수립 +4. 검색 실행 +5. renderer 재실행 +6. critic 실행 +7. critic 실패 시 suggestions 저장 +8. critic이 external evidence를 요청하면 검색 후 critic 재실행 +9. retry budget 소진 시 마지막 상태 반환 + +### 10.2 Web Search + +설정 위치: `WebSearchConfig` + +지원 provider: + +- DuckDuckGo + +주요 옵션: + +- `WEB_SEARCH_ENABLED` +- `WEB_SEARCH_TOP_K` +- `WEB_SEARCH_TIMEOUT_SECONDS` +- `WEB_SEARCH_ALLOWED_DOMAINS` +- `WEB_SEARCH_BLOCKED_DOMAINS` +- ontology/facts renderer/critic별 enable flag + +도입 의견: + +- 범용 플랫폼에서 외부 근거 검색은 매우 유용하나, 운영 환경에서는 검색 결과 provenance와 source trust policy가 필요하다. +- allowed/blocked domain 설정은 프로젝트별 정책으로 승격해야 한다. + +--- + +## 11. Facts Aggregation / Entity Disambiguation + +구현 위치: + +- `tool/aggregate.py` +- `tool/agg/aggregate.py` +- `tool/agg/normalizer.py` +- `tool/agg/clustering.py` +- `tool/agg/rewriter.py` +- `tool/agg/uri_builder.py` + +### 11.1 목적 + +각 content unit에서 독립 생성된 facts graph는 같은 entity를 서로 다른 URI로 표현할 수 있다. Aggregator는 이를 정규화하고 병합한다. + +### 11.2 파이프라인 + +1. content unit별 entity 수집 +2. entity를 semantic context가 포함된 representation으로 정규화 +3. sentence-transformers embedding 생성 +4. cosine similarity 기반 candidate clustering +5. symbolic identity check로 부적절한 merge 방지 +6. canonical representative 선택 +7. URI policy에 따라 최종 URI 생성 +8. graph rewrite 수행 +9. 필요 시 `owl:sameAs` 링크 추가 + +### 11.3 Entity 분류 + +`EntityClassification`: + +- `fact` +- `known_ontology` +- `tentative_ontology` + +표준 RDF/OWL/RDFS/XSD/Schema/PROV namespace는 built-in vocabulary로 취급한다. + +### 11.4 URI 정책 + +`URIBuilder`는 role에 따라 URI local name을 정규화한다. + +- class: PascalCase +- property: lowerCamelCase +- instance: normalized name 또는 structured id 유지 + +도입 판단: 높음. +범용 플랫폼에서 문서/크롤링 단위별 entity 중복 문제를 줄이는 핵심 기능이다. + +--- + +## 12. Triple Store 통합 + +구현 위치: `tool/triple_manager/` + +### 12.1 공통 인터페이스 + +`TripleStoreManager`가 정의하는 핵심 메서드: + +- `fetch_ontologies() -> list[Ontology]` +- `serialize_graph(graph, **kwargs)` +- `serialize(o: Ontology | RDFGraph, **kwargs)` +- `clean(dataset: str | None = None)` + +### 12.2 FilesystemTripleStoreManager + +역할: + +- 로컬 디렉터리에 ontology/facts graph 저장 +- 개발/테스트/간단 배치에 적합 + +### 12.3 FusekiTripleStoreManager + +역할: + +- Apache Jena Fuseki dataset에 RDF 저장 +- ontology와 facts dataset 분리 지원 +- named graph 기반 저장에 적합 +- `/flush?dataset=...` 동작과 연결 + +### 12.4 Neo4jTripleStoreManager + +역할: + +- Neo4j/n10s 기반 RDF graph 저장 +- GraphRAG나 graph query UI와 연결하기 좋음 + +도입 의견: + +- 범용 온톨로지 플랫폼의 canonical store는 Fuseki/RDF store가 더 자연스럽다. +- Neo4j는 분석/시각화/GraphRAG projection store로 병행하는 구성이 좋다. + +--- + +## 13. API 기능명세 + +구현 위치: `ontocast/cli/serve.py` + +### 13.1 GET `/health` + +목적: 서비스 상태 확인 + +성공 응답: + +```json +{ + "status": "healthy", + "version": "0.1.1", + "llm_provider": "openai" +} +``` + +실패 조건: + +- LLM 미초기화 +- 내부 예외 + +### 13.2 GET `/info` + +목적: 서비스 metadata 및 capability 제공 + +응답 필드: + +- `name` +- `version` +- `description` +- `capabilities` +- `input_types` +- `output_types` + +### 13.3 POST `/process` + +목적: 문서를 처리해 ontology/facts RDF를 생성 + +지원 Content-Type: + +- `application/json` +- `multipart/form-data` + +Query Parameters: + +| 이름 | 설명 | +|---|---| +| `dataset` | Fuseki dataset override | +| `render_mode` | `ontology`, `facts`, `ontology_and_facts` | +| `ontology_user_instruction` | ontology 추출 지시 | +| `facts_user_instruction` | facts 추출 지시 | + +Form fields: + +- file +- `ontology_user_instruction` +- `facts_user_instruction` + +JSON body: + +```json +{ + "text": "처리할 텍스트", + "url": "https://source.example/doc", + "ontology_user_instruction": "장소와 조직 중심으로 온톨로지를 구성", + "facts_user_instruction": "인물-조직 관계를 우선 추출" +} +``` + +성공 응답: + +```json +{ + "status": "success", + "data": { + "facts": "... turtle ...", + "ontology": "... turtle ..." + }, + "metadata": { + "status": "success", + "chunks_processed": 10, + "chunks_remaining": 0, + "budget": { + "chars_sent": 12345, + "chars_received": 6789, + "calls_count": 12, + "ontology_triples_generated": 50, + "facts_triples_generated": 300, + "ontology_operations_count": 5, + "facts_operations_count": 10 + } + } +} +``` + +실패 응답: + +```json +{ + "status": "error", + "error": "오류 메시지", + "error_type": "ExceptionType", + "error_details": { + "stage": "failure stage", + "reason": "failure reason" + } +} +``` + +### 13.4 POST `/flush` + +목적: triple store 데이터 삭제 + +Query Parameters: + +| 이름 | 설명 | +|---|---| +| `dataset` | Fuseki에서 특정 dataset만 삭제 | + +주의: + +- irreversible operation +- Neo4j/filesystem에서는 dataset parameter 무시 + +--- + +## 14. CLI 기능명세 + +`pyproject.toml`의 `[project.scripts]` 기준 공개 CLI: + +| 명령 | 엔트리포인트 | 기능 | +|---|---|---| +| `ontocast` | `ontocast.cli.serve:run` | API 서버 실행 또는 input path 배치 처리 | +| `cmp-states` | `ontocast.cli.cmp_states:main` | 저장된 AgentState 비교 | +| `pdfs-to-markdown` | `ontocast.cli.pdfs_to_markdown:main` | PDF를 markdown으로 변환 | +| `plot-graph` | `ontocast.cli.plot_graph:main` | graph 시각화/문서 내 mermaid 갱신 | +| `test-api` | `ontocast.cli.test_api:main` | API 호출 테스트 | + +추가 CLI 파일: + +- `batch_process.py`: 비동기 파일 배치 처리 +- `merge_ontologies.py`: ontology 병합 +- `split_chunks.py`: JSON을 markdown으로 변환 후 chunk 분할 + +--- + +## 15. 설정 기능명세 + +### 15.1 필수 설정 + +OpenAI 사용 시: + +| 변수 | 설명 | +|---|---| +| `LLM_PROVIDER=openai` | LLM provider | +| `LLM_API_KEY` | OpenAI API key | +| `ONTOCAST_WORKING_DIRECTORY` | 작업 디렉터리 | + +### 15.2 LLM 설정 + +| 변수 | 기본값 | 설명 | +|---|---|---| +| `LLM_PROVIDER` | `openai` | `openai`, `ollama` | +| `LLM_MODEL_NAME` | `gpt-4o-mini` | 모델명 | +| `LLM_TEMPERATURE` | `0.0` | temperature | +| `LLM_BASE_URL` | `None` | Ollama 등 custom endpoint | +| `LLM_API_KEY` | `None` | provider API key | + +### 15.3 서버 설정 + +| 변수/필드 | 기본값 | 설명 | +|---|---|---| +| `PORT` | `8999` | API server port | +| `base_recursion_limit` | `1000` | LangGraph recursion limit base | +| `estimated_chunks` | `30` | chunk 수 추정 | +| `max_visits_per_node` | `1` | renderer/critic 재시도 횟수 | +| `render_mode` | `ontology_and_facts` | 처리 모드 | +| `ontology_max_triples` | `50000` | ontology graph 최대 triple | +| `parallel_workers` | `4` | content unit 병렬 worker | +| `enable_ontology_consolidation` | `false` | 후처리 consolidation | + +### 15.4 Triple Store 설정 + +Fuseki: + +| 변수 | 설명 | +|---|---| +| `FUSEKI_URI` | Fuseki endpoint | +| `FUSEKI_AUTH` | 인증 정보 | +| `FUSEKI_DATASET` | facts dataset | +| `FUSEKI_ONTOLOGIES_DATASET` | ontology dataset | + +Neo4j: + +| 변수 | 설명 | +|---|---| +| `NEO4J_URI` | Neo4j URI | +| `NEO4J_AUTH` | 인증 정보 | +| `NEO4J_PORT` | HTTP port | +| `NEO4J_BOLT_PORT` | Bolt port | + +Filesystem: + +| 변수 | 설명 | +|---|---| +| `ONTOCAST_WORKING_DIRECTORY` | graph 저장/작업 디렉터리 | +| `ONTOCAST_ONTOLOGY_DIRECTORY` | ontology 파일 디렉터리 | + +### 15.5 Aggregation 설정 + +| 변수 | 기본값 | 설명 | +|---|---|---| +| `AGG_EMBEDDING_MODEL` | `paraphrase-multilingual-MiniLM-L12-v2` | entity embedding 모델 | +| `AGG_SIMILARITY_THRESHOLD` | `0.80` | clustering threshold | + +--- + +## 16. 데이터 모델 명세 + +### 16.1 ContentUnit + +구현 위치: `onto/content_unit.py` + +역할: 문서 chunk 또는 ontology/facts unit 표현 + +핵심 필드: + +- `text` +- `index` +- `doc_iri` +- `graph` +- `type` +- `iri` + +### 16.2 RDFGraph + +구현 위치: `onto/rdfgraph.py` + +역할: rdflib Graph 확장 + +주요 기능: + +- Turtle parse/serialize +- prefix/namespace sanitize +- known prefix patching +- `+=` 연산 지원 테스트 존재 + +### 16.3 Ontology + +구현 위치: `onto/ontology.py` + +역할: RDFGraph + ontology metadata + version lineage + +### 16.4 BudgetTracker + +구현 위치: `onto/state.py` + +역할: + +- LLM call 수 +- 송수신 문자 수 +- ontology/facts triple 생성 수 +- ontology/facts operation 수 + +### 16.5 Suggestions / TripleFix + +구현 위치: `onto/model.py` + +역할: + +- critic output을 renderer 재시도 prompt에 연결 +- source text evidence 기반 개선 지시 저장 + +--- + +## 17. 테스트 분석 + +테스트 폴더 기준 주요 검증 영역: + +| 테스트 | 검증 대상 | +|---|---| +| `test_pipeline.py` | 전체 pipeline 흐름 | +| `test_agent_facts.py` | facts agent | +| `test_ontology_manager.py` | ontology manager | +| `test_ontology_lineage_refresh.py` | ontology lineage/hash | +| `test_graph_update.py` | GraphUpdate 적용 | +| `test_merge_ontologies.py` | ontology merge | +| `test_semantic_chunker.py` | semantic chunking | +| `test_rdfgraph_iadd.py` | RDFGraph 연산 | +| `aggregation/test_*.py` | entity clustering, normalizer, rewriter, provenance, URI builder | + +품질 판단: + +- 핵심 단위 테스트는 존재한다. +- 플랫폼에 가져올 때는 API contract test, storage integration test, 대용량 batch test, Korean document extraction test를 추가해야 한다. + +--- + +## 18. 범용 온톨로지 구축 플랫폼에 가져올 기능 + +### 18.1 1순위: 거의 원형 유지 + +| 기능 | 가져올 모듈 | 이유 | +|---|---|---| +| RDF graph/ontology 모델 | `onto/rdfgraph.py`, `onto/ontology.py` | 플랫폼 핵심 domain model | +| AgentState/UnitState | `onto/state.py`, `onto/unit_states.py` | workflow state 표준화 | +| GraphUpdate/SPARQL 모델 | `onto/sparql_models.py` | LLM 기반 증분 갱신의 핵심 | +| LangGraph workflow | `stategraph/` | 문서 처리 pipeline 기본 골격 | +| ontology/facts renderer/critic | `agent/render_*`, `agent/criticise_*` | LLM agent 핵심 기능 | +| ToolBox | `toolbox.py` | dependency wiring | +| triple manager interface | `tool/triple_manager/` | storage abstraction | +| aggregation | `tool/agg/` | entity disambiguation | +| cache/budget | `tool/cache.py`, `tool/llm.py`, `BudgetTracker` | 운영 비용 관리 | + +### 18.2 2순위: 수정 후 도입 + +| 기능 | 수정 필요 | +|---|---| +| `select_ontology.py` | None 선택 index 버그 가능성 수정 | +| `convert_document.py` | 다중 파일/corpus 처리 모델로 확장 | +| API 서버 | 인증, 작업 ID, 비동기 job queue, progress endpoint 추가 | +| Config | 프로젝트별 저장 설정, secret vault, UI 설정과 통합 | +| External evidence | source trust policy, 검색 provenance 저장 | +| Versioning | approval workflow, diff UI, rollback 기능 연결 | + +### 18.3 3순위: 참고만 할 기능 + +| 기능 | 이유 | +|---|---| +| Robyn 서버 구조 | FastAPI 기반 기존 플랫폼이면 API layer는 재작성 가능 | +| CLI 일부 | 플랫폼 관리 CLI로 재설계 필요 | +| docs auto-generation | 당장 핵심 기능은 아님 | +| Docker 예시 | 운영 환경에 맞춰 재구성 필요 | + +--- + +## 19. 플랫폼 확장 설계 제안 + +### 19.1 목표 플랫폼 모듈 + +범용 온톨로지 구축 플랫폼은 OntoCast 코어 위에 다음 계층을 추가하는 형태가 적합하다. + +```mermaid +flowchart TB + UI[Ontology Studio UI] --> API[Platform API] + API --> JOB[Job Queue / Worker] + API --> PROJECT[Project & Dataset Manager] + JOB --> ONTOCAST[OntoCast Core Engine] + ONTOCAST --> RDF[RDF Store / Fuseki] + ONTOCAST --> NEO[Neo4j Projection] + ONTOCAST --> FS[Artifact Storage] + API --> REVIEW[Human Review Workflow] + REVIEW --> RDF +``` + +### 19.2 추가해야 할 상위 기능 + +| 영역 | 필요 기능 | +|---|---| +| 프로젝트 관리 | ontology project, dataset, namespace, domain policy | +| 문서 수집 | 파일 업로드, 웹 크롤링, URL ingestion, batch import | +| 작업 관리 | job 생성, 상태 조회, 취소, 재시도, 로그 | +| 온톨로지 스튜디오 | class/property/entity 편집, graph diff, 승인 | +| 품질 검수 | critic report UI, human feedback, source evidence 확인 | +| 버전 관리 | semantic version, hash lineage, rollback, release | +| 저장소 | Fuseki canonical RDF, Neo4j projection, artifact store | +| 검색/RAG | SPARQL query, graph search, GraphRAG endpoint | +| 운영 | 비용 추적, LLM call audit, cache 관리 | + +--- + +## 20. 정확한 기능명세 + +### 20.1 문서 처리 기능 + +| ID | 기능명 | 설명 | 입력 | 출력 | +|---|---|---|---|---| +| DOC-001 | 문서 업로드 처리 | API로 받은 JSON/multipart 파일을 처리 대상으로 등록 | file 또는 JSON body | AgentState.files | +| DOC-002 | 문서 변환 | PDF/지원 파일을 markdown/text로 변환 | bytes | text | +| DOC-003 | JSON 문서 파싱 | JSON의 `text`, `url`, instruction 필드를 추출 | JSON bytes | input_text, source_url, instructions | +| DOC-004 | 텍스트 청킹 | 입력 텍스트를 content unit 목록으로 분할 | input_text | list[ContentUnit] | +| DOC-005 | 청크 수 제한 | head chunks만 처리 | max_chunks | 제한된 content_units | + +### 20.2 온톨로지 기능 + +| ID | 기능명 | 설명 | 입력 | 출력 | +|---|---|---|---|---| +| ONT-001 | 온톨로지 목록 로드 | filesystem/triple store에서 ontology 목록 로드 | ontology directory/store | OntologyManager.ontologies | +| ONT-002 | 온톨로지 속성 보강 | title/id/description 누락 시 LLM으로 요약 | RDF graph | OntologyProperties | +| ONT-003 | 온톨로지 선택 | 문서 excerpt와 ontology 목록을 보고 적합 ontology 선택 | content_units, ontologies | current_ontology | +| ONT-004 | 신규 온톨로지 생성 | 기존 ontology가 없으면 fresh ontology 생성 | text chunk | Ontology | +| ONT-005 | 온톨로지 증분 갱신 | 기존 ontology에 필요한 class/property 변경 생성 | ontology graph, text | GraphUpdate | +| ONT-006 | 온톨로지 비평 | ontology 품질 평가 및 개선안 생성 | ontology, text | OntologyCritiqueReport | +| ONT-007 | 온톨로지 재시도 | critic suggestions를 renderer에 반영해 재생성 | Suggestions | updated ontology/update | +| ONT-008 | 온톨로지 delta 병합 | unit별 ontology delta를 하나의 graph로 병합 | ontology_units | normalized ontology | +| ONT-009 | provenance 분리 | reification/provenance triple을 ontology graph에서 side graph로 분리 | RDFGraph | clean graph, provenance graph | +| ONT-010 | 온톨로지 consolidation | 중복/겹침 class/property를 정리 | ontology, excerpt | consolidated ontology | +| ONT-011 | 온톨로지 versioning | 변경 graph에서 새 version/hash 생성 | old ontology, new graph | updated ontology | +| ONT-012 | 온톨로지 저장 | ontology를 filesystem/Fuseki/Neo4j에 저장 | Ontology | persisted graph | + +### 20.3 Facts 추출 기능 + +| ID | 기능명 | 설명 | 입력 | 출력 | +|---|---|---|---|---| +| FACT-001 | Fresh facts 생성 | 빈 content unit graph에 facts Turtle 생성 | ontology, text | RDFGraph | +| FACT-002 | Facts 증분 갱신 | 기존 facts graph에 SPARQL update 생성 | facts graph, ontology, text | GraphUpdate | +| FACT-003 | Facts 비평 | facts graph가 원문을 잘 반영하는지 평가 | facts graph, ontology, text | FactsCritiqueReport | +| FACT-004 | Facts 재시도 | critic suggestions 반영 후 renderer 재실행 | Suggestions | revised facts graph | +| FACT-005 | 병렬 facts 처리 | content unit별 facts loop 병렬 실행 | content_units | parallel_facts_units | +| FACT-006 | Facts 병합 | unit graph를 entity disambiguation 후 통합 | parallel_facts_units | aggregated_facts | +| FACT-007 | Facts 저장 | aggregated facts를 store에 저장 | RDFGraph | persisted facts graph | + +### 20.4 Entity Aggregation 기능 + +| ID | 기능명 | 설명 | +|---|---|---| +| AGG-001 | entity 수집 | graph 내 URIRef entity 수집 | +| AGG-002 | entity representation 생성 | label, type, graph context를 이용해 정규화 표현 생성 | +| AGG-003 | embedding 생성 | sentence-transformers로 entity vector 생성 | +| AGG-004 | candidate clustering | similarity threshold 기반 cluster 후보 생성 | +| AGG-005 | symbolic validation | role/type/lexical alias 기준으로 잘못된 merge 방지 | +| AGG-006 | canonical 선택 | ontology entity 우선, 단순 URI 우선 등 기준으로 대표 선택 | +| AGG-007 | URI 정규화 | role별 PascalCase/camelCase/instance URI 생성 | +| AGG-008 | graph rewrite | old entity URI를 canonical URI로 치환 | +| AGG-009 | sameAs 생성 | 병합 관계를 `owl:sameAs`로 보존 | + +### 20.5 저장소 기능 + +| ID | 기능명 | 설명 | +|---|---|---| +| STORE-001 | ontology fetch | 저장소에서 ontology 목록 조회 | +| STORE-002 | graph serialize | RDFGraph 저장 | +| STORE-003 | ontology serialize | Ontology 저장 | +| STORE-004 | dataset flush | 저장소 데이터 삭제 | +| STORE-005 | Fuseki dataset switching | 요청별 dataset 변경 | +| STORE-006 | filesystem sync | filesystem ontology를 triple store로 동기화 | + +### 20.6 API 기능 + +| ID | 기능명 | Endpoint | +|---|---|---| +| API-001 | health check | `GET /health` | +| API-002 | service info | `GET /info` | +| API-003 | document process | `POST /process` | +| API-004 | triple store flush | `POST /flush` | + +### 20.7 운영/관측 기능 + +| ID | 기능명 | 설명 | +|---|---|---| +| OBS-001 | LLM usage tracking | call 수, 송수신 문자 수 추적 | +| OBS-002 | triple generation tracking | ontology/facts triple 수 추적 | +| OBS-003 | operation tracking | GraphUpdate operation 수 추적 | +| OBS-004 | failure stage tracking | 실패 단계와 원인 저장 | +| OBS-005 | LLM cache | 동일 prompt/config 응답 캐싱 | + +--- + +## 21. 발견된 리스크 및 보완점 + +### 21.1 코드상 주의점 + +1. `select_ontology.py`의 None 선택 index 불일치 가능성 + - dynamic model은 `1..num_ontologies+1`을 허용하지만 코드에는 `answer_index == 0` 처리 분기가 있다. + - 도입 전 수정 필요. + +2. API version 표기 불일치 + - `pyproject.toml` 버전은 `0.3.0`인데 `/health`, `/info`는 `0.1.1`을 반환한다. + - 플랫폼에서는 package version을 단일 source of truth로 연결해야 한다. + +3. `convert_document()`의 다중 파일 처리 한계 + - 내부 주석상 one file 처리. + - 다중 문서 corpus ingestion에는 부적합. + +4. 서버 API가 동기적인 긴 처리에 가까움 + - `/process`가 workflow 완료 후 응답한다. + - 대용량 문서/다중 파일에서는 job queue와 progress endpoint가 필요하다. + +5. 인증/권한 없음 + - 오픈소스 코어 서버이므로 플랫폼 운영에는 인증, tenant, 프로젝트 권한이 필요하다. + +6. `/flush` 위험성 + - 인증 없이 연결하면 전체 triple store 삭제 가능. + - 운영 API에서는 관리자 권한과 confirmation token이 필요하다. + +### 21.2 기능적 한계 + +- ontology 편집 UI 없음 +- human-in-the-loop 승인 흐름 없음 +- ontology schema constraint/SHACL 검증은 핵심 흐름에 보이지 않음 +- 작업 이력/감사 로그 부족 +- Korean domain 문서에 대한 prompt/평가 최적화는 별도 필요 +- 다국어 embedding 모델은 기본값이 multilingual이지만 ontology term naming policy는 영어 중심으로 보임 + +--- + +## 22. 도입 로드맵 제안 + +### Phase 1. 코어 이식 + +- `onto/`, `agent/`, `stategraph/`, `tool/` 핵심 모듈을 별도 package로 이식 +- Apache 2.0 license notice 유지 +- `select_ontology` index bug 수정 +- package version/API version 정리 +- Korean prompt profile 추가 + +### Phase 2. 플랫폼 API 래핑 + +- 기존 Robyn API를 직접 쓰기보다 현재 프로젝트 API framework에 service layer로 연결 +- `/jobs` 기반 비동기 실행 구조 도입 +- 작업 상태, 로그, budget, output artifact 저장 + +### Phase 3. 저장소 전략 확정 + +- Fuseki를 canonical RDF store로 사용 +- Neo4j는 projection/search/RAG용 선택 저장소로 사용 +- filesystem은 artifact/debug output으로 사용 + +### Phase 4. 검수/편집 UI + +- ontology graph viewer +- class/property/entity editor +- GraphUpdate diff viewer +- critic suggestion accept/reject +- source text evidence 연결 + +### Phase 5. 운영 고도화 + +- project/tenant/permission +- cache 관리 +- LLM cost dashboard +- batch ingestion +- rollback/release workflow +- SHACL/OWL reasoning validation + +--- + +## 23. 권장 아키텍처 명세 + +범용 온톨로지 구축 플랫폼에서 OntoCast를 다음처럼 배치한다. + +| 계층 | 구현 | +|---|---| +| Core Engine | OntoCast의 `agent`, `stategraph`, `onto`, `tool` | +| Platform Service | 작업 생성/조회/검수/승인 API | +| Storage | Fuseki canonical RDF, Neo4j projection, artifact filesystem/S3 | +| UI | ontology studio, extraction review, graph diff | +| Worker | OntoCast workflow 실행, batch 처리 | +| Governance | versioning, approval, lineage, audit | + +Core Engine의 public interface는 다음 정도로 단순화하는 것이 좋다. + +```python +class OntologyBuildService: + async def process_document( + self, + project_id: str, + dataset_id: str, + text: str, + source_url: str | None, + render_mode: str, + ontology_instruction: str, + facts_instruction: str, + ) -> OntologyBuildResult: + ... +``` + +반환 모델: + +```python +class OntologyBuildResult: + status: str + ontology_ttl: str + facts_ttl: str + ontology_version: str | None + ontology_hash: str | None + graph_uri: str + chunks_processed: int + budget: dict + warnings: list[str] + critique_summary: str | None +``` + +--- + +## 24. 최종 평가 + +OntoCast는 “문서 기반 온톨로지/지식그래프 자동 구축 엔진”으로 매우 직접적인 참고 가치가 있다. 특히 GraphUpdate 기반 증분 갱신, renderer/critic retry loop, content unit 병렬 처리, entity aggregation은 앞으로 만들 범용 온톨로지 구축 플랫폼의 핵심 엔진으로 적합하다. + +가져갈 때의 전략은 “API 서버까지 그대로 제품화”가 아니라 “core package를 거의 원형 유지하면서 플랫폼 서비스 계층으로 감싸기”가 가장 좋다. 이렇게 하면 오픈소스의 검증된 처리 흐름은 살리고, 우리 프로젝트에 필요한 프로젝트 관리, 검수 UI, 저장소 정책, 권한, 배치 운영은 별도로 안정적으로 얹을 수 있다. diff --git a/오픈소스분석자료/OpenDeepResearcher_분석_및_기능명세.md b/오픈소스분석자료/OpenDeepResearcher_분석_및_기능명세.md new file mode 100644 index 0000000..65a3e75 --- /dev/null +++ b/오픈소스분석자료/OpenDeepResearcher_분석_및_기능명세.md @@ -0,0 +1,1327 @@ +# OpenDeepResearcher 분석 및 기능명세 + +분석 대상: `C:\Users\lasta\MyProject\AI\참고\OpenDeepResearcher-main` + +작성 목적: 오픈 프로젝트 `OpenDeepResearcher-main`을 범용 온톨로지 구축 플랫폼의 기본 소스로 재사용하기 위해, 구조와 기능을 상세히 분석하고 현재 프로젝트에 이식 가능한 기능 명세를 정의한다. + +## 1. 프로젝트 개요 + +`OpenDeepResearcher`는 사용자의 연구 질문을 입력받아 LLM이 검색어를 만들고, 검색 API로 웹 문서를 찾고, 각 문서의 유용성을 LLM으로 평가한 뒤, 필요한 정보만 추출하고, 추가 검색 필요 여부를 다시 판단하는 반복형 딥리서치 노트북이다. + +저장소는 라이브러리/패키지 형태가 아니라 Jupyter Notebook 중심의 예제 프로젝트다. 핵심 코드는 두 노트북 안에 거의 동일하게 포함되어 있다. + +| 파일 | 역할 | +|---|---| +| `README.md` | 프로젝트 개요, 요구 API, 사용법 설명 | +| `open_deep_researcher.ipynb` | CLI/input 기반 딥리서치 루프 원본 | +| `open_deep_researcher_gradio.ipynb` | Gradio UI가 붙은 변형 | +| `LICENSE` | MIT License | + +이 프로젝트의 본질은 완성된 제품이라기보다 “검색 기반 연구 루프의 최소 구현”이다. 따라서 현재 범용 온톨로지 구축 플랫폼에서는 코드를 그대로 복사하기보다, 루프 구조와 프롬프트 역할, 비동기 처리 방식, 링크 중복 제거 로직, 검색 확장 판단 로직을 거의 원형에 가깝게 모듈화해 흡수하는 것이 적합하다. + +## 2. 라이선스 및 재사용 조건 + +라이선스는 MIT License다. + +재사용 가능 범위: + +- 소스 복사, 수정, 병합, 배포, 상업적 사용 가능 +- 단, 저작권 고지와 MIT 라이선스 문구를 소프트웨어의 주요 복사본 또는 실질적 일부에 포함해야 함 +- “AS IS” 조건이므로 품질, 정확성, 특정 목적 적합성에 대한 보증은 없음 + +내 프로젝트에서 기본 소스로 사용할 경우 권장 조치: + +- `THIRD_PARTY_NOTICES.md` 또는 `NOTICE`에 `OpenDeepResearcher`, 원 저작권자 `mshumer`, MIT License를 명시 +- 원본에서 차용한 모듈 파일 상단에 간단한 출처 주석 추가 +- API 키, 모델명, 엔드포인트는 하드코딩하지 않고 기존 설정 체계로 이동 + +## 3. 기술 스택 + +| 영역 | 사용 기술 | 설명 | +|---|---|---| +| 실행 형태 | Jupyter Notebook, Google Colab 가정 | 독립 패키지가 아니라 노트북 셀 실행 방식 | +| 비동기 처리 | `asyncio`, `aiohttp`, `nest_asyncio` | 검색, 페이지 fetch, LLM 평가/추출을 병렬 처리 | +| 검색 | SERPAPI Google Search | 질의별 검색 결과 URL 수집 | +| 페이지 텍스트화 | Jina Reader API `https://r.jina.ai/` | URL을 텍스트로 변환해 LLM 입력으로 사용 | +| LLM 호출 | OpenRouter Chat Completions | 검색어 생성, 유용성 평가, 컨텍스트 추출, 추가 검색 판단, 최종 보고서 생성 | +| UI | Gradio | Gradio 노트북에서 입력/출력 화면 제공 | + +필수 API 키: + +| 환경 값 | 원본 변수명 | 용도 | +|---|---|---| +| OpenRouter API Key | `OPENROUTER_API_KEY` | LLM 호출 | +| SERPAPI API Key | `SERPAPI_API_KEY` | Google 검색 | +| Jina API Key | `JINA_API_KEY` | 웹페이지 텍스트 추출 | + +기본 모델: + +```text +anthropic/claude-3.5-haiku +``` + +현재 프로젝트 이식 시에는 OpenRouter에 고정하지 않고 기존 `crawler_platform.app.core.extractor.ai_provider`의 OpenAI-compatible 구조와 맞춰 `provider`, `base_url`, `model`, `api_key_env`로 일반화하는 것이 좋다. + +## 4. 전체 아키텍처 + +원본의 전체 흐름은 다음과 같다. + +```mermaid +flowchart TD + A["사용자 연구 질문"] --> B["LLM: 초기 검색어 생성"] + B --> C["SERPAPI: 검색어별 Google 검색"] + C --> D["검색 결과 URL 통합 및 중복 제거"] + D --> E["Jina: URL별 웹페이지 텍스트 추출"] + E --> F["LLM: 페이지 유용성 Yes/No 평가"] + F --> G{"유용한가?"} + G -- "Yes" --> H["LLM: 관련 컨텍스트 추출"] + G -- "No" --> I["폐기"] + H --> J["컨텍스트 누적"] + J --> K["LLM: 추가 검색 필요 여부 판단"] + K -- "검색어 리스트" --> C + K -- "" --> L["LLM: 최종 보고서 생성"] +``` + +구성 요소를 책임 기준으로 나누면 다음과 같다. + +| 구성 요소 | 원본 함수 | 책임 | +|---|---|---| +| LLM 클라이언트 | `call_openrouter_async` | Chat Completions API 호출 | +| 검색어 생성기 | `generate_search_queries_async` | 사용자 질문을 검색 질의 목록으로 변환 | +| 검색 클라이언트 | `perform_search_async` | 검색어를 URL 리스트로 변환 | +| 웹 텍스트 fetcher | `fetch_webpage_text_async` | URL 본문을 텍스트로 변환 | +| 관련성 평가기 | `is_page_useful_async` | 페이지가 질문에 유용한지 판정 | +| 컨텍스트 추출기 | `extract_relevant_context_async` | 페이지에서 질문 관련 정보만 추출 | +| 반복 계획기 | `get_new_search_queries_async` | 누적 컨텍스트를 보고 다음 검색어 또는 종료 판단 | +| 보고서 생성기 | `generate_final_report_async` | 누적 컨텍스트 기반 최종 답변 작성 | +| 링크 처리 파이프라인 | `process_link` | fetch → 평가 → 추출을 URL 단위로 수행 | +| 메인 루프 | `async_main`, `async_research` | 반복 실행, 상태 누적, 종료 제어 | + +## 5. 노트북별 상세 분석 + +### 5.1 `open_deep_researcher.ipynb` + +CLI형 또는 콘솔 입력형 구현이다. + +주요 특징: + +- `input()`으로 사용자 질문과 최대 반복 횟수를 받음 +- 초기 검색어를 LLM으로 생성 +- 반복마다 검색어별 SERPAPI 요청을 동시에 실행 +- 검색 결과 URL을 딕셔너리로 중복 제거 +- URL별 Jina fetch, LLM 유용성 평가, LLM 컨텍스트 추출을 동시에 실행 +- 누적 컨텍스트를 LLM에 제공해 다음 검색어 또는 `` 판단 +- 종료 후 최종 보고서를 생성하고 출력 + +상태 변수: + +| 변수 | 의미 | +|---|---| +| `aggregated_contexts` | 모든 반복에서 추출한 유용 컨텍스트 누적 | +| `all_search_queries` | 지금까지 사용한 모든 검색어 | +| `new_search_queries` | 현재 반복에서 실행할 검색어 | +| `iteration_limit` | 최대 반복 횟수 | +| `unique_links` | 한 반복 안에서 중복 제거된 URL과 URL을 발견한 검색어 매핑 | + +핵심 장점: + +- 구조가 단순하고 이해하기 쉽다. +- 모든 외부 I/O를 비동기로 처리해 속도상 이점이 있다. +- 페이지 fetch, 관련성 판정, 컨텍스트 추출을 기능별로 분리했다. +- 검색이 부족한지 LLM이 판단하는 자기 확장 루프가 있다. + +핵심 한계: + +- `eval(response)`로 LLM 출력을 파싱하므로 보안상 위험하다. +- 검색 결과 URL의 전역 중복 제거가 없다. 반복 간 동일 URL 재처리 가능성이 있다. +- 출처 URL, 제목, 검색어, 평가 결과가 최종 컨텍스트와 함께 구조화 저장되지 않는다. +- 실패 재시도, rate limit, timeout, backoff가 없다. +- 토큰 예산 관리가 단순하다. 페이지 본문은 앞 20,000자만 사용한다. +- 최종 보고서에 인용/근거 링크가 구조적으로 연결되지 않는다. +- 온톨로지 엔티티/관계 추출 기능은 없다. + +### 5.2 `open_deep_researcher_gradio.ipynb` + +Gradio UI를 붙인 구현이다. 연구 루프 자체는 원본과 거의 동일하다. + +추가된 함수: + +| 함수 | 역할 | +|---|---| +| `async_research(user_query, iteration_limit)` | 콘솔 입력 없이 연구 루프 실행 후 결과와 로그 반환 | +| `run_research(user_query, iteration_limit=10)` | `asyncio.run`으로 비동기 루프 실행 | +| `gradio_run(user_query, iteration_limit)` | Gradio 이벤트 핸들러, 예외 처리 | + +UI 구성: + +| Gradio 컴포넌트 | 용도 | +|---|---| +| `Textbox(lines=2)` | 연구 질문 입력 | +| `Number(value=10)` | 최대 반복 횟수 입력 | +| `Textbox(label="Final Report")` | 최종 보고서 출력 | +| `Textbox(label="Intermediate Steps Log")` | 실행 로그 출력 | + +현재 프로젝트에는 이미 FastAPI와 프론트엔드가 있으므로 Gradio 코드는 직접 이식 대상은 아니다. 다만 `async_research`처럼 UI에서 호출 가능한 순수 함수형 API로 연구 루프를 분리한 점은 참고할 가치가 있다. + +## 6. 함수별 기능 명세 + +### 6.1 `call_openrouter_async` + +목적: OpenRouter Chat Completions API를 비동기로 호출한다. + +입력: + +| 파라미터 | 타입 | 설명 | +|---|---|---| +| `session` | `aiohttp.ClientSession` | 공유 HTTP 세션 | +| `messages` | `list[dict]` | Chat Completions 메시지 | +| `model` | `str` | 사용할 모델명, 기본값 `DEFAULT_MODEL` | + +처리: + +1. `Authorization: Bearer {OPENROUTER_API_KEY}` 헤더 구성 +2. `OPENROUTER_URL`로 POST 요청 +3. 응답이 200이면 `choices[0].message.content` 반환 +4. 구조 오류 또는 HTTP 오류이면 `None` 반환 + +출력: + +| 성공 | 실패 | +|---|---| +| assistant 메시지 문자열 | `None` | + +이식 시 개선 명세: + +- OpenRouter 전용 함수가 아니라 `AsyncLLMClient.complete(messages, model, response_schema=None)` 형태로 추상화 +- timeout, retry, backoff, rate limit 처리 +- 오류를 `print`하지 않고 구조화 로그와 DB 실행 기록에 저장 +- 비용/토큰 사용량 저장 +- JSON 모드 또는 스키마 응답 옵션 지원 + +### 6.2 `generate_search_queries_async` + +목적: 사용자 질문에서 최대 4개의 검색어를 생성한다. + +입력: + +| 파라미터 | 타입 | 설명 | +|---|---|---| +| `session` | `aiohttp.ClientSession` | HTTP 세션 | +| `user_query` | `str` | 원본 연구 질문 | + +원본 프롬프트 요구: + +- 전문 연구 보조자 역할 +- 최대 4개 distinct, precise search query +- Python 문자열 리스트만 출력 + +출력: + +```python +["query1", "query2", "query3"] +``` + +원본 한계: + +- `eval(response)` 사용 +- 검색어 품질 기준이 약함 +- 검색어 언어, 도메인, 시간 범위, 출처 유형 제약이 없음 + +온톨로지 플랫폼용 개선 명세: + +검색어 객체를 문자열이 아닌 구조로 반환해야 한다. + +```json +{ + "queries": [ + { + "query": "perfume note taxonomy ontology extraction", + "purpose": "Find ontology classes and relation candidates", + "target_entity_types": ["Perfume", "Note", "Accord"], + "expected_source_type": "reference" + } + ] +} +``` + +필수 검증: + +- `queries`는 1개 이상 4개 이하 +- 중복 검색어 제거 +- 빈 문자열 제거 +- 이전 검색어와 의미적으로 거의 동일한 검색어 제거 + +### 6.3 `perform_search_async` + +목적: 검색어 하나를 SERPAPI Google 검색에 보내 URL 리스트를 얻는다. + +입력: + +| 파라미터 | 타입 | 설명 | +|---|---|---| +| `session` | `aiohttp.ClientSession` | HTTP 세션 | +| `query` | `str` | 검색어 | + +요청 파라미터: + +| 키 | 값 | +|---|---| +| `q` | 검색어 | +| `api_key` | `SERPAPI_API_KEY` | +| `engine` | `google` | + +출력: + +- `organic_results[*].link`만 추출한 URL 리스트 +- 실패 시 빈 리스트 + +이식 시 개선 명세: + +- `SearchProvider` 인터페이스 도입 +- SERPAPI, Tavily, Bing, Google CSE, 로컬 색인 등을 교체 가능하게 구성 +- 검색 결과에 URL만 남기지 말고 제목, snippet, rank, source, query를 함께 저장 +- 도메인 allow/block list 지원 +- PDF, HTML, GitHub, 논문, 문서 등 source type 태깅 + +권장 데이터 모델: + +```json +{ + "url": "https://example.com/page", + "title": "Page title", + "snippet": "Search result snippet", + "rank": 1, + "query": "original search query", + "provider": "serpapi", + "retrieved_at": "ISO-8601" +} +``` + +### 6.4 `fetch_webpage_text_async` + +목적: Jina Reader API로 URL의 웹페이지 텍스트를 가져온다. + +입력: + +| 파라미터 | 타입 | 설명 | +|---|---|---| +| `session` | `aiohttp.ClientSession` | HTTP 세션 | +| `url` | `str` | 원본 URL | + +처리: + +- `full_url = f"{JINA_BASE_URL}{url}"` +- Jina API에 GET 요청 +- 성공 시 텍스트 반환 + +출력: + +| 성공 | 실패 | +|---|---| +| 페이지 텍스트 | 빈 문자열 | + +현재 프로젝트와의 관계: + +현재 프로젝트에는 이미 다음 기능이 있다. + +- `crawler_platform.app.core.crawler.fetchers.make_fetcher` +- `crawler_platform.app.core.crawler.plugins.ParserRegistry` +- `SiteCrawler` +- `GraphResearchLoop` + +따라서 Jina fetcher를 반드시 그대로 쓸 필요는 없다. 다만 외부 웹 텍스트 추출 대체 경로로 `JinaTextFetcher` 어댑터를 추가하면 좋다. + +이식 시 개선 명세: + +- 기존 fetcher/parser와 동일한 결과 객체로 변환 +- URL, final_url, title, raw_text, clean_text, markdown, status_code, warnings 포함 +- robots.txt 정책을 기존 `RobotsPolicy`와 통합 +- 실패 시 fallback fetcher 사용 가능 + +### 6.5 `is_page_useful_async` + +목적: 웹페이지 본문이 사용자 질문에 유용한지 LLM으로 이진 판정한다. + +입력: + +| 파라미터 | 타입 | 설명 | +|---|---|---| +| `session` | `aiohttp.ClientSession` | HTTP 세션 | +| `user_query` | `str` | 원본 질문 | +| `page_text` | `str` | 페이지 본문 | + +원본 프롬프트: + +- critical research evaluator +- 질문과 페이지 내용을 보고 유용성 판단 +- 정확히 `Yes` 또는 `No`만 출력 +- 본문은 앞 20,000자만 사용 + +출력: + +```text +Yes +No +``` + +한계: + +- 이유, 점수, 불확실성이 없다. +- 현재 온톨로지에서 어떤 gap을 채우는지 판단하지 않는다. +- 페이지 품질, 신뢰도, 중복성, 출처 유형을 반영하지 않는다. + +온톨로지 플랫폼용 개선 명세: + +```json +{ + "useful": true, + "score": 0.82, + "reason": "Contains explicit product-note relationships relevant to target predicates.", + "matched_entity_types": ["Perfume", "Note"], + "matched_predicates": ["hasTopNote", "hasBaseNote"], + "fills_gaps": ["missing note relationships for product pages"], + "source_quality": "primary|secondary|low", + "recommended_action": "extract|crawl_links|skip" +} +``` + +이 기능은 현재 프로젝트의 `RelevanceEngine`과 결합하는 것이 좋다. 원본의 Yes/No 판정은 LLM 기반 의미 판정으로 유지하되, 기존 점수 기반 링크 우선순위와 함께 사용한다. + +### 6.6 `extract_relevant_context_async` + +목적: 유용하다고 판단된 페이지에서 질문 답변에 필요한 관련 컨텍스트만 추출한다. + +입력: + +| 파라미터 | 타입 | 설명 | +|---|---|---| +| `session` | `aiohttp.ClientSession` | HTTP 세션 | +| `user_query` | `str` | 원본 질문 | +| `search_query` | `str` | 해당 페이지를 발견한 검색어 | +| `page_text` | `str` | 페이지 본문 | + +원본 출력: + +- 일반 plain text +- 별도 구조 없음 + +온톨로지 플랫폼용 개선 명세: + +컨텍스트 추출 결과는 반드시 출처와 증거 범위를 포함해야 한다. + +```json +{ + "source": { + "url": "https://example.com/item", + "search_query": "query used", + "title": "Page title" + }, + "contexts": [ + { + "text": "short extracted evidence", + "summary": "what this evidence supports", + "entity_candidates": [ + {"name": "Bergamot", "type": "Note"} + ], + "relation_candidates": [ + { + "subject": "Product A", + "predicate": "hasTopNote", + "object": "Bergamot" + } + ], + "confidence": 0.78 + } + ] +} +``` + +현재 프로젝트에서는 이 결과를 다음 두 경로 중 하나로 연결할 수 있다. + +1. `ExtractionPageContext`로 변환해 기존 `Extractor.extract_from_context`에 전달 +2. `ExtractedEntity`, `ExtractedClaim`, `ExtractionBundle`로 직접 변환 + +권장 방향은 1번이다. 원본의 컨텍스트 추출기는 “정보 압축기”로 두고, 실제 온톨로지 엔티티/클레임 추출은 기존 `LLMJsonExtractor`와 validation 계층을 쓰는 편이 일관성이 높다. + +### 6.7 `get_new_search_queries_async` + +목적: 지금까지 수행한 검색어와 누적 컨텍스트를 보고 추가 검색이 필요한지 판단한다. + +입력: + +| 파라미터 | 타입 | 설명 | +|---|---|---| +| `session` | `aiohttp.ClientSession` | HTTP 세션 | +| `user_query` | `str` | 원본 질문 | +| `previous_search_queries` | `list[str]` | 이전 검색어 | +| `all_contexts` | `list[str]` | 누적 컨텍스트 | + +출력: + +| 상황 | 출력 | +|---|---| +| 추가 검색 필요 | Python list 형식의 검색어 목록 | +| 충분함 | `` | +| 실패 | 빈 리스트 | + +중요성: + +이 함수가 OpenDeepResearcher의 핵심이다. 단순 검색 파이프라인을 “자기 확장형 연구 루프”로 만드는 역할을 한다. + +한계: + +- 연구 완료 기준이 모호하다. +- 검색어 중복 방지가 약하다. +- 온톨로지 gap, 충돌, 엔티티 커버리지와 연결되어 있지 않다. +- 누적 컨텍스트가 길어지면 토큰 초과 가능성이 크다. + +온톨로지 플랫폼용 개선 명세: + +```json +{ + "status": "continue|done", + "completion_reason": "sufficient evidence for requested ontology gaps", + "coverage": { + "entity_types": { + "Perfume": 0.8, + "Brand": 0.6, + "Note": 0.9 + }, + "predicates": { + "hasBrand": 0.7, + "hasTopNote": 0.5 + } + }, + "new_queries": [ + { + "query": "site:example.com perfume top notes bergamot", + "purpose": "Fill missing hasTopNote claims", + "priority": 0.86 + } + ], + "stop_conditions_met": [ + "max useful source diversity reached", + "no unresolved high-priority gaps" + ] +} +``` + +완료 판단 기준: + +- 대상 엔티티 타입별 최소 수집량 충족 +- 핵심 predicate별 evidence coverage 충족 +- 최근 반복에서 신규 유용 컨텍스트가 일정 수 이하 +- 동일 URL/도메인 반복 비율 증가 +- LLM이 명시적으로 추가 gap을 찾지 못함 +- 최대 반복/최대 비용/최대 시간 도달 + +### 6.8 `generate_final_report_async` + +목적: 누적 컨텍스트를 기반으로 최종 연구 보고서를 생성한다. + +입력: + +| 파라미터 | 타입 | 설명 | +|---|---|---| +| `session` | `aiohttp.ClientSession` | HTTP 세션 | +| `user_query` | `str` | 원본 질문 | +| `all_contexts` | `list[str]` | 모든 컨텍스트 | + +출력: + +- 구조화된 문자열 보고서 + +온톨로지 플랫폼에서의 위치: + +최종 보고서는 사용자 설명용 산출물이다. 범용 온톨로지 구축 플랫폼의 핵심 산출물은 보고서가 아니라 다음이어야 한다. + +- 후보 엔티티 +- 후보 관계/클레임 +- 증거 텍스트 +- 출처 URL +- 신뢰도 +- 충돌/중복 판정 +- 온톨로지 gap 변화 +- 다음 수집 과제 + +따라서 보고서 생성기는 부가 기능으로 두고, 연구 세션 요약 또는 운영 리포트 생성에 활용한다. + +권장 보고서 구조: + +```text +1. 연구 목표 +2. 수집 범위 +3. 핵심 발견 +4. 온톨로지 후보 +5. 증거 기반 클레임 요약 +6. 남은 지식 공백 +7. 출처 목록 +8. 다음 크롤링/검색 제안 +``` + +### 6.9 `process_link` + +목적: URL 하나에 대해 fetch → 유용성 평가 → 컨텍스트 추출을 수행한다. + +입력: + +| 파라미터 | 설명 | +|---|---| +| `link` | 처리할 URL | +| `user_query` | 원본 질문 | +| `search_query` | URL을 발견한 검색어 | + +처리: + +1. Jina로 페이지 텍스트 fetch +2. 빈 텍스트면 `None` +3. LLM으로 유용성 평가 +4. `Yes`이면 관련 컨텍스트 추출 +5. 컨텍스트가 있으면 반환 + +출력: + +| 상황 | 출력 | +|---|---| +| 유용한 페이지 | 추출 컨텍스트 문자열 | +| 무용/실패 | `None` | + +이식 시 권장 반환 타입: + +```json +{ + "url": "https://example.com/page", + "status": "extracted|skipped|failed", + "search_query": "query", + "usefulness": { + "useful": true, + "score": 0.82, + "reason": "..." + }, + "context_count": 3, + "contexts": [], + "error": null +} +``` + +## 7. 반복 루프 기능 명세 + +### 7.1 입력 + +| 필드 | 타입 | 기본값 | 설명 | +|---|---|---|---| +| `user_query` | `str` | 필수 | 연구 목표 또는 온톨로지 구축 목표 | +| `iteration_limit` | `int` | `10` | 최대 반복 횟수 | +| `max_queries_per_iteration` | `int` | `4` | 반복당 최대 검색어 수 | +| `max_results_per_query` | `int` | 검색 API 기본값 | 검색어당 결과 수 | +| `max_pages_per_iteration` | `int` | 별도 없음 | 원본에는 없음, 추가 필요 | +| `model` | `str` | `anthropic/claude-3.5-haiku` | LLM 모델 | +| `search_provider` | `str` | `serpapi` | 검색 제공자 | +| `fetch_provider` | `str` | `jina` | 텍스트 fetch 제공자 | + +### 7.2 출력 + +원본 출력: + +- 최종 보고서 문자열 +- Gradio 버전은 중간 로그 문자열 추가 + +내 프로젝트용 출력: + +```json +{ + "session_id": 123, + "status": "completed|partial|failed|cancelled", + "query": "original user query", + "iterations": 3, + "search_queries": [], + "visited_urls": [], + "useful_sources": [], + "extracted_contexts": [], + "ontology_candidates": { + "entities": [], + "claims": [] + }, + "coverage": {}, + "remaining_gaps": [], + "final_report": "..." +} +``` + +### 7.3 상태 전이 + +```mermaid +stateDiagram-v2 + [*] --> Initialized + Initialized --> QueryGenerated + QueryGenerated --> Searching + Searching --> LinksDeduplicated + LinksDeduplicated --> FetchingPages + FetchingPages --> EvaluatingPages + EvaluatingPages --> ExtractingContexts + ExtractingContexts --> PlanningNext + PlanningNext --> Searching: new queries + PlanningNext --> Reporting: done + Searching --> PartialFailure: provider error + FetchingPages --> PartialFailure: fetch error + PartialFailure --> PlanningNext + Reporting --> Completed + Completed --> [*] +``` + +### 7.4 종료 조건 + +원본 종료 조건: + +- LLM이 `` 반환 +- LLM이 새 검색어를 반환하지 않음 +- 반복 횟수가 `iteration_limit`에 도달 + +추가해야 할 종료 조건: + +- 유용 컨텍스트 증가량이 낮음 +- 새 URL 발견률이 낮음 +- 비용/토큰 제한 도달 +- 사용자 취소 +- 검색 API quota 소진 +- 동일 도메인 반복 과다 +- 온톨로지 gap coverage 목표 달성 + +## 8. 현재 프로젝트와의 통합 분석 + +현재 프로젝트에는 이미 범용 온톨로지 구축 플랫폼의 뼈대가 상당히 들어 있다. + +관련 기존 모듈: + +| 현재 프로젝트 모듈 | 역할 | OpenDeepResearcher와의 연결 | +|---|---|---| +| `crawler_platform.app.core.research.graph_research_loop.GraphResearchLoop` | 그래프/엔티티 중심 탐색 루프 | 외부 검색 기반 gap 탐색을 추가할 핵심 위치 | +| `ResearchMemoryStore` | 연구 세션, 히스토리, 큐, memory 저장 | OpenDeepResearcher의 로그/컨텍스트 누적을 구조화 저장 | +| `ExplorationQueue` | 탐색 대상 우선순위 큐 | 검색 결과 URL을 queue item으로 변환 가능 | +| `RelevanceEngine` | URL/엔티티 관련성 점수화 | LLM page usefulness 판정과 결합 | +| `GapTaskPlanner` | 지식 공백 기반 작업 생성 | 추가 검색어 생성 입력으로 사용 | +| `SiteCrawler` | 사이트 내부 크롤링 | 검색으로 발견한 seed URL을 사이트 크롤링으로 확장 | +| `LLMJsonExtractor` | 온톨로지 엔티티/클레임 JSON 추출 | 관련 컨텍스트에서 구조화 지식 추출 | +| `KnowledgeRepository` | 프로젝트, source, page, entity, claim 저장 | 연구 결과 영속화 | + +통합 방향: + +1. OpenDeepResearcher를 독립 노트북이 아니라 `WebResearchPlanner` 또는 `ExternalResearchLoop` 모듈로 분리한다. +2. 검색어 생성과 추가 검색 판단은 기존 `GapTaskPlanner`의 출력과 `ResearchMemoryStore.memory`를 입력으로 받는다. +3. 검색 결과 URL은 `ExplorationQueue`에 `target_type="url"`로 넣는다. +4. URL fetch와 파싱은 가능하면 기존 `make_fetcher`/`ParserRegistry`를 사용한다. +5. Jina는 별도 fetch provider로 추가한다. +6. 페이지 유용성 판정은 기존 `RelevanceEngine.score_url` 결과와 LLM 의미 판정을 합산한다. +7. 컨텍스트 추출 후 `ExtractionPageContext`를 만들고 기존 extractor로 엔티티/클레임을 저장한다. +8. 최종 보고서는 연구 세션 요약 API에서 생성한다. + +## 9. 권장 신규 모듈 설계 + +### 9.1 패키지 위치 + +권장 파일 구조: + +```text +crawler_platform/app/core/research/ + external_research_loop.py + search_provider.py + async_llm_client.py + context_extractor.py + research_planner.py + source_usefulness.py +``` + +### 9.2 핵심 클래스 + +#### `AsyncResearchLLMClient` + +책임: + +- OpenAI-compatible 또는 OpenRouter API 비동기 호출 +- JSON 응답 파싱 +- retry/timeout/backoff +- token/cost metadata 수집 + +주요 메서드: + +```python +async def complete(self, messages: list[dict[str, str]], *, model: str | None = None) -> str +async def complete_json(self, messages: list[dict[str, str]], *, schema: dict | None = None) -> dict +``` + +#### `SearchProvider` + +책임: + +- 검색어를 검색 결과 목록으로 변환 + +주요 메서드: + +```python +async def search(self, query: str, *, limit: int = 10) -> list[SearchResult] +``` + +#### `ResearchPlanner` + +책임: + +- 초기 검색어 생성 +- 추가 검색 필요 판단 +- 종료 판단 +- gap 기반 query 생성 + +주요 메서드: + +```python +async def initial_queries(self, goal: str, ontology: dict) -> list[ResearchQuery] +async def next_step(self, state: ResearchState) -> ResearchDecision +``` + +#### `SourceUsefulnessEvaluator` + +책임: + +- 페이지가 연구 목표/온톨로지 gap에 유용한지 판단 +- LLM 판정과 rule score 결합 + +주요 메서드: + +```python +async def evaluate(self, goal: str, page: ParsedPage, gaps: list[dict]) -> UsefulnessDecision +``` + +#### `ResearchContextExtractor` + +책임: + +- 페이지 텍스트에서 연구 관련 컨텍스트 추출 +- 출처와 evidence를 유지 + +주요 메서드: + +```python +async def extract(self, goal: str, page: ParsedPage, query: ResearchQuery) -> list[ResearchContext] +``` + +#### `ExternalResearchLoop` + +책임: + +- OpenDeepResearcher의 전체 반복 루프를 현재 프로젝트 구조로 실행 +- 검색 결과를 저장소와 큐에 반영 +- 연구 세션 히스토리와 memory 업데이트 + +주요 메서드: + +```python +async def run(self, request: ExternalResearchRequest) -> ExternalResearchResult +``` + +## 10. 데이터 모델 명세 + +### 10.1 `ResearchQuery` + +```json +{ + "query": "string", + "purpose": "string", + "priority": 0.0, + "target_entity_types": ["string"], + "target_predicates": ["string"], + "source_type": "web|paper|docs|github|unknown", + "iteration": 0 +} +``` + +필수 필드: + +- `query` +- `purpose` +- `priority` + +검증: + +- `query`는 3자 이상 +- `priority`는 0.0 이상 1.0 이하 +- 동일 세션 내 normalized query 중복 금지 + +### 10.2 `SearchResult` + +```json +{ + "url": "string", + "title": "string|null", + "snippet": "string|null", + "rank": 1, + "query": "string", + "provider": "serpapi", + "metadata": {} +} +``` + +검증: + +- URL scheme은 `http` 또는 `https` +- fragment 제거 +- trailing slash normalization +- 같은 normalized URL은 한 세션에서 한 번만 처리 + +### 10.3 `UsefulnessDecision` + +```json +{ + "useful": true, + "score": 0.0, + "reason": "string", + "recommended_action": "extract|crawl_links|skip", + "matched_entity_types": [], + "matched_predicates": [], + "fills_gaps": [] +} +``` + +검증: + +- `score`는 0.0 이상 1.0 이하 +- `recommended_action`은 enum +- `useful=false`이면 `score < min_relevance` 권장 + +### 10.4 `ResearchContext` + +```json +{ + "url": "string", + "title": "string|null", + "search_query": "string", + "text": "string", + "summary": "string", + "evidence_text": "string", + "entity_candidates": [], + "relation_candidates": [], + "confidence": 0.0 +} +``` + +검증: + +- `text` 또는 `evidence_text`는 비어 있으면 안 됨 +- 긴 본문 복사를 막기 위해 evidence는 짧게 제한 +- relation candidate는 subject/predicate/object 중 최소 subject와 predicate 필요 + +### 10.5 `ResearchDecision` + +```json +{ + "status": "continue|done", + "reason": "string", + "new_queries": [], + "coverage": {}, + "remaining_gaps": [], + "stop_conditions": [] +} +``` + +## 11. API 기능 명세 + +현재 FastAPI에 추가할 수 있는 API 명세다. + +### 11.1 외부 딥리서치 실행 + +```http +POST /research/external/run +``` + +요청: + +```json +{ + "project_name": "perfume", + "source_name": "web", + "goal": "Build ontology candidates for perfume notes and accords", + "iteration_limit": 5, + "max_queries_per_iteration": 4, + "max_results_per_query": 10, + "min_usefulness": 0.45, + "search_provider": "serpapi", + "fetch_provider": "existing|jina", + "llm_provider": "openai_compatible", + "llm_model": "model-name", + "llm_base_url": "http://localhost:1234/v1", + "same_domain_only": false +} +``` + +응답: + +```json +{ + "session_id": 1, + "status": "completed", + "iterations": 3, + "visited_count": 42, + "useful_count": 11, + "context_count": 25, + "entity_count": 18, + "claim_count": 37, + "remaining_gaps": [], + "report": "..." +} +``` + +### 11.2 연구 세션 조회 + +기존 `/research/sessions/{job_id}`에 외부 검색 세션의 상세 항목을 포함한다. + +추가 필드: + +```json +{ + "search_queries": [], + "search_results": [], + "usefulness_decisions": [], + "contexts": [], + "coverage": {}, + "cost": { + "llm_calls": 0, + "search_calls": 0, + "fetch_calls": 0 + } +} +``` + +### 11.3 연구 세션 취소 + +```http +POST /research/sessions/{job_id}/cancel +``` + +필요 이유: + +- OpenDeepResearcher 원본에는 취소 기능이 없다. +- 웹 UI에서 긴 리서치 작업을 실행할 경우 필수다. + +## 12. 프롬프트 명세 + +### 12.1 검색어 생성 프롬프트 + +목표: + +- 온톨로지 구축 목표를 검색 가능한 질의로 분해 +- entity type, predicate, source type을 명시 + +출력은 JSON only: + +```json +{ + "queries": [ + { + "query": "string", + "purpose": "string", + "priority": 0.0, + "target_entity_types": [], + "target_predicates": [], + "source_type": "web" + } + ] +} +``` + +필수 규칙: + +- 최대 4개 +- 서로 다른 의도를 가져야 함 +- 이미 수행한 검색어와 중복 금지 +- target ontology와 직접 관련 없는 일반 검색어 금지 + +### 12.2 페이지 유용성 평가 프롬프트 + +목표: + +- 페이지가 온톨로지 gap을 채우는 데 유용한지 판단 + +출력은 JSON only: + +```json +{ + "useful": true, + "score": 0.0, + "reason": "string", + "recommended_action": "extract", + "matched_entity_types": [], + "matched_predicates": [], + "fills_gaps": [] +} +``` + +평가 기준: + +- 명시적 사실 또는 관계가 있는가 +- 대상 entity/predicate와 연결되는가 +- 출처가 신뢰 가능한가 +- 중복 정보가 아닌가 +- 페이지 본문이 충분히 추출되었는가 + +### 12.3 컨텍스트 추출 프롬프트 + +목표: + +- 전체 페이지 요약이 아니라 온톨로지 구축에 필요한 evidence만 추출 + +출력은 JSON only: + +```json +{ + "contexts": [ + { + "text": "string", + "summary": "string", + "evidence_text": "string", + "entity_candidates": [], + "relation_candidates": [], + "confidence": 0.0 + } + ] +} +``` + +규칙: + +- 본문에 없는 사실 생성 금지 +- 네비게이션, 광고, 푸터, 배송/정책 문구 제외 +- evidence는 짧게 유지 +- ontology predicate와 매핑 가능한 relation candidate 우선 + +### 12.4 다음 검색 판단 프롬프트 + +목표: + +- 더 검색할지, 종료할지, 어떤 gap을 더 채울지 결정 + +출력은 JSON only: + +```json +{ + "status": "continue", + "reason": "string", + "new_queries": [], + "remaining_gaps": [], + "coverage": {}, + "stop_conditions": [] +} +``` + +규칙: + +- 충분하면 `status="done"`과 빈 `new_queries` +- 계속할 경우 최대 4개 query +- 새 query는 이전 query와 중복 금지 +- 검색 목적과 우선순위를 포함 + +## 13. 원본 코드의 위험 요소와 수정 필요 사항 + +| 위험 요소 | 원본 위치 | 문제 | 수정 방향 | +|---|---|---|---| +| `eval(response)` | 검색어 파싱, 추가 검색어 파싱 | LLM 출력 실행 위험 | `json.loads`, `ast.literal_eval`, JSON schema 사용 | +| API 키 하드코딩 | 설정 상수 | 보안 및 운영 부적합 | 환경 변수/설정 파일/프로젝트 설정으로 이동 | +| 전역 중복 URL 관리 없음 | 메인 루프 | 반복 간 URL 재처리 가능 | 세션 단위 `visited_urls` 저장 | +| 컨텍스트 구조 없음 | `extract_relevant_context_async` | 출처/근거 추적 어려움 | 구조화 JSON과 evidence 필드 | +| 출처 인용 없음 | 최종 보고서 | 검증 어려움 | URL, title, evidence 연결 | +| retry/backoff 없음 | 모든 API 호출 | 일시적 실패에 취약 | retry 정책 추가 | +| rate limit 제어 없음 | `asyncio.gather` | API 제한 초과 가능 | semaphore/concurrency limit | +| 토큰 예산 단순 절단 | `page_text[:20000]` | 중요 정보 손실 가능 | content chunking, source zone 기반 선별 | +| 품질 필터 약함 | Yes/No 평가 | 낮은 품질 페이지 통과 가능 | score, source quality, duplication check | +| 온톨로지 저장 없음 | 전체 | 보고서 생성에 그침 | 기존 repository/entity/claim 저장과 연결 | + +## 14. 거의 변형 없이 사용할 수 있는 부분 + +다음은 원본 구조를 거의 유지해도 되는 부분이다. + +1. 반복형 연구 루프 개념 + - 초기 검색어 생성 + - 검색 실행 + - 링크 중복 제거 + - 페이지 평가 + - 관련 컨텍스트 추출 + - 추가 검색 판단 + - 최종 요약 + +2. 비동기 실행 패턴 + - 검색어별 검색을 `asyncio.gather`로 병렬 처리 + - URL별 fetch/evaluate/extract를 병렬 처리 + +3. 검색 확장 프롬프트의 역할 분리 + - 검색어 생성 프롬프트 + - 페이지 유용성 평가 프롬프트 + - 컨텍스트 추출 프롬프트 + - 추가 검색 판단 프롬프트 + +4. Gradio 버전의 로그 누적 방식 + - UI에 중간 진행 로그를 보여주는 개념은 현재 웹 프론트엔드 progress 표시로 재사용 가능 + +5. `` 또는 종료 토큰 개념 + - 다만 실제 구현은 JSON `status="done"`으로 바꾸는 것이 안전하다. + +## 15. 변형이 필요한 부분 + +다음은 반드시 현재 프로젝트 방식에 맞춰 수정해야 한다. + +| 원본 방식 | 변경 필요 방식 | +|---|---| +| Notebook 내부 상수 | 프로젝트 설정/환경변수/DB 저장 설정 | +| OpenRouter 전용 호출 | OpenAI-compatible async client | +| SERPAPI 고정 | 검색 provider 인터페이스 | +| Jina 고정 | fetch provider 또는 기존 fetcher fallback | +| plain text context | evidence 포함 구조화 context | +| 최종 보고서 중심 | entity/claim/evidence 중심 | +| print 로그 | DB research session history | +| Gradio UI | 기존 FastAPI + 웹 프론트엔드 | +| `eval` 파싱 | JSON schema/validator | +| 반복 내 중복 제거 | 세션 전체 URL/query 중복 제거 | + +## 16. 범용 온톨로지 구축 플랫폼에서의 목표 기능명세 + +### 16.1 기능명: 외부 검색 기반 온톨로지 연구 루프 + +설명: + +사용자가 입력한 연구 목표 또는 자동 감지된 온톨로지 gap을 기반으로 외부 웹 검색을 수행하고, 관련 문서에서 엔티티/관계 후보를 추출하여 지식 그래프 구축에 반영하는 반복형 연구 기능. + +사용자 가치: + +- seed URL 없이도 외부 웹에서 지식 후보를 발견할 수 있다. +- 지식 공백을 기반으로 자동 검색 계획을 세울 수 있다. +- 검색 결과가 단순 보고서가 아니라 온톨로지 엔티티/클레임으로 이어진다. +- 연구 과정과 근거를 세션 단위로 추적할 수 있다. + +### 16.2 주요 사용자 시나리오 + +#### 시나리오 A: 새 도메인 온톨로지 후보 수집 + +1. 사용자가 도메인과 목표를 입력한다. +2. 시스템이 도메인 ontology seed를 바탕으로 검색어를 생성한다. +3. 외부 검색을 수행한다. +4. 유용한 페이지를 선별한다. +5. 컨텍스트와 엔티티/관계 후보를 추출한다. +6. 부족한 entity type/predicate를 파악해 추가 검색한다. +7. 최종적으로 후보 ontology와 출처 목록을 제시한다. + +#### 시나리오 B: 기존 지식 그래프의 공백 보완 + +1. `GapTaskPlanner`가 부족한 predicate 또는 entity type을 찾는다. +2. 시스템이 gap별 검색어를 생성한다. +3. 검색 결과를 수집하고 유용성을 평가한다. +4. 기존 그래프에 없는 claim 후보만 우선 저장한다. +5. 충돌 가능 claim은 review 상태로 남긴다. + +#### 시나리오 C: 특정 엔티티 확장 연구 + +1. 사용자가 특정 entity를 선택한다. +2. 시스템이 entity 이름, 타입, 기존 관계를 기반으로 검색어를 만든다. +3. 관련 source에서 추가 속성/관계를 추출한다. +4. entity 중심 neighborhood graph를 확장한다. + +### 16.3 기능 요구사항 + +| ID | 요구사항 | +|---|---| +| ODR-FR-001 | 시스템은 연구 목표에서 최대 N개의 초기 검색어를 생성해야 한다. | +| ODR-FR-002 | 시스템은 검색어별 외부 검색을 비동기로 실행해야 한다. | +| ODR-FR-003 | 시스템은 검색 결과 URL을 세션 단위로 중복 제거해야 한다. | +| ODR-FR-004 | 시스템은 각 URL의 본문을 fetch/parser 계층을 통해 텍스트화해야 한다. | +| ODR-FR-005 | 시스템은 각 페이지의 유용성을 점수와 이유로 평가해야 한다. | +| ODR-FR-006 | 시스템은 유용한 페이지에서 관련 evidence context를 추출해야 한다. | +| ODR-FR-007 | 시스템은 추출 context를 온톨로지 extractor에 전달해 entity/claim 후보를 생성해야 한다. | +| ODR-FR-008 | 시스템은 entity/claim 후보를 기존 repository에 저장해야 한다. | +| ODR-FR-009 | 시스템은 누적 결과와 gap을 기반으로 추가 검색 여부를 결정해야 한다. | +| ODR-FR-010 | 시스템은 반복 종료 후 연구 세션 요약 보고서를 생성해야 한다. | +| ODR-FR-011 | 시스템은 모든 검색어, URL, 평가, 추출 결과, 오류를 세션 history에 저장해야 한다. | +| ODR-FR-012 | 시스템은 사용자가 실행 중인 연구 세션을 취소할 수 있어야 한다. | + +### 16.4 비기능 요구사항 + +| ID | 요구사항 | +|---|---| +| ODR-NFR-001 | 외부 API 호출은 timeout과 retry를 가져야 한다. | +| ODR-NFR-002 | 동시 요청 수는 provider별로 제한 가능해야 한다. | +| ODR-NFR-003 | API 키는 코드에 하드코딩하지 않는다. | +| ODR-NFR-004 | LLM JSON 출력은 schema validation을 통과해야 한다. | +| ODR-NFR-005 | 연구 세션은 partial failure를 허용하고 가능한 결과를 보존해야 한다. | +| ODR-NFR-006 | 최종 entity/claim은 evidence와 source URL을 잃지 않아야 한다. | +| ODR-NFR-007 | 동일 URL과 동일 query는 세션 내 중복 실행하지 않는다. | +| ODR-NFR-008 | 긴 페이지는 chunking 또는 content zone 기반으로 처리한다. | +| ODR-NFR-009 | 비용, 호출 수, 처리 시간 지표를 기록한다. | + +## 17. 구현 우선순위 + +### Phase 1: 원본 루프의 안전한 모듈화 + +- `eval` 제거 +- 검색/LLM/fetch 클라이언트 분리 +- FastAPI에서 호출 가능한 `ExternalResearchLoop` 추가 +- 검색 결과와 로그를 `ResearchMemoryStore`에 저장 +- 최종 보고서 문자열 생성까지 구현 + +### Phase 2: 온톨로지 추출 연결 + +- 유용 페이지를 `ExtractionPageContext`로 변환 +- 기존 `LLMJsonExtractor` 호출 +- `KnowledgeRepository.save_extraction_bundle`로 저장 +- extracted claim과 source/evidence 연결 확인 + +### Phase 3: gap-aware research + +- `GapTaskPlanner` 결과를 검색어 생성 프롬프트에 포함 +- entity type/predicate coverage 산출 +- 추가 검색 판단을 gap 기반으로 변경 + +### Phase 4: UI/운영 기능 + +- 웹 프론트엔드에 외부 연구 실행 화면 추가 +- 세션 로그, 검색어, 유용성 평가, 추출 claim 표시 +- 취소, 재시도, 결과 승인/반려 기능 추가 + +## 18. 테스트 명세 + +### 18.1 단위 테스트 + +| 테스트 | 검증 내용 | +|---|---| +| 검색어 JSON 파싱 | 잘못된 LLM 응답을 안전하게 거부 | +| URL normalization | fragment/trailing slash 중복 제거 | +| 검색 결과 dedupe | 같은 URL이 한 번만 처리됨 | +| usefulness parser | 점수/enum/schema 검증 | +| next decision parser | `continue`/`done` 처리 | +| context extractor parser | evidence 없는 결과 거부 | + +### 18.2 통합 테스트 + +| 테스트 | 검증 내용 | +|---|---| +| mock search provider loop | 검색 → fetch → 평가 → 추출 → 종료 전체 흐름 | +| partial API failure | 일부 URL 실패해도 세션이 partial/completed로 보존 | +| repository 저장 | page, entity, claim, history 저장 확인 | +| duplicate iteration | 반복 간 동일 URL 재처리 방지 | +| gap-aware query | gap 입력이 query purpose에 반영됨 | + +### 18.3 운영 테스트 + +| 테스트 | 검증 내용 | +|---|---| +| concurrency limit | 동시 호출 수 제한 | +| timeout | 오래 걸리는 provider 호출 중단 | +| cancellation | 세션 취소 요청 반영 | +| cost tracking | LLM/search/fetch 호출 수 기록 | + +## 19. 결론 + +`OpenDeepResearcher`는 코드 규모는 작지만, 범용 온톨로지 구축 플랫폼에 매우 중요한 “외부 검색 기반 자기 확장 연구 루프”의 핵심 패턴을 제공한다. 현재 프로젝트는 이미 크롤러, parser, extractor, repository, graph research loop, memory store를 갖고 있으므로, 원본을 그대로 제품 코드로 넣기보다는 다음 부분을 원형에 가깝게 차용하는 것이 가장 효율적이다. + +- LLM 기반 검색어 생성 +- 검색 결과 병렬 수집 +- URL 중복 제거 +- 페이지 유용성 LLM 판정 +- 관련 컨텍스트 추출 +- 누적 컨텍스트 기반 추가 검색 판단 +- 최종 연구 요약 생성 + +다만 현재 플랫폼의 목표는 보고서 생성이 아니라 온톨로지 지식 구축이므로, 최종 산출물은 반드시 `entity`, `claim`, `evidence`, `source`, `confidence`, `gap coverage` 중심으로 재설계해야 한다. 이 관점에서 OpenDeepResearcher는 “최종 보고서 도구”가 아니라 “외부 지식 발견 엔진”의 시작점으로 활용하는 것이 가장 적절하다. diff --git a/오픈소스분석자료/Trafilatura_분석_및_기능명세.md b/오픈소스분석자료/Trafilatura_분석_및_기능명세.md new file mode 100644 index 0000000..4164055 --- /dev/null +++ b/오픈소스분석자료/Trafilatura_분석_및_기능명세.md @@ -0,0 +1,749 @@ +# Trafilatura 분석 및 기능 명세 + +분석 대상: `C:\Users\lasta\MyProject\AI\참고\trafilatura-master` +분석일: 2026-05-13 +대상 버전: `trafilatura.__version__ = 2.0.0` +라이선스: Apache-2.0 + +## 1. 결론 요약 + +Trafilatura는 웹 문서에서 본문, 댓글, 메타데이터, 링크 후보를 추출하기 위한 Python 라이브러리이자 CLI 도구다. 범용 온톨로지 구축 플랫폼에서는 “웹 원천 데이터 수집 및 정제 계층”의 기본 소스로 거의 그대로 사용하기에 적합하다. + +가장 가치가 큰 기능은 다음이다. + +- 웹 페이지 다운로드: `fetch_url()`, `fetch_response()` +- HTML 본문 추출: `extract()`, `bare_extraction()` +- 구조 보존 추출: XML/HTML/Markdown/JSON/TEI 출력 +- 메타데이터 추출: 제목, 저자, 날짜, canonical URL, 사이트명, 설명, 태그, 라이선스, 대표 이미지 +- 링크 발견: RSS/Atom/JSON Feed, sitemap, robots.txt sitemap, focused crawler +- 중복 제거: 본문 세그먼트 LRU 중복 검사, 문서 fingerprint용 SimHash +- 설정 객체: `Extractor`, 결과 객체: `Document` + +플랫폼 통합 권장 방식은 `bare_extraction(output_format="python", with_metadata=True)`를 기본으로 삼는 것이다. 이 방식은 문자열 출력으로 손실되기 전의 `Document` 객체, `body` XML tree, `commentsbody`, 정규화 텍스트, 메타데이터를 받을 수 있어 온톨로지 구축 전처리 단계와 연결하기 쉽다. + +주의할 점은 전역 상태다. 다운로드 pool, 중복 LRU cache, crawler URL store가 모듈 전역에 존재한다. 단일 작업에서는 편하지만, 멀티 프로젝트/멀티 테넌트 플랫폼에서는 작업 단위 격리, cache reset, 도메인별 crawl 상태 저장을 별도 래퍼에서 관리해야 한다. + +## 2. 프로젝트 구조 + +주요 디렉터리와 파일: + +| 경로 | 역할 | +|---|---| +| `trafilatura/__init__.py` | 공개 API re-export | +| `trafilatura/core.py` | 추출 파이프라인 진입점 | +| `trafilatura/main_extractor.py` | Trafilatura 기본 본문/댓글 추출 알고리즘 | +| `trafilatura/htmlprocessing.py` | HTML 정리, 태그 변환, 링크 밀도 제거 | +| `trafilatura/metadata.py` | 메타 태그, JSON-LD, OpenGraph, 날짜/저자/URL 추출 | +| `trafilatura/json_metadata.py` | JSON-LD/schema.org 메타데이터 파싱 | +| `trafilatura/downloads.py` | HTTP 다운로드, Response 객체, 병렬 다운로드 | +| `trafilatura/feeds.py` | RSS/Atom/JSON feed 발견 및 URL 추출 | +| `trafilatura/sitemaps.py` | sitemap/robots.txt 기반 URL 발견 | +| `trafilatura/spider.py` | focused crawler | +| `trafilatura/deduplication.py` | LRU segment dedup, SimHash fingerprint | +| `trafilatura/xml.py` | JSON/CSV/XML/TEI/TXT 변환 | +| `trafilatura/settings.py` | `Extractor`, `Document`, 전역 상수 | +| `trafilatura/settings.cfg` | 사용자 조정 가능한 기본 설정 | +| `docs/` | 사용법/설정/다운로드/크롤링/중복제거 문서 | +| `tests/` | 단위 테스트, 실제 웹 페이지 평가 데이터 | + +## 3. 외부 의존성 + +`pyproject.toml` 기준 필수 의존성: + +- `certifi`: TLS 인증서 +- `charset_normalizer >= 3.4.0`: 인코딩 탐지 +- `courlan >= 1.3.2`: URL 정규화, 필터링, URL store, 링크 추출 +- `htmldate >= 1.9.2`: 날짜 추출 +- `justext >= 3.0.1`: fallback 본문 추출 +- `lxml`: HTML/XML 파싱 및 XPath +- `urllib3 >= 1.26, < 3`: HTTP client + +선택 의존성 `trafilatura[all]`: + +- `brotli`, `zstandard`: 압축 응답 처리 강화 +- `py3langid`: 언어 판별 +- `pycurl`: 빠른 HTTP backend +- `urllib3[socks]`: SOCKS proxy +- `htmldate[speed]`: 날짜 추출 속도 개선 + +## 4. 핵심 데이터 모델 + +### 4.1 `Extractor` + +파일: `trafilatura/settings.py` + +추출 옵션을 담는 설정 객체다. 함수 인자가 많기 때문에 플랫폼에서는 개별 인자보다 `Extractor` 객체를 만들어 넘기는 방식을 권장한다. + +주요 속성: + +| 속성 | 의미 | 기본값 | +|---|---|---| +| `format` | 출력 형식 | `txt` | +| `fast` | fallback 알고리즘 생략 | `False` | +| `focus` | `balanced`, `precision`, `recall` | `balanced` | +| `comments` | 댓글 추출 | `True` | +| `formatting` | 굵게/기울임 등 구조 보존 | `False`, Markdown이면 자동 True | +| `links` | 링크 target 보존 | `False` | +| `images` | 이미지 정보 보존 | `False` | +| `tables` | table 추출 | `True` | +| `dedup` | 중복 세그먼트 제거 | `False` | +| `lang` | 목표 언어 필터 | `None` | +| `url` | 원문 URL | `None` | +| `with_metadata` | 메타데이터 추출/출력 포함 | `False` | +| `only_with_metadata` | 핵심 메타데이터 없으면 폐기 | `False` | +| `tei_validation` | TEI 출력 검증 | `False` | +| `date_params` | `htmldate` 날짜 추출 옵션 | 현재 날짜 max_date | +| `author_blacklist` | 제외할 저자명 집합 | 빈 set | +| `url_blacklist` | 제외할 URL 집합 | 빈 set | + +### 4.2 `Document` + +파일: `trafilatura/settings.py` + +추출 결과와 메타데이터를 담는 객체다. + +필드: + +| 필드 | 설명 | +|---|---| +| `title` | 제목 | +| `author` | 저자 | +| `url` | canonical URL 또는 입력 URL | +| `hostname` | 호스트명 | +| `description` | 설명/요약 메타 | +| `sitename` | 사이트명/매체명 | +| `date` | 발행일 | +| `categories` | 카테고리 목록 | +| `tags` | 태그 목록 | +| `fingerprint` | SimHash 기반 문서 fingerprint | +| `id` | 호출자가 넘긴 record id | +| `license` | 라이선스 정보 | +| `body` | 본문 XML tree | +| `comments` | 댓글 텍스트 | +| `commentsbody` | 댓글 XML tree | +| `raw_text` | 내부 추출 텍스트 | +| `text` | 최종 출력 문자열 또는 python 모드 텍스트 | +| `language` | 감지 언어 | +| `image` | 대표 이미지 | +| `pagetype` | OpenGraph page type | +| `filedate` | 파일 날짜 | + +`Document.as_dict()`로 dict 변환이 가능하다. + +## 5. 본문 추출 파이프라인 + +핵심 진입점: + +- `extract(filecontent, ...) -> Optional[str]` +- `extract_with_metadata(filecontent, ...) -> Optional[Document]` +- `bare_extraction(filecontent, ...) -> Optional[Document]` + +권장 기본 호출: + +```python +from trafilatura import bare_extraction + +doc = bare_extraction( + html, + url=url, + output_format="python", + with_metadata=True, + include_comments=False, + include_tables=True, + include_formatting=True, + include_links=True, + deduplicate=True, +) +``` + +처리 순서: + +1. `load_html()`로 문자열/bytes/LXML 입력을 HTML tree로 변환한다. +2. `target_language`가 있고 fast mode이거나 언어 판별 모듈이 없으면 HTML lang 속성을 먼저 검사한다. +3. `with_metadata=True`이면 `extract_metadata()`로 메타데이터를 추출한다. +4. `only_with_metadata=True`이면 `date`, `title`, `url`이 모두 없을 때 문서를 폐기한다. +5. 사용자가 지정한 `prune_xpath`를 tree에서 제거한다. +6. `tree_cleaning()`으로 불필요 태그와 섹션을 제거한다. +7. `convert_tags()`로 HTML 태그를 내부 구조 태그로 변환한다. +8. `include_comments=True`이면 댓글 영역을 먼저 추출하고 본문 tree에서 분리한다. +9. `extract_content()`로 Trafilatura 기본 본문 추출을 수행한다. +10. `fast=False`이면 readability/jusText 계열 fallback과 비교하여 더 나은 결과를 선택한다. +11. 본문 길이가 너무 짧고 precision 모드가 아니면 `baseline()` fallback을 수행한다. +12. `deduplicate=True`이면 LRU cache 기반 중복 본문을 폐기한다. +13. `target_language`가 있으면 최종 텍스트 언어를 검사한다. +14. 요청 형식에 따라 TXT/Markdown/JSON/CSV/HTML/XML/TEI로 변환한다. + +## 6. 출력 형식 + +지원 형식: + +| 형식 | 함수 인자 | 용도 | +|---|---|---| +| Python object | `bare_extraction(output_format="python")` | 온톨로지 파이프라인 권장 | +| Plain text | `output_format="txt"` | 단순 텍스트 저장 | +| Markdown | `output_format="markdown"` | 구조 일부 보존, LLM 입력에 유용 | +| JSON | `output_format="json"` | API 저장/교환 | +| CSV | `output_format="csv"` | 배치 처리 결과 | +| HTML | `output_format="html"` | 정제된 HTML preview | +| XML | `output_format="xml"` | 구조 보존 | +| XML-TEI | `output_format="xmltei"` | 인문학/말뭉치 표준 | + +온톨로지 구축 플랫폼에서는 다음 전략이 적합하다. + +- 원천 보존: raw HTML 별도 저장 +- 추출 본문: `Document.text` +- 구조 본문: `Document.body` +- 메타데이터: `Document.as_dict()` 중 primitive 필드 +- LLM/IE 입력: Markdown 또는 XML 변환본 +- 장기 말뭉치 교환: XML-TEI 선택 가능 + +## 7. 메타데이터 추출 기능 + +파일: `trafilatura/metadata.py`, `trafilatura/json_metadata.py` + +추출 출처: + +- OpenGraph: `og:title`, `og:description`, `og:site_name`, `og:image`, `og:type`, `og:url` +- Twitter cards: `twitter:title`, `twitter:description`, `twitter:image`, `twitter:site`, `twitter:url` +- 표준 meta name: `author`, `description`, `keywords`, `publisher`, `dc.*`, `dcterms.*`, `citation_*` +- JSON-LD/schema.org: `application/ld+json` +- HTML heading: `h1`, `h2` +- canonical/base/alternate link +- `htmldate.find_date()` 기반 날짜 추출 +- Creative Commons license URL/text 패턴 + +기능 명세: + +| 기능명 | 입력 | 출력 | 실패 조건 | +|---|---|---|---| +| `extract_metadata` | HTML tree/string, URL, date params | `Document` | HTML 파싱 실패 시 빈/부분 Document | +| `extract_meta_json` | HTML tree, Document | JSON-LD 반영 Document | JSON 파싱 실패 시 fallback parser 사용 | +| `extract_opengraph` | HTML tree | dict | 해당 meta 없으면 None 값 | +| `extract_title` | HTML tree | str 또는 None | 제목 후보 없음 | +| `extract_author` | HTML tree | str 또는 None | 저자 후보 없음 | +| `extract_url` | HTML tree, default URL | str 또는 None | URL 후보 없음/invalid | +| `extract_license` | HTML tree | str 또는 None | license 후보 없음 | + +플랫폼 적용: + +- `title`, `author`, `date`, `sitename`, `url`, `description`, `tags`, `categories`를 `SourceDocument` 메타로 저장한다. +- `url`과 `date`는 provenance 및 temporal ontology 축의 핵심 속성으로 사용한다. +- `tags/categories`는 초기 후보 개념(seed concept)으로 사용할 수 있으나, 사이트별 노이즈가 많으므로 confidence를 낮게 둔다. + +## 8. 다운로드 기능 + +파일: `trafilatura/downloads.py` + +주요 API: + +| 기능명 | 설명 | +|---|---| +| `fetch_url(url)` | HTML을 다운로드하고 decode한 문자열 반환 | +| `fetch_response(url, decode=False, with_headers=False)` | `Response` 객체 반환 | +| `buffered_downloads()` | URL buffer 병렬 다운로드 | +| `buffered_response_downloads()` | Response 객체 병렬 다운로드 | +| `is_live_page(url)` | URL 접근 가능성 확인 | + +`Response` 필드: + +- `data`: bytes +- `headers`: optional dict +- `html`: optional decoded string +- `status`: HTTP status +- `url`: 최종 URL + +설정: + +| 설정 | 기본값 | 설명 | +|---|---:|---| +| `DOWNLOAD_TIMEOUT` | 30 | 요청 timeout | +| `MAX_FILE_SIZE` | 20000000 | 최대 파일 크기 | +| `MIN_FILE_SIZE` | 10 | 최소 파일 크기 | +| `SLEEP_TIME` | 5.0 | 동일 host 요청 간격 | +| `MAX_REDIRECTS` | 2 | redirect 허용 횟수 | +| `USER_AGENTS` | empty | 사용자 user-agent 후보 | +| `COOKIE` | empty | 요청 cookie | + +플랫폼 적용: + +- 이미 프로젝트에 크롤러가 있다면 `fetch_url()`을 직접 대체하기보다 HTML 추출 단계에 Trafilatura를 붙이는 것이 안전하다. +- Trafilatura downloader를 쓰는 경우 도메인별 throttling과 robots 정책을 서비스 단에서 명시적으로 기록해야 한다. +- `fetch_response(decode=True, with_headers=True)`는 최종 URL, status, header provenance 저장에 적합하다. + +## 9. 링크 발견 기능 + +### 9.1 Feed 발견 + +파일: `trafilatura/feeds.py` + +주요 API: + +```python +from trafilatura.feeds import find_feed_urls + +urls = find_feed_urls("https://example.com", target_lang="ko") +``` + +지원: + +- Atom +- RSS/RDF +- JSON Feed +- HTML 내 `` +- HTML 내 feed 후보 `` +- Google News RSS fallback +- 언어 필터/도메인 유사도 필터 + +기능 명세: + +| 기능명 | 입력 | 출력 | 정책 | +|---|---|---|---| +| `find_feed_urls` | URL, target_lang, external, sleep_time | 정렬/중복 제거 URL 목록 | 기본적으로 유사 도메인만 허용 | +| `determine_feed` | HTML 문자열 | feed URL 목록 | feed MIME/type/URL 패턴 사용 | +| `extract_links` | feed 문자열 | article URL 목록 | feedburner/feedproxy 예외 처리 | + +### 9.2 Sitemap 발견 + +파일: `trafilatura/sitemaps.py` + +주요 API: + +```python +from trafilatura.sitemaps import sitemap_search + +urls = sitemap_search("https://example.com", target_lang="ko") +``` + +지원: + +- `robots.txt`의 Sitemap 항목 +- 일반 sitemap guess: `sitemap.xml`, `sitemap.xml.gz`, `sitemap_index.xml`, `sitemap_news.xml` +- XML sitemap +- TXT sitemap +- hreflang 기반 target language +- nested sitemap +- URL filter +- 유사 도메인 필터 + +기능 명세: + +| 기능명 | 입력 | 출력 | 제한 | +|---|---|---|---| +| `sitemap_search` | URL, target_lang, external, sleep_time, max_sitemaps | URL 목록 | 기본 최대 sitemap 10,000개 | +| `find_robots_sitemaps` | base URL | sitemap URL 목록 | robots.txt 10KB 초과 시 폐기 | +| `is_plausible_sitemap` | URL, contents | bool | HTML 응답이면 sitemap 아님 | + +### 9.3 Focused crawler + +파일: `trafilatura/spider.py` + +주요 API: + +```python +from trafilatura.spider import focused_crawler + +todo, known = focused_crawler( + "https://example.com", + max_seen_urls=10, + max_known_urls=100000, + lang="ko", +) +``` + +특징: + +- 시작 URL 기준 내부 링크 탐색 +- robots.txt 파싱 및 `can_fetch("*", link)` 적용 +- `Crawl-Delay` 반영 +- navigation page 우선 탐색 +- `todo`, `known_links`를 외부에서 주입해 단계별 crawl 가능 +- 언어 필터 가능 + +주의: + +- `URL_STORE`가 모듈 전역이다. 여러 crawl 작업을 동시에 실행하면 상태 충돌 위험이 있다. +- 플랫폼에서는 Trafilatura crawler를 그대로 병렬 호출하기보다, 작업별 프로세스 격리 또는 `courlan.UrlStore` 래핑이 필요하다. + +## 10. 중복 제거와 fingerprint + +파일: `trafilatura/deduplication.py` + +기능: + +- 세그먼트 중복 제거: `duplicate_test(element, options)` +- 문서 fingerprint: `content_fingerprint(content)` +- SimHash 비교: `Simhash.similarity()` +- 도메인 문자열 유사도: `is_similar_domain()` +- token sampling과 BLAKE2b hash + +기능 명세: + +| 기능명 | 입력 | 출력 | 용도 | +|---|---|---|---| +| `duplicate_test` | LXML element, Extractor/options | bool | 반복 boilerplate 제거 | +| `content_fingerprint` | 문자열 | hex SimHash | 문서 near-duplicate 키 | +| `Simhash.similarity` | 다른 Simhash | 0.0-1.0 | 유사 문서 판정 | +| `generate_bow_hash` | 문자열 | bytes | bag-of-words hash | + +설정: + +- `MIN_DUPLCHECK_SIZE = 100` +- `MAX_REPETITIONS = 2` +- 전역 LRU cache 크기: `LRU_SIZE = 4096` + +플랫폼 적용: + +- 추출 중 `deduplicate=True`는 페이지 내부 반복 요소 제거에 유용하다. +- 문서 단위 중복 제거는 `content_fingerprint(title + raw_text)`를 저장하고, 플랫폼 DB에서 유사도/중복 정책을 별도로 운영하는 것이 좋다. +- 전역 `LRU_TEST`는 긴 배치 작업에서 도메인 간 영향을 줄 수 있으므로 작업 단위로 `trafilatura.meta.reset_caches()` 호출을 고려한다. + +## 11. HTML 처리와 구조 보존 + +파일: `trafilatura/htmlprocessing.py`, `trafilatura/main_extractor.py`, `trafilatura/xml.py` + +보존 가능한 구조: + +- 문단: `p` +- 제목: `head` +- 목록: `list`, `item` +- 인용: `quote` +- 코드: `code` +- 줄바꿈: `lb` +- 삭제/변경: `del` +- 강조/서식: `hi` +- 링크: `ref target="..."` +- 이미지: `graphic` +- 표: `table`, `row`, `cell` + +온톨로지 플랫폼 관점에서는 `Document.body` XML tree가 중요하다. + +활용 예: + +- 제목/소제목을 문서 chunk hierarchy로 사용 +- 목록 항목을 독립 주장 후보로 분해 +- 표를 entity-attribute 후보로 변환 +- 링크를 외부 참조 관계로 저장 +- 이미지 alt/title을 보조 설명 텍스트로 저장 +- 코드 블록은 일반 자연어 추출 대상에서 제외하거나 별도 타입으로 저장 + +## 12. 설정 명세 + +파일: `trafilatura/settings.cfg` + +플랫폼 기본 권장값: + +| 목적 | 설정/인자 | 권장 | +|---|---|---| +| 고품질 본문 추출 | `fast=False` | 기본 | +| 대량 수집 속도 | `fast=True` | 낮은 priority batch | +| 온톨로지 입력 | `output_format="python"` | 기본 | +| 구조 보존 | `include_formatting=True`, `include_links=True` | 권장 | +| 댓글 제외 | `include_comments=False` | 기본 권장 | +| 표 포함 | `include_tables=True` | 권장 | +| 이미지 후보 | `include_images=True` | 필요 시 | +| 언어 필터 | `target_language="ko"` 등 | 소스 도메인별 설정 | +| URL blacklisting | `url_blacklist` | 플랫폼 정책 DB와 연동 | +| 저자 제외 | `author_blacklist` | “편집부”, “관리자” 등 제거 | +| 날짜 strictness | `date_extraction_params` | max_date 고정 | + +`settings.cfg`에서 조정할 값: + +| 키 | 의미 | +|---|---| +| `DOWNLOAD_TIMEOUT` | 다운로드 제한 시간 | +| `MAX_FILE_SIZE` | 입력 최대 크기 | +| `MIN_FILE_SIZE` | 입력 최소 크기 | +| `SLEEP_TIME` | 동일 도메인 요청 간격 | +| `MAX_REDIRECTS` | redirect 횟수 | +| `MIN_EXTRACTED_SIZE` | fallback 발동 기준 본문 길이 | +| `MIN_OUTPUT_SIZE` | 최종 본문 최소 길이 | +| `EXTRACTION_TIMEOUT` | CLI 추출 timeout | +| `MIN_DUPLCHECK_SIZE` | 중복 검사 최소 길이 | +| `MAX_REPETITIONS` | 허용 반복 횟수 | +| `EXTENSIVE_DATE_SEARCH` | 날짜 탐색 범위 | +| `EXTERNAL_URLS` | feed/sitemap 외부 URL 허용 | + +## 13. 범용 온톨로지 구축 플랫폼 통합 설계 + +### 13.1 권장 파이프라인 + +```text +Seed URL + -> Feed/Sitemap/Crawler URL discovery + -> URL normalization/dedup + -> Download raw HTML + -> Trafilatura bare_extraction + -> Document metadata/provenance 저장 + -> 구조 유지 chunking + -> 언어/품질/중복 필터 + -> 엔티티/관계/속성 후보 추출 + -> ontology schema mapping + -> graph/vector/document store 적재 +``` + +### 13.2 Trafilatura 채택 범위 + +거의 그대로 사용 권장: + +- `core.py`의 `extract`, `bare_extraction`, `extract_with_metadata` +- `metadata.py`의 메타데이터 추출 +- `downloads.py`의 단일 다운로드 및 Response 모델 +- `feeds.py`의 feed discovery +- `sitemaps.py`의 sitemap discovery +- `deduplication.py`의 `content_fingerprint`, `Simhash` +- `xml.py`의 출력 변환 +- `settings.py`의 `Extractor`, `Document` + +래퍼 필요: + +- `spider.py`: 전역 `URL_STORE` 때문에 작업별 격리 필요 +- `downloads.py`: 전역 HTTP pool과 user-agent/cookie 정책 관리 필요 +- `deduplication.py`: 전역 LRU cache 초기화 정책 필요 +- CLI 계층: 플랫폼 내부에서는 직접 CLI보다 Python API 사용 권장 + +수정 또는 확장 후보: + +- 한국어/다국어 품질 점수 산정 +- ontology chunk id/provenance 부여 +- 표를 relation candidate로 변환하는 후처리 +- link target을 source graph edge로 변환 +- source별 extraction profile 관리 +- 실패 사유와 추출 품질 metrics 기록 + +### 13.3 플랫폼 내 모듈 제안 + +| 플랫폼 모듈 | Trafilatura 연결 | +|---|---| +| `SourceDiscoveryService` | `find_feed_urls`, `sitemap_search`, `focused_crawler` | +| `PageFetchService` | `fetch_response` | +| `ContentExtractionService` | `bare_extraction` | +| `MetadataNormalizer` | `Document` 필드 정규화 | +| `DocumentChunker` | `Document.body` XML tree 기반 | +| `QualityScorer` | 본문 길이, 언어, 메타 completeness, 중복 여부 | +| `ProvenanceStore` | URL, hostname, date, fingerprint, raw HTML path | +| `OntologyCandidateBuilder` | title/head/list/table/ref/tag 기반 candidate 생성 | + +## 14. 정확한 기능 명세 + +### F-001 웹 페이지 다운로드 + +- 입력: URL, SSL 옵션, config/options +- 처리: HTTP GET, redirect 제한, timeout, 크기 제한, charset decode +- 출력: HTML 문자열 또는 `Response` +- 예외/실패: non-200, 크기 미달/초과, SSL/네트워크 오류 +- 수용 기준: 성공 시 최종 URL과 status를 기록할 수 있어야 한다. + +### F-002 HTML 파싱 + +- 입력: HTML string/bytes/LXML element +- 처리: 압축 파일 처리, encoding detect, faulty HTML repair, LXML tree 생성 +- 출력: `HtmlElement` +- 실패: 빈 입력, 파싱 불가 +- 수용 기준: 문자열과 이미 파싱된 LXML tree 모두 동일 추출 파이프라인에 들어갈 수 있어야 한다. + +### F-003 본문 추출 + +- 입력: HTML tree, `Extractor` +- 처리: cleanup, tag conversion, main extractor, fallback 비교, baseline rescue +- 출력: `Document.body`, `Document.text`, `Document.raw_text` +- 실패: 본문 최소 길이 미달, 언어 불일치, 중복 폐기 +- 수용 기준: `favor_precision`, `favor_recall`, `fast` 옵션에 따라 결과가 조정되어야 한다. + +### F-004 댓글 추출 + +- 입력: HTML tree, `include_comments` +- 처리: 댓글 XPath 후보 추출 후 본문 tree에서 제거 +- 출력: `Document.comments`, `Document.commentsbody` +- 실패: 댓글 길이 미달이면 빈 댓글로 처리 +- 수용 기준: 댓글 포함 여부를 소스/도메인별로 설정할 수 있어야 한다. + +### F-005 표 추출 + +- 입력: HTML ``, `include_tables` +- 처리: row/cell 구조 변환, header cell role 지정, colspan span 계산 +- 출력: XML tree의 `table/row/cell` +- 실패: table 옵션 off이면 제외 +- 수용 기준: ontology relation 후보 생성 단계에서 행/열 구조를 읽을 수 있어야 한다. + +### F-006 링크/이미지 보존 + +- 입력: HTML anchor/img, base URL +- 처리: 상대 URL 절대화, 내부 `ref`/`graphic` 태그 변환 +- 출력: XML tree 내 `target`, image attribute +- 실패: 옵션 off이면 텍스트만 남거나 제거 +- 수용 기준: source graph edge와 media evidence로 저장 가능해야 한다. + +### F-007 메타데이터 추출 + +- 입력: HTML tree, 입력 URL, date params, blacklist +- 처리: OpenGraph, Twitter, meta, JSON-LD, canonical, heading, htmldate +- 출력: `Document` metadata fields +- 실패: 후보 없음 시 None +- 수용 기준: `only_with_metadata=True`에서 `date/title/url` 필수 조건을 적용해야 한다. + +### F-008 언어 필터 + +- 입력: HTML lang 또는 추출 텍스트, target language +- 처리: HTML lang quick check 또는 `py3langid` 판별 +- 출력: pass/fail, `Document.language` +- 실패: target 불일치 시 문서 폐기 +- 수용 기준: 언어 판별 패키지가 없을 때는 HTML lang 기반 검사로 degrade해야 한다. + +### F-009 Feed URL 발견 + +- 입력: 홈페이지/feed URL, target_lang, external +- 처리: feed 직접 파싱, HTML feed link discovery, Google News fallback +- 출력: 기사 URL 목록 +- 실패: 다운로드 실패, feed 없음 +- 수용 기준: 중복 제거와 도메인 유사도 필터가 적용되어야 한다. + +### F-010 Sitemap URL 발견 + +- 입력: 홈페이지/sitemap URL, target_lang, external, max_sitemaps +- 처리: robots.txt sitemap, sitemap guess, nested sitemap, XML/TXT parsing, hreflang +- 출력: 페이지 URL 목록 +- 실패: base URL unreachable, invalid sitemap +- 수용 기준: sitemap index가 깊어도 `max_sitemaps` 제한을 지켜야 한다. + +### F-011 Focused crawling + +- 입력: homepage, max_seen_urls, max_known_urls, todo, known_links, lang +- 처리: robots.txt, crawl delay, navigation URL 우선순위, URL store update +- 출력: 다음 방문 URL 목록, 알려진 URL 목록 +- 실패: 시작 URL invalid, frontier 고갈 +- 수용 기준: 작업 재개를 위해 `todo`, `known_links`를 저장/재주입할 수 있어야 한다. + +### F-012 중복 제거 + +- 입력: XML element 또는 문서 텍스트 +- 처리: LRU exact segment count, SimHash fingerprint +- 출력: duplicate bool 또는 fingerprint +- 실패: 짧은 텍스트는 중복 검사 bypass +- 수용 기준: 문서 fingerprint가 DB unique/near-duplicate 정책과 연결되어야 한다. + +### F-013 출력 변환 + +- 입력: `Document`, `Extractor.format` +- 처리: XML cleanup, JSON/CSV/HTML/TXT/Markdown/XML/TEI 변환 +- 출력: 문자열 +- 실패: unsupported format이면 AttributeError/ValueError +- 수용 기준: 동일 문서에서 최소 TXT, JSON, XML 출력을 생성할 수 있어야 한다. + +### F-014 설정 관리 + +- 입력: `settings.cfg`, `Extractor`, 함수 인자 +- 처리: config parse, 옵션 병합 +- 출력: 실행 시 옵션 객체 +- 실패: config 파일 없음, 필수 key 없음 +- 수용 기준: 플랫폼 source profile에서 추출 설정을 생성할 수 있어야 한다. + +### F-015 cache reset + +- 입력: 없음 +- 처리: module-level cache clear +- 출력: 없음 +- 실패: 없음 +- 수용 기준: 장시간 batch 또는 tenant 전환 시 cache를 초기화할 수 있어야 한다. + +## 15. 품질 및 테스트 자산 + +테스트 구성: + +- `tests/unit_tests.py`: 기본 추출/유틸 단위 테스트 +- `tests/metadata_tests.py`: 메타데이터 추출 +- `tests/json_metadata_tests.py`: JSON-LD 메타 +- `tests/downloads_tests.py`: 다운로드 +- `tests/feeds_tests.py`: feed +- `tests/sitemaps_tests.py`: sitemap +- `tests/spider_tests.py`: crawler +- `tests/deduplication_tests.py`: 중복 제거 +- `tests/xml_tei_tests.py`: XML/TEI +- `tests/eval/`, `tests/cache/`: 실제 웹 페이지 평가 corpus + +플랫폼 통합 테스트로 추가해야 할 것: + +- 한국어 뉴스/블로그/쇼핑/위키/공공기관 샘플 +- JS-heavy 사이트에서 raw HTML 한계 확인 +- HTML table -> relation candidate 변환 테스트 +- `bare_extraction` 결과의 provenance field completeness 테스트 +- 동일 URL/동일 본문/near duplicate 처리 테스트 +- source profile별 precision/recall 옵션 비교 테스트 + +## 16. 위험 요소와 대응 + +| 위험 | 설명 | 대응 | +|---|---|---| +| JS 렌더링 미지원 | 정적 HTML 기반 추출 | Playwright/Crawl4AI 등으로 렌더링 후 HTML을 Trafilatura에 입력 | +| 전역 상태 | HTTP pool, URL_STORE, LRU cache | 작업 단위 프로세스 격리 또는 reset | +| 사이트별 메타 노이즈 | tags/category/author가 부정확할 수 있음 | confidence와 source별 rule 적용 | +| 언어 판별 선택 의존성 | `py3langid` 없으면 제한적 | optional dependency 설치 또는 별도 언어 판별기 연결 | +| 표/이미지 실험적 옵션 | 모든 출력 형식에서 완전 보존되지 않음 | `output_format="python"` 또는 XML tree 직접 사용 | +| robots/정책 준수 | downloader 직접 사용 시 정책 책임 필요 | 플랫폼 crawler policy layer에서 관리 | +| 날짜 추출 recall/precision | `EXTENSIVE_DATE_SEARCH`에 따라 오탐 가능 | source별 date extraction profile 운영 | + +## 17. 구현 권장 래퍼 인터페이스 + +```python +from dataclasses import dataclass +from typing import Optional +from trafilatura import bare_extraction +from trafilatura.settings import Extractor + +@dataclass +class ExtractedWebDocument: + url: str + title: Optional[str] + author: Optional[str] + date: Optional[str] + sitename: Optional[str] + description: Optional[str] + text: str + body_xml: object + metadata: dict + fingerprint: Optional[str] + +def extract_for_ontology(html: str, url: str, lang: Optional[str] = None) -> Optional[ExtractedWebDocument]: + options = Extractor( + output_format="python", + url=url, + with_metadata=True, + comments=False, + tables=True, + formatting=True, + links=True, + images=True, + dedup=True, + lang=lang, + ) + doc = bare_extraction(html, options=options) + if not doc or not doc.text: + return None + return ExtractedWebDocument( + url=doc.url or url, + title=doc.title, + author=doc.author, + date=doc.date, + sitename=doc.sitename, + description=doc.description, + text=doc.text, + body_xml=doc.body, + metadata=doc.as_dict(), + fingerprint=doc.fingerprint, + ) +``` + +## 18. 채택 우선순위 + +1. `bare_extraction()` 기반 ContentExtractionService 구현 +2. `Document` metadata를 프로젝트 DB schema에 매핑 +3. `Document.body` XML tree 기반 chunker 구현 +4. `content_fingerprint()` 기반 중복 문서 정책 추가 +5. `sitemap_search()`와 `find_feed_urls()`를 source discovery에 연결 +6. 필요한 경우 `focused_crawler()`는 전역 상태 격리 후 채택 +7. JS rendering 결과 HTML을 Trafilatura에 넣는 hybrid extractor 구성 + +## 19. 최종 판단 + +Trafilatura는 범용 온톨로지 구축 플랫폼에서 “웹 문서를 ontology-ready document로 정제하는 핵심 엔진”으로 채택할 가치가 높다. 본문 추출 알고리즘, 메타데이터 추출, 링크 발견, 중복 제거, 구조 출력이 이미 모듈화되어 있으며 Apache-2.0 라이선스라 기본 소스 활용에도 적합하다. + +가장 좋은 통합 방향은 원본 코드를 크게 변형하지 않고, 플랫폼 내부에 얇은 adapter layer를 두는 것이다. Adapter layer는 source profile, 작업 격리, provenance 저장, 품질 점수, 온톨로지 후보 생성만 담당하고 Trafilatura의 본문/메타/URL 발견 로직은 원형에 가깝게 유지하는 편이 안정적이다. diff --git a/온톨로지플랫폼_통합설계서.md b/온톨로지플랫폼_통합설계서.md new file mode 100644 index 0000000..abc17d7 --- /dev/null +++ b/온톨로지플랫폼_통합설계서.md @@ -0,0 +1,790 @@ +# 범용 온톨로지 구축 플랫폼 통합 설계서 + +작성일: 2026-05-13 +대상: 본 문서를 받아 실제 구현을 수행할 모든 AI 에이전트 및 개발자 +근거 자료: `C:\Users\lasta\MyProject\AI\오픈소스분석자료` 의 8개 분석 명세 (Crawl4AI, Firecrawl, Guardrails, Knowledge Agent, Neo4j GraphRAG, OntoCast, OpenDeepResearcher, Trafilatura) + +--- + +## 0. 본 문서의 사용 규칙 (Agent Instruction) + +본 문서를 받는 AI 에이전트는 다음을 준수한다. + +1. 본 문서는 8개 분석 자료를 바탕으로 도출된 **최종 통합 설계**다. 개별 분석 자료와 본 문서가 충돌할 경우 본 문서가 우선한다. +2. 본 문서가 "그대로 사용"이라고 명시한 모듈은 원본 소스를 가급적 수정하지 않고 라이브러리 또는 vendored copy 형태로 도입한다. "어댑터 작성"이라고 명시한 부분만 우리 코드로 만든다. +3. "통합은 한 번에 하나씩"의 원칙을 지킨다. Phase N의 검수 게이트(Acceptance Gate)를 통과하기 전에는 Phase N+1로 진행하지 않는다. +4. 코드 작성 시 어느 분석 자료의 어느 절을 근거로 했는지 PR 설명에 명시한다. 예: `OntoCast 분석 §6 GraphUpdate 모델`. +5. 본 문서가 "제외(Excluded)"라고 명시한 프로젝트는 코드/의존성에 포함하지 않는다. + +--- + +## 1. Executive Summary (한눈에 보는 결론) + +| 항목 | 결정 | +|---|---| +| **시작 프로젝트(Base)** | **OntoCast** — RDF/온톨로지 코어 엔진, LangGraph 워크플로우, GraphUpdate 증분 갱신, Renderer/Critic 루프, Entity Aggregation, TripleStoreManager가 이미 갖추어져 있어 "온톨로지 구축 플랫폼"의 골격으로 가장 적합 | +| **통합 대상 (총 4개)** | ① Trafilatura ② Crawl4AI ③ Guardrails ④ Neo4j GraphRAG | +| **부분 차용 (코드 아닌 패턴/프롬프트)** | Knowledge Agent — LangGraph 멀티에이전트 패턴, LightRAG 추출 프롬프트 | +| **제외** | Firecrawl (TS 스택 분리 부담, Crawl4AI와 기능 중복), OpenDeepResearcher (`eval()` 보안 문제, Knowledge Agent로 대체 가능) | +| **총 통합 프로젝트 수** | **5개** (Base 1 + 통합 4) — 사용자의 "적은 수의 프로젝트" 요구 충족 | + +핵심 원칙: **OntoCast = 두뇌, Crawl4AI = 손, Trafilatura = 눈, Guardrails = 안전벨트, Neo4j GraphRAG = 거울/도서관.** + +--- + +## 2. 8개 오픈소스 강점 매트릭스 (Strength Matrix) + +각 소스가 "가장 잘하는 영역" 하나씩만 추려서 기능 중복을 정리한 표. + +| # | 프로젝트 | 카테고리 | 대체 불가 강점 | 라이선스 | 언어 | 채택 여부 | +|---|---|---|---|---|---|---| +| 1 | **OntoCast** | 온톨로지 코어 | **GraphUpdate 기반 SPARQL 증분 갱신** + Renderer/Critic retry loop + Entity aggregation(embedding + URI 정규화 + owl:sameAs) | Apache 2.0 | Python 3.12+ | **Base** | +| 2 | **Trafilatura** | 본문 추출 | 본문/메타데이터/날짜/저자/언어 추출의 **산업 표준 정밀도**. XML body tree 보존, SimHash fingerprint, feed/sitemap discovery | Apache 2.0 | Python | **통합** | +| 3 | **Crawl4AI** | 크롤링 | **동적 페이지(Playwright)+LLM 친화 Markdown** 변환의 결정판. Deep crawl(BFS/DFS/Best-First) + URL Seeder + Adaptive crawler + browser pool/dispatcher/cache | Apache 2.0 | Python 3.10+ | **통합** | +| 4 | **Guardrails** | 검증 | **Pydantic 기반 LLM 출력 강제 + on_fail 정책(reask/fix/filter/refrain) + JSON path field validator** | Apache 2.0 | Python 3.10+ | **통합** | +| 5 | **Neo4j GraphRAG** | KG 저장/검색 | **GraphSchema + GraphPruning + Neo4jWriter + EntityResolver + Vector/Hybrid/Text2Cypher Retriever + GraphRAG** | Apache 2.0 | Python 3.10+ | **통합** | +| 6 | Knowledge Agent | 멀티에이전트 | LangGraph 기반 Analyst→Researcher→Curator→Auditor→Fixer→Advisor 패턴 + LightRAG 엔티티/관계 추출 프롬프트 | 비명시 | Python | **패턴/프롬프트만 차용** | +| 7 | Firecrawl | 크롤링 API | scrape/map/crawl/search/parse API 명세 우수, fire-engine fallback | AGPL 의심 | TypeScript | **제외** (스택 분리 부담 + 라이선스 리스크 + Crawl4AI와 중복) | +| 8 | OpenDeepResearcher | 검색 루프 | LLM 기반 검색어 생성 + 자기확장 루프 + `` 판단 | MIT | Python (notebook) | **제외** (`eval()` 보안 문제 + Knowledge Agent의 Researcher가 더 구조화됨) | + +--- + +## 3. 시작 프로젝트(Base)로 OntoCast를 선택한 근거 + +| 비교 항목 | OntoCast | Knowledge Agent | Neo4j GraphRAG | +|---|---|---|---| +| 사용자 목표 적합성("온톨로지 구축") | ◎ RDF/OWL 중심 | △ 지식그래프 보조 도구 | ○ Property Graph 중심 | +| 핵심 자산의 대체 난이도 | ◎ GraphUpdate 증분 갱신은 다른 어디서도 못 구함 | △ LangGraph 패턴은 재작성 용이 | ○ Library 형태로 갖다 쓰면 됨 | +| Renderer/Critic retry loop | ◎ 내장 | △ Refiner 비슷한 개념만 | × 없음 | +| Entity Aggregation/URI 정규화 | ◎ 내장 (`tool/agg/`) | × 없음 | △ Resolver 있으나 단순 | +| LangGraph 워크플로우 | ◎ 내장 (`stategraph/`) | ◎ 내장 | × 없음 | +| Triple Store 추상화 (Fuseki/Neo4j/FS) | ◎ 내장 | × LightRAG에 종속 | △ Neo4j만 | +| ToolBox dependency container | ◎ 내장 | × | × | +| 라이선스 명확성 | ◎ Apache 2.0 | × 비명시 | ◎ Apache 2.0 | +| 코드 안정성 | ○ 일부 버그 (§13.1) | × 다수 버그 (Curator/Auditor/Fixer/Advisor) | ◎ 테스트 광범위 | + +**결론**: OntoCast의 "GraphUpdate 증분 갱신 + Renderer/Critic 루프 + Entity Aggregation"은 다른 소스로 대체 불가능한 차별 자산이다. 이를 Base로 두고 나머지를 라이브러리로 흡수한다. + +--- + +## 4. 통합 아키텍처 (Layered Architecture) + +``` +┌──────────────────────────────────────────────────────────────────┐ +│ L7. UI Layer (Ontology Studio, Review Console, Search UI) │ +│ — 새로 작성 (React/Next.js 등 자유 선택) │ +├──────────────────────────────────────────────────────────────────┤ +│ L6. Platform API (FastAPI) │ +│ — 새로 작성. /projects, /jobs, /sources, /ontology, │ +│ /review, /search, /admin │ +├──────────────────────────────────────────────────────────────────┤ +│ L5. Job Orchestration (Job Queue + Worker) │ +│ — 새로 작성. Celery/RQ/Arq 중 선택, Redis/Postgres state │ +├──────────────────────────────────────────────────────────────────┤ +│ L4. Ontology Core Engine │ +│ ★ OntoCast (Base, 거의 원형 유지) │ +│ — stategraph/, agent/, onto/, tool/, toolbox.py │ +├──────────────────────────────────────────────────────────────────┤ +│ L3. Quality & Validation Gate │ +│ ★ Guardrails (라이브러리로 통합) │ +│ — Guard.for_pydantic(OntologyExtractionResult) │ +│ — Renderer/Critic 출력 검증, reask 루프 │ +├──────────────────────────────────────────────────────────────────┤ +│ L2. Content Acquisition Layer │ +│ ★ Crawl4AI (라이브러리, 동적/대량 크롤링) │ +│ ★ Trafilatura (라이브러리, 본문/메타데이터 정밀 추출) │ +│ — Crawl4AI 우선, Trafilatura는 후처리 정밀화 옵션 │ +├──────────────────────────────────────────────────────────────────┤ +│ L1. Storage Layer │ +│ • Fuseki (Canonical RDF Store, OntoCast TripleStoreManager) │ +│ • Neo4j (Projection / Vector / Fulltext, GraphRAG) │ +│ ★ Neo4j GraphRAG (라이브러리, Projection/Retriever 담당) │ +│ • PostgreSQL (Job/User/Project/Review 메타데이터) │ +│ • Object Storage (raw HTML, PDF, screenshot 등 artifact) │ +└──────────────────────────────────────────────────────────────────┘ +``` + +### 4.1 데이터 흐름 (Document → Ontology → Search) + +``` +[Source URL/File] + → [L2 수집] Crawl4AI (동적/대량) or Trafilatura (정적/메타데이터 정밀) + → [L2 정제] Trafilatura bare_extraction(output_format="python") + metadata + → [L4 코어] OntoCast pipeline + CONVERT → CHUNK → SELECT_ONTOLOGY → + (BOOTSTRAP|RENDER_ONTOLOGY_UPDATE) → + [L3 검증] Guardrails Guard 통과 → + NORMALIZE → CONSOLIDATE → RENDER_FACTS → + [L3 검증] Guardrails Guard 통과 → + MERGE (Entity Aggregation) → SERIALIZE + → [L1 저장] + Fuseki: Canonical RDF (Ontology TTL + Facts TTL) + Neo4j: Projection via Neo4j GraphRAG KGWriter + + Chunk vector index + Lexical graph (Document/Chunk) + Postgres: Job metadata, review state + Object Storage: raw HTML/PDF + → [L6 API] /search 호출 시 Neo4j GraphRAG의 + VectorCypherRetriever + Text2CypherRetriever + GraphRAG +``` + +### 4.2 RDF ↔ Property Graph 이중 저장 근거 + +| 저장소 | 역할 | 근거 | +|---|---|---| +| **Fuseki (Canonical RDF)** | 진실 원본 (source of truth). OntoCast `GraphUpdate`가 SPARQL UPDATE로 직접 반영. OWL/SHACL 추론 가능 | OntoCast §12.3 | +| **Neo4j (Projection)** | 검색/시각화/RAG 전용. Fuseki 변경 시 비동기로 동기화. Lexical graph(Document/Chunk) + Entity 노드 + Chunk embedding property | Neo4j GraphRAG §7.8, OntoCast §12.4 | + +**중요 원칙**: RDF가 진실, Neo4j는 사본. 양쪽에 동시 쓰지 않는다. Fuseki에 commit → Worker가 Neo4j Projection 갱신. + +--- + +## 5. 단계별 통합 로드맵 (Phased Integration Plan) + +각 Phase의 마지막에 **Acceptance Gate**(검수 게이트)가 있다. Gate를 통과해야 다음 Phase 진행. + +### Phase 0: Base 안정화 (OntoCast만) + +**목표**: OntoCast를 단독으로 실행 가능한 상태로 만든다. 다른 통합은 시작하지 않는다. + +**작업**: +1. OntoCast 저장소를 vendored copy 형태로 `platform/core/ontocast/`에 배치 (Apache 2.0 license 고지 유지) +2. OntoCast 분석 §13.1에 명시된 버그 수정: + - `select_ontology.py`의 None 선택 index 불일치 수정 (analyst가 `num_ontologies + 1` 인덱스를 반환할 때 None으로 처리) + - API version 표기 불일치 정리 (`pyproject.toml`과 `/health`, `/info` 응답 통일) +3. `convert_document()`의 "one file at a time" 제약을 다중 파일 처리 가능하도록 확장 +4. Robyn 서버는 제거하고 FastAPI 로 재작성 (기존 ToolBox/AgentState/stategraph는 그대로 사용) +5. 환경설정: `.env` + `Config` (Pydantic Settings) 유지, Fuseki/filesystem/Neo4j 중 **filesystem만 활성화**하여 시작 +6. 기본 데이터셋: `data/` 폴더의 예제 PDF/JSON으로 end-to-end 1회 실행 성공 + +**Acceptance Gate 0**: +- [ ] 단일 PDF 또는 JSON 입력 → ontology TTL + facts TTL이 filesystem에 생성됨 +- [ ] `/health`, `/info`, `/process` (FastAPI 버전) 정상 동작 +- [ ] BudgetTracker가 LLM call/triple count를 정확히 기록 +- [ ] LangGraph 워크플로우(CONVERT→CHUNK→...→SERIALIZE) 전 노드가 traceable + +### Phase 1: Trafilatura 통합 (정적 본문/메타데이터 정밀화) + +**목표**: 입력이 URL일 때 원본 페이지에서 본문/제목/저자/날짜/언어/canonical URL을 정확히 뽑아 `ContentUnit` metadata에 채워 넣는다. + +**선택 이유 (Trafilatura 먼저 통합하는 이유)**: +- **가장 작은 통합**: 단일 함수(`bare_extraction`) 호출만으로 끝남. 전역 상태 외에는 의존성 거의 없음. +- 즉시 가치: OntoCast의 chunk metadata가 빈약했던 부분이 메워짐 (Trafilatura 분석 §11). +- 라이선스 동일 (Apache 2.0). + +**작업**: +1. 의존성 추가: `trafilatura[all]>=2.0.0` +2. `platform/core/extractors/web_extractor.py` 신설. Trafilatura 분석 §17의 `extract_for_ontology` 함수를 거의 그대로 가져와 어댑터로 사용 +3. OntoCast의 `ConverterTool`에 URL/HTML 입력 분기 추가: HTML이면 Trafilatura로 1차 정제 후 `text`, `Document.body` XML tree, metadata를 `ContentUnit`에 주입 +4. `ContentUnit`에 다음 필드 추가: + - `source_url` (canonical) + - `title`, `author`, `publish_date`, `language`, `sitename` + - `fingerprint` (SimHash, Trafilatura `content_fingerprint`) +5. 중복 문서 감지: `fingerprint` 기반 near-duplicate check를 OntoCast 처리 전에 수행 (이미 처리한 문서는 skip) +6. Trafilatura의 전역 LRU cache는 배치 작업 종료 시 `trafilatura.meta.reset_caches()` 호출 + +**Acceptance Gate 1**: +- [ ] URL 입력 → 본문/메타데이터가 정확히 추출되어 `ContentUnit`에 저장됨 +- [ ] 한국어 뉴스/블로그/쇼핑 페이지 각각 1개씩 본문 추출 정확도 수동 검증 +- [ ] 동일 URL 재입력 시 fingerprint 기반 dedup으로 skip +- [ ] Phase 0의 모든 기능이 여전히 정상 동작 (회귀 없음) + +### Phase 2: Crawl4AI 통합 (동적/대량 수집) + +**목표**: Trafilatura가 못 다루는 동적 페이지(JS rendering), 사이트 전체 수집(deep crawl), URL seeding을 Crawl4AI로 처리한다. + +**작업**: +1. 의존성 추가: `crawl4ai>=0.x` (분석 시점 최신 안정) +2. `platform/core/crawler/crawl4ai_adapter.py` 신설. `AsyncWebCrawler` + `CrawlerRunConfig` 팩토리 작성 +3. 수집 Profile 정의 (Crawl4AI 분석 §21.2의 권장 profile 그대로 채택): + - `fast_static`: HTTP fetch만, Trafilatura로 정제 + - `dynamic_page`: Playwright + JS wait + - `full_capture`: screenshot/PDF/MHTML + - `structured_extract`: CSS/XPath schema + - `deep_discovery`: URL Seeder + Deep crawl (BFS) +4. **결정 규칙** (어떤 Profile 선택할지): + - URL이 robots.txt에서 JS-heavy로 알려진 도메인 → `dynamic_page` + - URL이 sitemap에 등록되어 있음 → `deep_discovery` + `fast_static` + - 기본 → `fast_static` +5. Crawl4AI 결과의 `markdown` 또는 `cleaned_html` → Trafilatura 후처리 → `ContentUnit` 생성 +6. 새 API 추가: + - `POST /sources/{id}/crawl` — 사이트 단위 deep crawl 작업 시작 + - `POST /sources/{id}/seed` — URL seeding (sitemap/Common Crawl) 미리보기 + - `GET /jobs/{job_id}/progress` — 진행 상황 스트리밍 (Crawl4AI dispatcher monitor 활용) +7. 캐시: Crawl4AI의 `CacheMode.ENABLED` 기본 + `check_cache_freshness=True` + +**Acceptance Gate 2**: +- [ ] JS 렌더링이 필요한 동적 페이지 1개 정상 수집 (수동 지정) +- [ ] sitemap이 있는 사이트의 deep crawl 100페이지 이내 완료 +- [ ] URL Seeder로 후보 URL 미리보기 정상 동작 +- [ ] 메모리 누수 없이 50회 연속 크롤 가능 (browser pool 관리) +- [ ] Phase 0~1 기능 회귀 없음 + +### Phase 3: Guardrails 통합 (LLM 출력 검증 게이트) + +**목표**: OntoCast의 Renderer(`render_ontology`, `render_facts`)와 Critic(`criticise_*`)이 생성하는 LLM 응답을 Guardrails로 강제 검증한다. 잘못된 응답이 RDF graph에 들어가는 것을 차단한다. + +**작업**: +1. 의존성 추가: `guardrails-ai>=0.x` (Hub/telemetry 비활성화 설정 필수) +2. `platform/core/validation/` 신설: + - `guards.py`: `OntologyGuard` facade + - `validators.py`: 온톨로지 전용 validator (Guardrails 분석 §15.4 그대로 채택) +3. Pydantic 모델 정의 (Guardrails 분석 §15.5): + ```python + class OntologyEntity(BaseModel): + id: str + label: str + type: Literal["class","individual","object_property","data_property"] + description: str | None = None + aliases: list[str] = [] + evidence: list[OntologyEvidence] = [] + confidence: float + + class OntologyRelation(BaseModel): + id: str + source_id: str + predicate: str + target_id: str + evidence: list[OntologyEvidence] = [] + confidence: float + + class OntologyExtractionResult(BaseModel): + entities: list[OntologyEntity] + relations: list[OntologyRelation] + warnings: list[str] = [] + ``` +4. OntoCast의 LLM 호출부 (`tool/llm.py`)를 래핑: + - Ontology Renderer 출력 → `Guard.for_pydantic(OntologyDelta)`로 검증 + - Facts Renderer 출력 → `Guard.for_pydantic(FactsDelta)`로 검증 + - `num_reasks=1`, `full_schema_reask=False` (부분 reask 우선) +5. Validator 활성화 (Guardrails 분석 §15.4): + - `EntityIdFormatValidator` (fix) + - `UniqueEntityIdValidator` (reask) + - `RelationEndpointExistsValidator` (reask) + - `ConfidenceRangeValidator` (fix) + - `EvidenceExistsValidator` (filter) + - `NoSelfRelationValidator` (filter) +6. Guard 실패 시 OntoCast의 critic suggestions에 추가하여 다음 retry에 반영 + +**Acceptance Gate 3**: +- [ ] LLM이 의도적으로 스키마를 위반한 응답(예: confidence > 1.0) → Guardrails가 자동 fix +- [ ] 존재하지 않는 entity ID를 참조한 relation → Guardrails가 reask 또는 filter +- [ ] reask 횟수가 `num_reasks` 한도 안에서 종료, 무한 루프 없음 +- [ ] Guardrails Hub/telemetry 의존성이 비활성화되어 외부 통신 없음 +- [ ] Phase 0~2 기능 회귀 없음 + +### Phase 4: Neo4j GraphRAG 통합 (Projection + 검색/RAG) + +**목표**: Fuseki에 commit된 RDF를 Neo4j Property Graph로 projection하고, Neo4j GraphRAG의 Retriever/GraphRAG로 검색/QA 기능을 추가한다. + +**작업**: +1. 의존성 추가: `neo4j-graphrag[openai,experimental]==1.16.0` (버전 고정) +2. APOC core 설치된 Neo4j 5.18.1+ 배포 (Docker compose 추가) +3. `platform/core/projection/rdf_to_neo4j.py` 신설: + - Fuseki의 `Ontology` + `Facts` graph → `Neo4jGraph(nodes, relationships)` 변환 + - `KGWriter`로 `Neo4jWriter` 사용해 upsert + - Lexical graph(Document/Chunk) + Entity 노드를 `LexicalGraphBuilder` 표준에 맞춰 생성 +4. Chunk embedding: + - OntoCast의 청킹 결과(`ContentUnit.text`)에 `OpenAIEmbeddings` 또는 `SentenceTransformerEmbeddings` 적용 + - Neo4j chunk vector index 자동 생성 (`indexes.py`) +5. 검색 API 신설: + - `POST /search/vector` → `VectorRetriever` + - `POST /search/hybrid` → `HybridRetriever` + - `POST /search/text2cypher` → `Text2CypherRetriever` (read-only 강제) + - `POST /search/graphrag` → `GraphRAG` 답변 생성 +6. Text2Cypher 보안 (Neo4j GraphRAG 분석 §13.4): + - read-only 검사 + 허용 schema 제한 + query timeout + result limit +7. Entity Resolver: + - `SinglePropertyExactMatchResolver` 기본 활성화 + - OntoCast Entity Aggregation 결과(`owl:sameAs`)와 함께 사용해 중복 병합 + +**Acceptance Gate 4**: +- [ ] Fuseki commit 후 5초 이내 Neo4j projection 동기화 완료 +- [ ] Lexical graph(Document/Chunk/Entity)와 provenance(`FROM_CHUNK`) 정상 생성 +- [ ] Vector 검색 결과의 chunk → entity provenance 추적 가능 +- [ ] Text2Cypher가 write/delete 쿼리를 차단 +- [ ] GraphRAG 답변에 evidence chunk URL 포함 +- [ ] Phase 0~3 기능 회귀 없음 + +### Phase 5: Knowledge Agent 패턴 차용 (멀티 에이전트 워크플로우) + +**목표**: Knowledge Agent의 Analyst→Researcher→Curator→Auditor→Fixer→Advisor 패턴을 OntoCast의 기존 LangGraph에 통합하여 "유지보수 루프"를 추가한다. **코드를 통째로 가져오지 않고 LangGraph 노드 정의와 프롬프트만 차용한다** (Knowledge Agent 코드에 다수의 버그 존재 — 분석 §9.1). + +**작업**: +1. OntoCast의 기존 `stategraph/`에 신규 노드 추가: + - `AnalystNode`: 현재 ontology + facts graph의 지식 공백 식별 + - `ResearcherNode`: 공백별 검색 계획 생성 → Crawl4AI search/seed 호출 + - `CuratorNode`: 수집된 URL 평가 → 선별 URL을 OntoCast `/process`로 투입 + - `AuditorNode`: graph 품질 감사 (Neo4j GraphRAG의 schema validation 활용) + - `FixerNode`: 감사 이슈 수정 (사람 승인 게이트 필수) + - `AdvisorNode`: 반복 문제 분석 + 시스템 개선 제안 +2. 프롬프트는 Knowledge Agent의 `prompts/*.txt`를 base로 시작하되, OntoCast의 `OntologyExtractionResult` 스키마에 맞게 수정 +3. LightRAG 엔티티/관계 타입(Knowledge Agent §7.1, §7.2)을 **프로젝트별 설정으로 분리**. 기본 타입 팩 + 사용자 정의 타입 팩 지원 +4. 사람 승인 게이트(Fixer): + - 자동 승인: confidence ≥ 0.95 + non-destructive + - 검토 큐: 그 외 모두 → Review UI에서 승인/반려 +5. 새 API: + - `POST /projects/{id}/maintenance/run` — 전체 유지보수 루프 + - `POST /projects/{id}/maintenance/analyze` — 분석만 + - `POST /projects/{id}/maintenance/fix` — 수정만 + +**Acceptance Gate 5**: +- [ ] 의도적으로 만든 지식 공백(특정 entity의 predicate 누락) → Analyst가 식별 → Researcher가 검색 수행 → Curator가 URL 선별 → 신규 facts 추가 +- [ ] Auditor가 중복 entity 후보를 정확히 식별 (Neo4j GraphRAG resolver 활용) +- [ ] Fixer가 destructive 변경 시 사람 승인 없이 진행하지 않음 +- [ ] Advisor 보고서에 "반복 실패한 entity type/predicate" 통계 포함 +- [ ] Phase 0~4 기능 회귀 없음 + +--- + +## 6. 상용 제품 수준 기능 명세 (Functional Spec) + +기능 ID 체계: `[영역코드]-[순번]`. 영역코드: PROJ, ACQ, ONT, REV, SRCH, OPS, GOV. + +### 6.1 프로젝트/멀티테넌트 관리 (PROJ) + +| ID | 기능명 | 설명 | 우선순위 | +|---|---|---|---| +| PROJ-001 | 온톨로지 프로젝트 생성 | 이름, 도메인, 기본 언어, 기본 타입 팩, Neo4j DB/Fuseki dataset 매핑 | P0 | +| PROJ-002 | 멤버/권한 관리 | Owner/Admin/Editor/Reviewer/Viewer 역할 | P0 | +| PROJ-003 | LLM Profile 선택 | provider/model/api_key/temperature/max_tokens (Neo4j GraphRAG `LLMConfig` 차용) | P0 | +| PROJ-004 | Embedding Profile 선택 | provider/model/dimension (cohere/openai/sentence-transformers) | P0 | +| PROJ-005 | 비용 예산 설정 | LLM call/token/검색 호출 일일·월간 한도 (OntoCast BudgetTracker 확장) | P1 | +| PROJ-006 | 프로젝트 fork/복제 | 기존 프로젝트의 스키마/설정만 복제 | P2 | +| PROJ-007 | 프로젝트 export/import | 전체 RDF + 설정을 zip으로 export, import | P1 | + +### 6.2 데이터 수집 (ACQ) + +| ID | 기능명 | 사용 컴포넌트 | 우선순위 | +|---|---|---|---| +| ACQ-001 | 단일 URL scrape | Crawl4AI `fast_static` + Trafilatura | P0 | +| ACQ-002 | 동적 페이지 scrape | Crawl4AI `dynamic_page` | P0 | +| ACQ-003 | 사이트 deep crawl | Crawl4AI `deep_discovery` (BFS/DFS/Best-First) | P1 | +| ACQ-004 | sitemap/feed 발견 | Trafilatura `sitemap_search`, `find_feed_urls` | P1 | +| ACQ-005 | URL seeding 미리보기 | Crawl4AI `AsyncUrlSeeder` | P1 | +| ACQ-006 | 파일 업로드 (PDF/DOCX/MD) | OntoCast converter + docling | P0 | +| ACQ-007 | 배치 URL import (CSV) | URL 목록 업로드 → 큐잉 | P1 | +| ACQ-008 | 중복 문서 dedup | Trafilatura SimHash fingerprint | P0 | +| ACQ-009 | 도메인 allow/block list | 프로젝트별 정책 DB | P0 | +| ACQ-010 | robots.txt 준수 모드 | strict/respect/ignore (감사 로그 필수) | P0 | +| ACQ-011 | 변경 감지 (changeTracking) | 주기 재크롤 + fingerprint 비교 | P2 | +| ACQ-012 | screenshot/PDF archive | Crawl4AI `full_capture` Profile | P2 | + +### 6.3 온톨로지 구축 (ONT) + +| ID | 기능명 | 사용 컴포넌트 | 우선순위 | +|---|---|---|---| +| ONT-001 | 수동 스키마 작성 | Neo4j GraphRAG `GraphSchema` 모델 그대로 사용 + UI | P0 | +| ONT-002 | 자동 스키마 추출 | Neo4j GraphRAG `SchemaFromTextExtractor` 또는 OntoCast `BOOTSTRAP_ONTOLOGY` | P0 | +| ONT-003 | 자유 추출 | OntoCast `schema="FREE"` 모드 | P1 | +| ONT-004 | 온톨로지 증분 갱신 | OntoCast `GraphUpdate` SPARQL (핵심 차별 기능) | P0 | +| ONT-005 | 온톨로지 비평/재시도 | OntoCast Critic loop + Guardrails | P0 | +| ONT-006 | 사실(facts) 추출 | OntoCast `RENDER_FACTS` + Guardrails | P0 | +| ONT-007 | Entity Aggregation | OntoCast `tool/agg/` (embedding clustering + URI 정규화 + owl:sameAs) | P0 | +| ONT-008 | Entity Resolver (사후) | Neo4j GraphRAG `FuzzyMatchResolver` + 검수 큐 | P1 | +| ONT-009 | 스키마 버전 관리 | OntoCast `GraphVersionManager` + draft/published/deprecated | P1 | +| ONT-010 | 스키마 diff 뷰어 | 버전 간 class/property/pattern 변경 | P1 | +| ONT-011 | 스키마 마이그레이션 | rename/merge/split + facts auto-migration | P2 | +| ONT-012 | SHACL/OWL 검증 | rdflib + owlready2 | P2 | +| ONT-013 | 다국어 label 관리 | rdfs:label + lang tag (ko/en/...) | P1 | + +### 6.4 검수/승인 (REV) + +| ID | 기능명 | 설명 | 우선순위 | +|---|---|---|---| +| REV-001 | 추출 후보 큐 | Renderer 결과를 Fuseki commit 전 검토용으로 저장 | P0 | +| REV-002 | Evidence 하이라이트 | entity/relation → `FROM_CHUNK` → 원문 텍스트 표시 (Neo4j GraphRAG lexical graph) | P0 | +| REV-003 | 단위 승인/반려 | node/relationship/property 단위 | P0 | +| REV-004 | Pruned 후보 검토 | Neo4j GraphRAG `GraphPruning`에서 제거된 후보를 schema 후보로 제안 | P1 | +| REV-005 | Merge 후보 검토 | Resolver 후보 그룹의 시각화 + 승인/반려 | P1 | +| REV-006 | 일괄 승인 정책 | confidence ≥ X + source 신뢰도 ≥ Y → 자동 승인 | P1 | +| REV-007 | 변경 이력 (audit) | 누가/언제/무엇을/왜 변경 (RDF reification 또는 별도 audit log) | P0 | +| REV-008 | 롤백 | 특정 시점의 RDF graph로 복원 | P2 | + +### 6.5 검색/RAG (SRCH) + +| ID | 기능명 | 사용 컴포넌트 | 우선순위 | +|---|---|---|---| +| SRCH-001 | Vector 검색 | Neo4j GraphRAG `VectorRetriever` | P0 | +| SRCH-002 | Hybrid 검색 (vector+fulltext) | `HybridRetriever` | P0 | +| SRCH-003 | Graph 확장 검색 | `VectorCypherRetriever` (chunk → entity → neighbor) | P0 | +| SRCH-004 | Text2Cypher (NL → Cypher) | `Text2CypherRetriever` (read-only) | P1 | +| SRCH-005 | SPARQL 직접 질의 | Fuseki SPARQL endpoint (관리자 권한) | P1 | +| SRCH-006 | GraphRAG QA | `GraphRAG` + evidence URL 포함 | P0 | +| SRCH-007 | Faceted 탐색 | entity type/predicate별 filter | P1 | +| SRCH-008 | Entity 상세 페이지 | 모든 property + 인입/인출 relation + evidence | P0 | +| SRCH-009 | Subgraph 시각화 | Neo4j Browser embed 또는 Cytoscape.js | P1 | +| SRCH-010 | 저장된 질의 (saved query) | 즐겨찾기 + 알림 | P2 | + +### 6.6 운영/관측 (OPS) + +| ID | 기능명 | 설명 | 우선순위 | +|---|---|---|---| +| OPS-001 | Job Queue/Worker | 모든 비동기 작업의 큐잉/재시도/취소 | P0 | +| OPS-002 | Job 진행률 스트리밍 | WebSocket/SSE로 실시간 진행 | P0 | +| OPS-003 | BudgetTracker | LLM call/token/검색/크롤 호출 비용 추적 | P0 | +| OPS-004 | LLM Response Cache | OntoCast `Cacher` 그대로 사용 | P0 | +| OPS-005 | Prometheus metrics | Crawl4AI 분석 §16.4 패턴 | P1 | +| OPS-006 | 감사 로그 | 모든 RDF 변경 + 사용자 액션 | P0 | +| OPS-007 | 에러 알림 | webhook + email + slack | P1 | +| OPS-008 | 백업/복구 | Fuseki + Neo4j + Postgres 일관성 있는 백업 | P1 | +| OPS-009 | 멀티 환경 | dev/staging/prod 분리 | P0 | + +### 6.7 거버넌스/보안 (GOV) + +| ID | 기능명 | 설명 | 우선순위 | +|---|---|---|---| +| GOV-001 | 인증 (OAuth2/OIDC) | Google/GitHub/Azure AD | P0 | +| GOV-002 | RBAC | 프로젝트별 역할 (PROJ-002) | P0 | +| GOV-003 | API Key | 외부 시스템 연동용 | P0 | +| GOV-004 | Rate Limiting | 사용자별/IP별 | P0 | +| GOV-005 | Text2Cypher 샌드박스 | read-only + timeout + result limit + allowlist | P0 | +| GOV-006 | 비밀(secret) vault | LLM API key 암호화 저장 | P0 | +| GOV-007 | 데이터 보존 정책 | raw HTML/PDF 보존 기간 + zero-retention 모드 | P1 | +| GOV-008 | 라이선스/출처 표기 | source별 license metadata 저장 + 결과에 노출 | P1 | + +--- + +## 7. 데이터 모델 표준 (Canonical Data Models) + +본 절은 5개 소스를 잇기 위한 공통 데이터 계약이다. 모든 어댑터는 이 모델로 변환한다. + +### 7.1 ContentUnit (문서 청크의 표준 표현) + +```python +class ContentUnit(BaseModel): + id: str # UUID + project_id: str + source_id: str + source_url: str | None # canonical URL (Trafilatura) + file_path: str | None + document_type: Literal["html","pdf","markdown","docx","inline_text"] + + text: str # 정제된 본문 + body_xml: bytes | None # Trafilatura Document.body (lxml serialized) + markdown: str | None # Crawl4AI markdown 또는 변환본 + + chunk_index: int + total_chunks: int + + title: str | None + author: str | None + publish_date: str | None # ISO-8601 + language: str | None + sitename: str | None + + fingerprint: str | None # Trafilatura SimHash + content_hash: str # SHA-256 of text + + metadata: dict # raw provider metadata + retrieved_at: str # ISO-8601 + extracted_by: str # "crawl4ai+trafilatura" +``` + +### 7.2 OntologyExtractionResult (Guardrails 검증 대상) + +Guardrails §15.5 그대로 채택. 위 §5 Phase 3 참조. + +### 7.3 GraphUpdate (OntoCast 그대로) + +OntoCast `sparql_models.GraphUpdate`를 그대로 사용. 변경 금지. + +### 7.4 Job + +```python +class Job(BaseModel): + id: str + project_id: str + type: Literal["scrape","crawl","extract","project","resolve","maintenance"] + status: Literal["queued","running","paused","completed","failed","cancelled"] + progress: float # 0.0 ~ 1.0 + + input: dict # 작업 입력 (URL, options, etc.) + output: dict | None # 결과 요약 + + started_at: str | None + finished_at: str | None + + budget: BudgetTracker # OntoCast 그대로 + error: str | None + audit_log_id: str +``` + +### 7.5 ReviewItem + +```python +class ReviewItem(BaseModel): + id: str + project_id: str + job_id: str + + target_type: Literal["entity","relation","property","merge_group","prune_candidate"] + target_data: dict # 후보 데이터 + evidence: list[dict] # chunk_id + text span + + confidence: float + source_trust: float + + status: Literal["pending","approved","rejected","auto_approved"] + decided_by: str | None # user_id + decided_at: str | None + reason: str | None +``` + +--- + +## 8. API 표준 (RESTful, 일부 WebSocket) + +전체는 OpenAPI 3.1 spec으로 별도 관리. 여기서는 핵심 endpoint만 명시. + +| Method | Path | 설명 | 인용 근거 | +|---|---|---|---| +| POST | `/projects` | 프로젝트 생성 | PROJ-001 | +| GET | `/projects/{id}` | 프로젝트 조회 | - | +| POST | `/projects/{id}/sources` | 데이터 소스 등록 | ACQ | +| POST | `/projects/{id}/sources/{sid}/scrape` | 단일 URL scrape | ACQ-001 | +| POST | `/projects/{id}/sources/{sid}/crawl` | deep crawl | ACQ-003 | +| POST | `/projects/{id}/sources/{sid}/seed` | URL seeding preview | ACQ-005 | +| POST | `/projects/{id}/upload` | 파일 업로드 | ACQ-006 | +| GET | `/projects/{id}/schemas` | 스키마 목록 | ONT-001 | +| POST | `/projects/{id}/schemas` | 스키마 작성 | ONT-001 | +| POST | `/projects/{id}/schemas/extract` | 자동 스키마 추출 | ONT-002 | +| POST | `/projects/{id}/schemas/{sid}/publish` | 스키마 발행 | ONT-009 | +| POST | `/projects/{id}/process` | 문서 처리 (전체 OntoCast 워크플로우) | OntoCast §13.3 | +| GET | `/jobs/{job_id}` | Job 조회 | OPS-001 | +| GET | `/jobs/{job_id}/progress` (WS) | 진행률 스트리밍 | OPS-002 | +| POST | `/jobs/{job_id}/cancel` | Job 취소 | OPS-001 | +| GET | `/projects/{id}/review` | 검수 큐 | REV-001 | +| POST | `/projects/{id}/review/{rid}/approve` | 승인 | REV-003 | +| POST | `/projects/{id}/review/{rid}/reject` | 반려 | REV-003 | +| POST | `/projects/{id}/search/vector` | Vector 검색 | SRCH-001 | +| POST | `/projects/{id}/search/hybrid` | Hybrid 검색 | SRCH-002 | +| POST | `/projects/{id}/search/text2cypher` | NL → Cypher | SRCH-004 | +| POST | `/projects/{id}/search/graphrag` | QA | SRCH-006 | +| GET | `/projects/{id}/entities/{eid}` | Entity 상세 | SRCH-008 | +| POST | `/projects/{id}/maintenance/run` | 유지보수 루프 | Phase 5 | + +--- + +## 9. 어떤 코드를 어디서 가져오는가 (Module Map) + +각 5개 소스에서 가져올 모듈을 정확히 명시. **"그대로"=수정 금지, "어댑터"=얇은 래퍼만 작성**. + +### 9.1 OntoCast (Base) + +| 가져올 모듈 | 방식 | 수정 사항 | +|---|---|---| +| `ontocast/onto/state.py` (AgentState) | 그대로 | - | +| `ontocast/onto/unit_states.py` | 그대로 | - | +| `ontocast/onto/sparql_models.py` (GraphUpdate) | 그대로 | - | +| `ontocast/onto/rdfgraph.py` | 그대로 | - | +| `ontocast/onto/ontology.py` | 그대로 | - | +| `ontocast/stategraph/` | 그대로 | Phase 5에서 노드 추가만 | +| `ontocast/agent/render_*.py` | 그대로 | - | +| `ontocast/agent/criticise_*.py` | 그대로 | - | +| `ontocast/agent/select_ontology.py` | **버그 수정** | None index 불일치 (분석 §13.1) | +| `ontocast/agent/convert_document.py` | **확장** | 다중 파일 처리 (분석 §13.1) | +| `ontocast/tool/agg/` | 그대로 | - | +| `ontocast/tool/triple_manager/` | 그대로 | - | +| `ontocast/tool/llm.py` | **래핑** | Guardrails Guard 통과 (Phase 3) | +| `ontocast/tool/cache.py` | 그대로 | - | +| `ontocast/toolbox.py` | 그대로 | - | +| `ontocast/cli/serve.py` (Robyn) | **재작성** | FastAPI로 (Phase 0) | + +### 9.2 Trafilatura + +| 가져올 함수 | 방식 | +|---|---| +| `bare_extraction(output_format="python")` | 그대로 import | +| `extract_metadata` | 그대로 import | +| `sitemap_search` | 그대로 import | +| `find_feed_urls` | 그대로 import | +| `content_fingerprint`, `Simhash` | 그대로 import | +| `trafilatura.meta.reset_caches` | 그대로 import (배치 종료 시 호출) | + +신규 어댑터: `platform/core/extractors/web_extractor.py` (§17 Trafilatura 분석의 `extract_for_ontology` 그대로) + +### 9.3 Crawl4AI + +| 가져올 클래스 | 방식 | +|---|---| +| `AsyncWebCrawler` | 그대로 | +| `BrowserConfig`, `CrawlerRunConfig`, `CacheMode` | 그대로 | +| `CrawlResult` | 그대로 | +| `LLMExtractionStrategy`, `JsonCssExtractionStrategy` | 그대로 | +| `BFSDeepCrawlStrategy`, `BestFirstCrawlingStrategy` | 그대로 | +| `AsyncUrlSeeder`, `SeedingConfig` | 그대로 | +| `MemoryAdaptiveDispatcher` | 그대로 | +| Docker FastAPI 서버 코드 | **사용 안 함** (자체 FastAPI 사용) | + +신규 어댑터: `platform/core/crawler/crawl4ai_adapter.py` (Crawl4AI 분석 §21.1의 권장 계층 구조) + +### 9.4 Guardrails + +| 가져올 모듈 | 방식 | +|---|---| +| `guardrails.Guard.for_pydantic` | 그대로 | +| `guardrails.AsyncGuard` | 그대로 | +| `guardrails.classes.validation_outcome.ValidationOutcome` | 그대로 | +| `guardrails.actions.*` | 그대로 | +| `guardrails.types.on_fail.OnFailAction` | 그대로 | +| `guardrails.validator_base.Validator` (커스텀 validator 작성용 base) | 그대로 | +| `guardrails.hub.*` | **사용 안 함** (외부 통신 차단) | +| `guardrails.telemetry.*` | **사용 안 함** | +| `guardrails.cli.*` | **사용 안 함** | + +신규 작성: `platform/core/validation/validators.py` (Guardrails 분석 §15.4의 12개 validator) + +### 9.5 Neo4j GraphRAG + +| 가져올 클래스 | 방식 | +|---|---| +| `GraphSchema`, `NodeType`, `RelationshipType`, `Pattern`, `ConstraintType` | 그대로 | +| `SimpleKGPipeline` | **사용 안 함** (OntoCast 워크플로우가 우선) | +| `LLMEntityRelationExtractor` | **사용 안 함** (OntoCast Renderer가 우선) | +| `Neo4jWriter`, `Neo4jGraph`, `Neo4jNode`, `Neo4jRelationship` | 그대로 (Projection 용) | +| `LexicalGraphBuilder` | 그대로 | +| `SinglePropertyExactMatchResolver`, `FuzzyMatchResolver` | 그대로 | +| `VectorRetriever`, `HybridRetriever`, `VectorCypherRetriever` | 그대로 | +| `Text2CypherRetriever` | 그대로 (read-only 강제) | +| `GraphRAG` | 그대로 | +| `embeddings/*`, `llm/*` | 그대로 (선택적) | + +신규 어댑터: `platform/core/projection/rdf_to_neo4j.py` (Fuseki RDF → `Neo4jGraph` 변환) + +### 9.6 Knowledge Agent (패턴/프롬프트만) + +| 가져올 자산 | 방식 | +|---|---| +| `prompts/analyst_prompt.txt` | **수정 후 사용** (OntoCast `OntologyExtractionResult` 스키마에 맞춤) | +| `prompts/planner_prompt.txt` | 수정 후 사용 | +| `prompts/refiner_prompt.txt` | 수정 후 사용 | +| `prompts/summarizer_prompt.txt` | 수정 후 사용 | +| `prompts/search_ranker_prompt.txt` | 수정 후 사용 | +| `prompts/ingester_prompt.txt` | 수정 후 사용 | +| `lightrag/prompt.py` (엔티티/관계 추출 프롬프트) | **참고만** (OntoCast Renderer가 이미 있음) | +| 코드 전체 | **사용 안 함** (다수 버그 — 분석 §9.1) | + +--- + +## 10. 예상 리스크와 대응 + +| 리스크 | 영향 | 대응 | +|---|---|---| +| OntoCast `select_ontology.py` 버그 | 워크플로우 실패 | Phase 0에서 즉시 수정 | +| Crawl4AI/Playwright 메모리 누수 | 운영 장애 | `max_pages_before_recycle` 설정 + browser pool monitor | +| Trafilatura 전역 LRU cache 충돌 | 다중 테넌트에서 결과 오염 | 작업 단위 `reset_caches()` | +| Guardrails Hub 외부 통신 | 보안/네트워크 의존 | Hub/telemetry 비활성화 환경변수 강제 | +| Neo4j GraphRAG `experimental` API 변경 | upstream 호환성 | 버전 고정 (`==1.16.0`) + 우리 코드는 어댑터로만 접근 | +| Fuseki ↔ Neo4j 동기화 지연 | 검색 결과와 진실 불일치 | "Last sync at" 표시 + 강제 동기화 API | +| LLM 비용 폭주 | 운영 비용 | OPS-003 BudgetTracker 한도 + 자동 차단 | +| Text2Cypher 인젝션 | 보안 | read-only 강제 + allowlist + timeout (GOV-005) | +| robots.txt 위반 | 법적 리스크 | ACQ-010 strict 모드 기본 + 감사 로그 | +| 라이선스 (특히 AGPL 회피) | 배포 제약 | Apache/MIT만 채택. Firecrawl 제외 결정 근거 | + +--- + +## 11. 기술 스택 요약 (전체) + +| 영역 | 선택 | +|---|---| +| 언어 | Python 3.12+ (OntoCast 요구사항이 가장 높음) | +| API | FastAPI (모든 통합 소스가 OpenAPI 친화) | +| LangGraph 워크플로우 | OntoCast 기존 사용 | +| Validation | Pydantic v2 + Guardrails | +| 비동기 | asyncio (Crawl4AI/Trafilatura 모두 지원) | +| Job Queue | Arq (Redis 기반, 가벼움) 또는 Celery (대규모) | +| 메타데이터 DB | PostgreSQL 16+ | +| Canonical RDF Store | Apache Jena Fuseki 5+ | +| Property Graph | Neo4j 5.18.1+ with APOC core | +| Object Storage | S3 호환 (MinIO 로컬, AWS S3 운영) | +| 캐시 | Redis 7+ | +| 관측 | OpenTelemetry + Prometheus + Grafana | +| 컨테이너 | Docker Compose (개발) / Kubernetes (운영) | +| 인증 | Authlib + OAuth2/OIDC | +| 프론트엔드 | (자유 선택, 권장 Next.js 14 + shadcn/ui) | + +--- + +## 12. 작업 단위 분해 (다른 AI 에이전트가 받아 작업할 단위) + +각 Phase 내부에서 PR 단위로 쪼갠 예시. 에이전트는 이 순서로 작업한다. + +### Phase 0 작업 단위 +- **0.1** OntoCast vendored copy + Apache 2.0 NOTICE 추가 +- **0.2** `select_ontology.py` 버그 수정 + 회귀 테스트 +- **0.3** `convert_document.py` 다중 파일 처리 확장 +- **0.4** Robyn → FastAPI 재작성 (`/health`, `/info`, `/process`, `/flush`) +- **0.5** Pydantic Settings 기반 `Config` 정리 (filesystem 모드만 활성) +- **0.6** End-to-end 통합 테스트 (예제 PDF 1개 → ontology TTL + facts TTL) +- **0.7** Acceptance Gate 0 체크리스트 확인 + +### Phase 1 작업 단위 +- **1.1** `trafilatura[all]` 의존성 추가 +- **1.2** `web_extractor.py` 어댑터 작성 (Trafilatura §17 인용) +- **1.3** `ContentUnit` 모델 확장 (title/author/date/fingerprint 등) +- **1.4** OntoCast `ConverterTool` 분기 추가 (URL/HTML 입력) +- **1.5** Fingerprint 기반 dedup 로직 +- **1.6** 한국어 페이지 3종 추출 검증 +- **1.7** Acceptance Gate 1 체크리스트 확인 + +### Phase 2 작업 단위 +- **2.1** `crawl4ai` 의존성 + Playwright 설치 +- **2.2** `crawl4ai_adapter.py` 작성 + 5개 Profile 정의 +- **2.3** Crawl Profile 결정 규칙 구현 +- **2.4** `/sources/{id}/crawl`, `/seed` API 추가 +- **2.5** Job Queue 통합 (Arq 또는 Celery) +- **2.6** WebSocket 진행률 스트리밍 +- **2.7** 메모리 누수 stress test +- **2.8** Acceptance Gate 2 체크리스트 확인 + +### Phase 3 작업 단위 +- **3.1** `guardrails-ai` 의존성 + Hub/telemetry 비활성화 설정 +- **3.2** `OntologyExtractionResult`/`OntologyDelta`/`FactsDelta` Pydantic 모델 +- **3.3** 12개 온톨로지 validator 작성 (Guardrails §15.4) +- **3.4** OntoCast `tool/llm.py` 래핑 (Renderer 호출에 Guard 적용) +- **3.5** reask 결과 → critic suggestions 반영 통합 +- **3.6** 실패 시나리오 테스트 (스키마 위반 응답 자동 차단) +- **3.7** Acceptance Gate 3 체크리스트 확인 + +### Phase 4 작업 단위 +- **4.1** `neo4j-graphrag==1.16.0` + Neo4j Docker compose 추가 +- **4.2** APOC 설치 검증 스크립트 +- **4.3** `rdf_to_neo4j.py` projection 어댑터 +- **4.4** Chunk embedding 파이프라인 +- **4.5** Vector/Hybrid/Text2Cypher/GraphRAG endpoint +- **4.6** Text2Cypher 보안 (read-only + allowlist) +- **4.7** Fuseki commit hook → Neo4j 동기화 +- **4.8** Acceptance Gate 4 체크리스트 확인 + +### Phase 5 작업 단위 +- **5.1** Knowledge Agent 프롬프트 6종 import + OntoCast 스키마 맞춤 수정 +- **5.2** LangGraph 신규 노드 6개 (Analyst/Researcher/Curator/Auditor/Fixer/Advisor) +- **5.3** 사람 승인 게이트 (Fixer destructive 동작) +- **5.4** Maintenance API 3종 +- **5.5** Advisor 통계 보고서 생성 +- **5.6** Acceptance Gate 5 체크리스트 확인 + +--- + +## 13. 다른 AI 에이전트를 위한 체크리스트 + +본 설계서를 받은 AI 에이전트가 작업을 시작하기 전 확인할 항목. + +- [ ] 본 문서의 §1~§12를 모두 읽었는가? +- [ ] 8개 분석 자료 중 본 Phase에 해당하는 것을 정독했는가? (해당 분석 자료 경로: `C:\Users\lasta\MyProject\AI\오픈소스분석자료\`) +- [ ] 작업할 Phase의 Acceptance Gate를 명확히 이해했는가? +- [ ] "그대로 사용" 모듈을 수정하려 하고 있지 않은가? +- [ ] "제외(Excluded)" 프로젝트의 코드를 가져오려 하고 있지 않은가? (Firecrawl, OpenDeepResearcher) +- [ ] PR 설명에 어느 분석 자료의 어느 절을 근거로 했는지 명시할 준비가 되었는가? +- [ ] 이전 Phase의 회귀 테스트가 통과하는지 확인할 계획이 있는가? +- [ ] License 고지(Apache 2.0 NOTICE)가 vendored copy에 포함되는가? + +--- + +## 14. 마지막 한 마디 + +이 설계의 본질은 다음 한 문장으로 요약된다. + +> **"OntoCast의 RDF 증분 갱신 두뇌에, Crawl4AI/Trafilatura의 수집 손과 눈, Guardrails의 안전벨트, Neo4j GraphRAG의 검색 도서관을 붙인다."** + +각 소스의 가장 잘하는 부분만 가져오고, 나머지는 과감히 버린다. 통합은 한 번에 하나씩. 사용자가 강조한 "버그 가능성 최소화"는 이 원칙을 지키는 것에서 시작한다. diff --git a/참고/crawl4ai-main.bat b/참고/crawl4ai-main.bat new file mode 100644 index 0000000..05c90d6 --- /dev/null +++ b/참고/crawl4ai-main.bat @@ -0,0 +1,4 @@ +cd C:\Users\lasta\MyProject\AI\참고\crawl4ai-main +python -m venv .venv +.venv\Scripts\activate +pip install -r requirements.txt \ No newline at end of file diff --git a/참고/instructor-main/.coveragerc b/참고/instructor-main/.coveragerc deleted file mode 100644 index 1c329af..0000000 --- a/참고/instructor-main/.coveragerc +++ /dev/null @@ -1,5 +0,0 @@ -[run] -source = - instructor/ -omit = - instructor/cli/* diff --git a/참고/instructor-main/.cursor/rules/documentation-sync.mdc b/참고/instructor-main/.cursor/rules/documentation-sync.mdc deleted file mode 100644 index 9817125..0000000 --- a/참고/instructor-main/.cursor/rules/documentation-sync.mdc +++ /dev/null @@ -1,36 +0,0 @@ ---- -description: when making code changes or adding documentation -globs: ["*.py", "*.md"] -alwaysApply: true ---- - -- When making code changes: - - Update related documentation files to reflect the changes - - Check docstrings and type hints are up to date - - Update any example code in markdown files - - Review README.md if the changes affect installation or usage - -- When creating new markdown files: - - Add the file to mkdocs.yml under the appropriate section - - Follow the existing hierarchy and indentation - - Use descriptive nav titles - - Example: - ```yaml - nav: - - Home: index.md - - Guides: - - Getting Started: guides/getting-started.md - - Your New File: guides/your-new-file.md - ``` - -- For API documentation: - - Ensure new functions/classes are documented - - Include type hints and docstrings - - Add usage examples - - Update API reference docs if auto-generated - -- Documentation Quality: - - Write at grade 10 reading level (see simple-language.mdc) - - Include working code examples - - Add links to related documentation - - Use consistent formatting and style \ No newline at end of file diff --git a/참고/instructor-main/.cursor/rules/followups.mdc b/참고/instructor-main/.cursor/rules/followups.mdc deleted file mode 100644 index 29b9a9f..0000000 --- a/참고/instructor-main/.cursor/rules/followups.mdc +++ /dev/null @@ -1,8 +0,0 @@ ---- -description: when AI agents are collaborating on code -globs: "*" -alwaysApply: true ---- -Make sure to come up with follow-up hot keys. They should be thoughtful and actionable and result in small additional code changes based on the context that you have available. - -using [J], [K], [L] diff --git a/참고/instructor-main/.cursor/rules/new-features-planning.mdc b/참고/instructor-main/.cursor/rules/new-features-planning.mdc deleted file mode 100644 index 3ab5e75..0000000 --- a/참고/instructor-main/.cursor/rules/new-features-planning.mdc +++ /dev/null @@ -1,45 +0,0 @@ ---- -description: when asked to implement new features or clients -globs: *.py -alwaysApply: true ---- - -- When being asked to make new features, make sure that you check out from main a new branch and make incremental commits - - Use conventional commit format: `(): ` - - Types: feat, fix, docs, style, refactor, perf, test, chore - - Example: `feat(validation): add email validation function` - - Keep commits focused on a single change - - Write descriptive commit messages in imperative mood - - Use `git commit -m "type(scope): subject" -m "body" -m "footer"` for multiline commits -- If the feature is very large, create a temporary `todo.md` -- And start a pull request using `gh` - - Create PRs with multiline bodies using: - ```bash - gh pr create --title "feat(component): add new feature" --body "$(cat < --add-reviewer jxnl,ivanleomk` - - Or include `-r jxnl,ivanleomk` when creating the PR -- use `gh pr view --comments | cat` to view all the comments -- For PR updates: - - Do not directly commit to an existing PR branch - - Instead, create a new PR that builds on top of the original PR's branch - - This creates a "stacked PR" pattern where: - 1. The original PR (base) contains the initial changes - 2. The new PR (stack) contains only the review-related updates - 3. Once the base PR is merged, the stack can be rebased onto main diff --git a/참고/instructor-main/.cursor/rules/readme.md b/참고/instructor-main/.cursor/rules/readme.md deleted file mode 100644 index 8ade856..0000000 --- a/참고/instructor-main/.cursor/rules/readme.md +++ /dev/null @@ -1,100 +0,0 @@ -# Cursor Rules - -Cursor rules are configuration files that help guide AI-assisted development in the Cursor IDE. They provide structured instructions for how the AI should behave in specific contexts or when working with certain types of files. - -## What is Cursor? - -[Cursor](https://cursor.sh) is an AI-powered IDE that helps developers write, understand, and maintain code more efficiently. It integrates AI capabilities directly into the development workflow, providing features like: - -- AI-assisted code completion -- Natural language code generation -- Intelligent code explanations -- Automated refactoring suggestions - -## Understanding Cursor Rules - -Cursor rules are defined in `.mdc` files within the `.cursor/rules` directory. Each rule file follows a specific naming convention: lowercase names with the `.mdc` extension (e.g., `simple-language.mdc`). - -Each rule file contains: - -1. **Metadata Header**: YAML frontmatter that defines: - ```yaml - --- - description: when to apply this rule - globs: file patterns to match (e.g., "*.py", "*.md", or "*" for all files) - alwaysApply: true/false # whether to apply automatically - --- - ``` - -2. **Rule Content**: Markdown-formatted instructions that guide the AI's behavior - -## Available Rules - -Currently, the following rules are defined: - -### `simple-language.mdc` -- **Purpose**: Ensures documentation is written at a grade 10 reading level -- **Applies to**: Markdown files (*.md) -- **Auto Apply**: No -- **Key Requirements**: - - Write at grade 10 reading level - - Ensure code blocks are self-contained with complete imports - -### `new-features-planning.mdc` -- **Purpose**: Guides feature implementation workflow -- **Applies to**: Python files (*.py) -- **Auto Apply**: Yes -- **Key Requirements**: - - Create new branch from main - - Make incremental commits - - Create todo.md for large features - - Start pull requests using GitHub CLI (`gh`) - - Include "This PR was written by [Cursor](https://cursor.sh)" in PRs - -### `followups.mdc` -- **Purpose**: Ensures thoughtful follow-up suggestions -- **Applies to**: All files -- **Auto Apply**: Yes -- **Key Requirements**: - - Generate actionable hotkey suggestions using: - - [J]: First follow-up action - - [K]: Second follow-up action - - [L]: Third follow-up action - - Focus on small, contextual code changes - - Suggestions should be thoughtful and actionable - -### `documentation-sync.mdc` -- **Purpose**: Maintains documentation consistency with code changes -- **Applies to**: Python and Markdown files (*.py, *.md) -- **Auto Apply**: Yes -- **Key Requirements**: - - Update docs when code changes - - Add new markdown files to mkdocs.yml - - Keep API documentation current - - Maintain documentation quality standards - -## Creating New Rules - -To create a new rule: - -1. Create a `.mdc` file in `.cursor/rules/` using lowercase naming -2. Add YAML frontmatter with required metadata: - ```yaml - --- - description: when to apply this rule - globs: file patterns to match - alwaysApply: true/false - --- - ``` -3. Write clear, specific instructions in Markdown -4. Test the rule with relevant file types - -## Best Practices - -- Keep rules focused and specific -- Use clear, actionable language -- Test rules thoroughly before committing -- Document any special requirements or dependencies -- Update rules as project needs evolve -- Use consistent file naming (lowercase with .mdc extension) -- Ensure globs patterns are explicit and documented diff --git a/참고/instructor-main/.cursor/rules/simple-language.mdc b/참고/instructor-main/.cursor/rules/simple-language.mdc deleted file mode 100644 index 48a9b19..0000000 --- a/참고/instructor-main/.cursor/rules/simple-language.mdc +++ /dev/null @@ -1,8 +0,0 @@ ---- -description: when writing documentation -globs: *.md -alwaysApply: false ---- - -- When writing documents and concepts make sure that you write at a grade 10 reading level -- make sure every code block has complete imports and makes no references to previous code blocks, each one needs to be self contained diff --git a/참고/instructor-main/.cursorignore b/참고/instructor-main/.cursorignore deleted file mode 100644 index 6f9f00f..0000000 --- a/참고/instructor-main/.cursorignore +++ /dev/null @@ -1 +0,0 @@ -# Add directories or file patterns to ignore during indexing (e.g. foo/ or *.csv) diff --git a/참고/instructor-main/.github/FUNDING.yml b/참고/instructor-main/.github/FUNDING.yml deleted file mode 100644 index 3d74727..0000000 --- a/참고/instructor-main/.github/FUNDING.yml +++ /dev/null @@ -1 +0,0 @@ -github: jxnl \ No newline at end of file diff --git a/참고/instructor-main/.github/ISSUE_TEMPLATE/bug_report.md b/참고/instructor-main/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 909a3b5..0000000 --- a/참고/instructor-main/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,28 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve ---- - -- [ ] This is actually a bug report. -- [ ] I am not getting good LLM Results -- [ ] I have tried asking for help in the community on discord or discussions and have not received a response. -- [ ] I have tried searching the documentation and have not found an answer. - -**What Model are you using?** - -- [ ] gpt-3.5-turbo -- [ ] gpt-4-turbo -- [ ] gpt-4 -- [ ] Other (please specify) - -**Describe the bug** -A clear and concise description of what the bug is. - -**To Reproduce** -Steps to reproduce the behavior, including code snippets of the model and the input data and openai response. - -**Expected behavior** -A clear and concise description of what you expected to happen. - -**Screenshots** -If applicable, add screenshots to help explain your problem. diff --git a/참고/instructor-main/.github/ISSUE_TEMPLATE/feature_request.md b/참고/instructor-main/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index a09db44..0000000 --- a/참고/instructor-main/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this project ---- - -**Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - -**Describe the solution you'd like** -A clear and concise description of what you want to happen. - -**Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - -**Additional context** -Add any other context or screenshots about the feature request here. diff --git a/참고/instructor-main/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md b/참고/instructor-main/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md deleted file mode 100644 index e25023d..0000000 --- a/참고/instructor-main/.github/PULL_REQUEST_TEMPLATE/pull_request_template.md +++ /dev/null @@ -1,13 +0,0 @@ -> Please use conventional commits to describe your changes. For example, `feat: add new feature` or `fix: fix a bug`. If you are unsure, leave the title as `...` and AI will handle it. - -## Describe your changes - -... - -## Issue ticket number and link - -## Checklist before requesting a review - -- [ ] I have performed a self-review of my code -- [ ] If it is a core feature, I have added thorough tests. -- [ ] If it is a core feature, I have added documentation. diff --git a/참고/instructor-main/.github/dependabot.yml b/참고/instructor-main/.github/dependabot.yml deleted file mode 100644 index 5394a87..0000000 --- a/참고/instructor-main/.github/dependabot.yml +++ /dev/null @@ -1,14 +0,0 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - -version: 2 -updates: - - package-ecosystem: "pip" # See documentation for possible values - directory: "/" # Location of package manifests - schedule: - interval: "daily" - groups: - poetry: - patterns: ["*"] diff --git a/참고/instructor-main/.github/workflows/ai-label.yml b/참고/instructor-main/.github/workflows/ai-label.yml deleted file mode 100644 index 86651ce..0000000 --- a/참고/instructor-main/.github/workflows/ai-label.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: AI Labeler - -on: - issues: - types: [opened, reopened] - pull_request: - types: [opened, reopened] - -jobs: - ai-labeler: - runs-on: ubuntu-latest - permissions: - contents: read - issues: write - pull-requests: write - steps: - - uses: actions/checkout@v4 - - uses: jlowin/ai-labeler@v0.4.0 - with: - include-repo-labels: true - openai-api-key: ${{ secrets.OPENAI_API_KEY }} diff --git a/참고/instructor-main/.github/workflows/evals.yml b/참고/instructor-main/.github/workflows/evals.yml deleted file mode 100644 index 492ca85..0000000 --- a/참고/instructor-main/.github/workflows/evals.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Weekly Tests - -on: - workflow_dispatch: - schedule: - - cron: "0 0 * * 0" # Runs at 00:00 UTC every Sunday - push: - branches: [main] - paths-ignore: - - "**" # Ignore all paths to ensure it only triggers on schedule - -jobs: - weekly-tests: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - enable-cache: true - - - name: Set up Python - run: uv python install 3.11 - - - name: Install dependencies - run: uv sync --all-extras --dev - - - name: Run all tests - run: uv run pytest tests/ --asyncio-mode=auto - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} diff --git a/참고/instructor-main/.github/workflows/python-publish.yml b/참고/instructor-main/.github/workflows/python-publish.yml deleted file mode 100644 index 8248f84..0000000 --- a/참고/instructor-main/.github/workflows/python-publish.yml +++ /dev/null @@ -1,38 +0,0 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Upload Python Package - -on: - release: - types: [published] - workflow_dispatch: - -permissions: - contents: read - -jobs: - release: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - enable-cache: true - - name: Set up Python - run: uv python install 3.10 - - name: Install the project - run: uv sync --all-extras - - name: Build the project - run: uv build - - name: Build and publish Python package - run: uv publish - env: - UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }} diff --git a/참고/instructor-main/.github/workflows/ruff.yml b/참고/instructor-main/.github/workflows/ruff.yml deleted file mode 100644 index 37dd342..0000000 --- a/참고/instructor-main/.github/workflows/ruff.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Ruff - -on: - push: - pull_request: - branches: [main] - -env: - WORKING_DIRECTORY: "." - CUSTOM_PACKAGES: "instructor examples tests" - -jobs: - Ruff: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - enable-cache: true - - name: Set up Python - run: uv python install 3.9 - - name: Install the project - run: uv sync --all-extras - - name: Ruff lint - run: uv run ruff check ${{ env.CUSTOM_PACKAGES }} - - name: Ruff format - run: uv run ruff format --check ${{ env.CUSTOM_PACKAGES }} diff --git a/참고/instructor-main/.github/workflows/scheduled-release.yml b/참고/instructor-main/.github/workflows/scheduled-release.yml deleted file mode 100644 index 62f3491..0000000 --- a/참고/instructor-main/.github/workflows/scheduled-release.yml +++ /dev/null @@ -1,267 +0,0 @@ -name: Scheduled Release - -on: - schedule: - # Every 2 weeks on Monday at 9 AM UTC - - cron: '0 9 * * 1/2' - workflow_dispatch: # Allow manual trigger - inputs: - skip_tests: - description: 'Skip LLM tests (use for testing workflow)' - required: false - default: false - type: boolean - dry_run: - description: 'Dry run - dont push changes or create release' - required: false - default: false - type: boolean - -jobs: - test-and-release: - runs-on: ubuntu-latest - if: github.ref == 'refs/heads/main' - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - token: ${{ secrets.GITHUB_TOKEN }} - - - name: Setup UV - uses: astral-sh/setup-uv@v3 - - - name: Install dependencies - run: | - uv sync --all-extras --dev - - - name: Run linting - run: | - uv run ruff check instructor examples tests - - - name: Run type checking - run: | - uv run pyright - - - name: Run core tests (no LLM) - run: | - uv run pytest tests/ -k "not openai and not llm and not anthropic and not gemini and not cohere and not mistral and not groq and not vertexai and not xai and not cerebras and not fireworks and not writer and not bedrock and not perplexity and not genai" --tb=short -v --maxfail=10 - - # Optional: Run LLM tests if you have API keys in secrets - - name: Run LLM tests - if: github.event.inputs.skip_tests != 'true' - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} - MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} - run: | - echo "Running basic LLM tests if API keys are available..." - # Run a subset of LLM tests to verify basic functionality - if [ ! -z "$OPENAI_API_KEY" ]; then - echo "Testing OpenAI integration..." - uv run pytest tests/llm/test_openai/test_basics.py --tb=short -v --maxfail=1 || echo "OpenAI tests failed" - fi - if [ ! -z "$ANTHROPIC_API_KEY" ]; then - echo "Testing Anthropic integration..." - uv run pytest tests/llm/test_anthropic/test_basics.py --tb=short -v --maxfail=1 || echo "Anthropic tests failed" - fi - echo "LLM tests completed (non-blocking)" - - - name: Check for changes since last release - id: changes - run: | - LAST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "") - if [ -z "$LAST_TAG" ]; then - echo "has_changes=true" >> $GITHUB_OUTPUT - echo "last_tag=none" >> $GITHUB_OUTPUT - echo "change_count=initial" >> $GITHUB_OUTPUT - else - CHANGES=$(git rev-list $LAST_TAG..HEAD --count) - echo "has_changes=$([[ $CHANGES -gt 0 ]] && echo true || echo false)" >> $GITHUB_OUTPUT - echo "change_count=$CHANGES" >> $GITHUB_OUTPUT - echo "last_tag=$LAST_TAG" >> $GITHUB_OUTPUT - fi - - echo "Last tag: $LAST_TAG" - echo "Changes since last tag: $(git rev-list $LAST_TAG..HEAD --count 2>/dev/null || echo 'N/A')" - - # Only proceed with release if tests passed AND there are changes - - name: Get current version - if: steps.changes.outputs.has_changes == 'true' - id: current_version - run: | - VERSION=$(uv run python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])") - echo "version=$VERSION" >> $GITHUB_OUTPUT - echo "Current version: $VERSION" - - - name: Determine version bump type - if: steps.changes.outputs.has_changes == 'true' - id: version_type - run: | - # Check commit messages since last tag to determine bump type - LAST_TAG="${{ steps.changes.outputs.last_tag }}" - if [ "$LAST_TAG" = "none" ]; then - COMMITS=$(git log --oneline HEAD~20..HEAD) - else - COMMITS=$(git log --oneline $LAST_TAG..HEAD) - fi - - echo "Recent commits:" - echo "$COMMITS" - - # Look for breaking changes or major features - if echo "$COMMITS" | grep -qE "(BREAKING|feat!|fix!)"; then - echo "bump_type=minor" >> $GITHUB_OUTPUT - echo "Detected breaking changes - using minor bump" - elif echo "$COMMITS" | grep -qE "feat:"; then - echo "bump_type=minor" >> $GITHUB_OUTPUT - echo "Detected new features - using minor bump" - else - echo "bump_type=patch" >> $GITHUB_OUTPUT - echo "Using patch bump for bug fixes and chores" - fi - - - name: Bump version - if: steps.changes.outputs.has_changes == 'true' - id: bump_version - run: | - CURRENT="${{ steps.current_version.outputs.version }}" - BUMP_TYPE="${{ steps.version_type.outputs.bump_type }}" - - IFS='.' read -r major minor patch <<< "$CURRENT" - - case $BUMP_TYPE in - major) - major=$((major + 1)) - minor=0 - patch=0 - ;; - minor) - minor=$((minor + 1)) - patch=0 - ;; - patch) - patch=$((patch + 1)) - ;; - esac - - NEW_VERSION="$major.$minor.$patch" - echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT - echo "Bumping from $CURRENT to $NEW_VERSION ($BUMP_TYPE)" - - # Update pyproject.toml - sed -i "s/version = \"$CURRENT\"/version = \"$NEW_VERSION\"/" pyproject.toml - - - name: Update lockfile - if: steps.changes.outputs.has_changes == 'true' - run: | - uv lock - - # Run tests again after version bump to make sure nothing broke - - name: Final test run - if: steps.changes.outputs.has_changes == 'true' - run: | - uv sync - uv run pytest tests/ -k "not openai and not llm and not anthropic and not gemini and not cohere and not mistral and not groq and not vertexai and not xai and not cerebras and not fireworks and not writer and not bedrock and not perplexity and not genai" --tb=short --maxfail=5 - - - name: Generate changelog - if: steps.changes.outputs.has_changes == 'true' - id: changelog - run: | - LAST_TAG="${{ steps.changes.outputs.last_tag }}" - NEW_VERSION="${{ steps.bump_version.outputs.new_version }}" - - if [ "$LAST_TAG" = "none" ]; then - CHANGELOG=$(git log --oneline HEAD~30..HEAD --pretty=format:"- %s" | head -20) - else - CHANGELOG=$(git log --oneline $LAST_TAG..HEAD --pretty=format:"- %s") - fi - - # Save changelog to file for GitHub release - cat > CHANGELOG.md << EOF - ## 🚀 What's Changed - - $CHANGELOG - - ## 🔗 Links - **Full Changelog**: https://github.com/${{ github.repository }}/compare/$LAST_TAG...v$NEW_VERSION - - --- - 🤖 *This release was automatically generated every 2 weeks* - EOF - - echo "changelog_file=CHANGELOG.md" >> $GITHUB_OUTPUT - - - name: Create release commit - if: steps.changes.outputs.has_changes == 'true' - run: | - git config --local user.email "action@github.com" - git config --local user.name "GitHub Action" - git add pyproject.toml uv.lock - git commit -m "chore: automated release v${{ steps.bump_version.outputs.new_version }} - - 🤖 Generated with [Claude Code](https://claude.ai/code) - - Co-Authored-By: GitHub Action " - git tag "v${{ steps.bump_version.outputs.new_version }}" - - - name: Push changes - if: steps.changes.outputs.has_changes == 'true' && github.event.inputs.dry_run != 'true' - run: | - git push origin main - git push origin "v${{ steps.bump_version.outputs.new_version }}" - - - name: Create GitHub Release - if: steps.changes.outputs.has_changes == 'true' && github.event.inputs.dry_run != 'true' - uses: ncipollo/release-action@v1 - with: - tag: "v${{ steps.bump_version.outputs.new_version }}" - name: "🚀 Release v${{ steps.bump_version.outputs.new_version }}" - bodyFile: "CHANGELOG.md" - draft: false - prerelease: false - - - name: Dry run summary - if: steps.changes.outputs.has_changes == 'true' && github.event.inputs.dry_run == 'true' - run: | - echo "🧪 DRY RUN MODE - No changes pushed" - echo "Would have released: v${{ steps.bump_version.outputs.new_version }}" - cat CHANGELOG.md - - # Optional: Publish to PyPI (uncomment if you want automatic PyPI releases) - # - name: Build and publish to PyPI - # if: steps.changes.outputs.has_changes == 'true' && secrets.PYPI_TOKEN != '' - # env: - # PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - # run: | - # uv build - # uv publish --token $PYPI_TOKEN - - # Summary outputs - - name: Summary - if: always() - run: | - echo "## 📊 Scheduled Release Summary" >> $GITHUB_STEP_SUMMARY - echo "- **Branch**: ${{ github.ref }}" >> $GITHUB_STEP_SUMMARY - echo "- **Has Changes**: ${{ steps.changes.outputs.has_changes }}" >> $GITHUB_STEP_SUMMARY - echo "- **Change Count**: ${{ steps.changes.outputs.change_count }}" >> $GITHUB_STEP_SUMMARY - if [ "${{ steps.changes.outputs.has_changes }}" = "true" ]; then - echo "- **Version**: ${{ steps.current_version.outputs.version }} → ${{ steps.bump_version.outputs.new_version }}" >> $GITHUB_STEP_SUMMARY - echo "- **Bump Type**: ${{ steps.version_type.outputs.bump_type }}" >> $GITHUB_STEP_SUMMARY - echo "- **Status**: ✅ Released" >> $GITHUB_STEP_SUMMARY - else - echo "- **Status**: ⏭️ Skipped (no changes)" >> $GITHUB_STEP_SUMMARY - fi - - - name: Notify on failure - if: failure() - run: | - echo "❌ Scheduled release failed - check the logs above" - echo "Common issues:" - echo "- Tests failed" - echo "- Linting issues" - echo "- Type checking errors" - echo "- Git push permissions" \ No newline at end of file diff --git a/참고/instructor-main/.github/workflows/test.yml b/참고/instructor-main/.github/workflows/test.yml deleted file mode 100644 index 84ef73d..0000000 --- a/참고/instructor-main/.github/workflows/test.yml +++ /dev/null @@ -1,324 +0,0 @@ -name: Test -on: - pull_request: - push: - branches: - - main - -jobs: - # Core tests without LLM providers - core-tests: - name: Core Tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - enable-cache: true - - name: Set up Python - run: uv python install 3.11 - - name: Install the project - run: uv sync --all-extras - - name: Run core tests - run: >- - uv run pytest tests/ --asyncio-mode=auto -n auto - -k 'not test_core_providers and not test_openai and not test_anthropic - and not test_gemini and not test_genai and not test_writer and not - test_vertexai and not docs' - env: - INSTRUCTOR_ENV: CI - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - - # Core provider tests for OpenAI - core-openai: - name: Core Provider Tests (OpenAI) - runs-on: ubuntu-latest - needs: core-tests - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - - steps: - - uses: actions/checkout@v2 - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - enable-cache: true - - name: Set up Python - run: uv python install 3.11 - - name: Install the project - run: uv sync --all-extras - - name: Skip core provider tests (OpenAI) - if: ${{ env.OPENAI_API_KEY == '' }} - run: echo "Skipping OpenAI core provider tests (missing OPENAI_API_KEY)." - - name: Run core provider tests (OpenAI) - if: ${{ env.OPENAI_API_KEY != '' }} - run: | - set +e - uv run pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "openai" - status=$? - set -e - if [ $status -eq 5 ]; then - echo "No tests collected; treating as success." - exit 0 - fi - exit $status - env: - INSTRUCTOR_ENV: CI - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - - # Core provider tests for Anthropic - core-anthropic: - name: Core Provider Tests (Anthropic) - runs-on: ubuntu-latest - needs: core-tests - env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - - steps: - - uses: actions/checkout@v2 - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - enable-cache: true - - name: Set up Python - run: uv python install 3.11 - - name: Install the project - run: uv sync --all-extras - - name: Skip core provider tests (Anthropic) - if: ${{ env.ANTHROPIC_API_KEY == '' }} - run: echo "Skipping Anthropic core provider tests (missing ANTHROPIC_API_KEY)." - - name: Run core provider tests (Anthropic) - if: ${{ env.ANTHROPIC_API_KEY != '' }} - run: | - set +e - uv run pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "anthropic" - status=$? - set -e - if [ $status -eq 5 ]; then - echo "No tests collected; treating as success." - exit 0 - fi - exit $status - env: - INSTRUCTOR_ENV: CI - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - - # Core provider tests for Google - core-google: - name: Core Provider Tests (Google) - runs-on: ubuntu-latest - needs: core-tests - env: - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - GOOGLE_GENAI_MODEL: ${{ secrets.GOOGLE_GENAI_MODEL }} - - steps: - - uses: actions/checkout@v2 - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - enable-cache: true - - name: Set up Python - run: uv python install 3.11 - - name: Install the project - run: uv sync --all-extras - - name: Skip core provider tests (Google) - if: ${{ env.GOOGLE_API_KEY == '' || env.GOOGLE_GENAI_MODEL == '' }} - run: echo "Skipping Google core provider tests (missing GOOGLE_API_KEY or GOOGLE_GENAI_MODEL)." - - name: Run core provider tests (Google) - if: ${{ env.GOOGLE_API_KEY != '' && env.GOOGLE_GENAI_MODEL != '' }} - run: | - set +e - uv run pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "google" - status=$? - set -e - if [ $status -eq 5 ]; then - echo "No tests collected; treating as success." - exit 0 - fi - exit $status - env: - INSTRUCTOR_ENV: CI - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - - # Core provider tests for other providers - core-other: - name: Core Provider Tests (Other) - runs-on: ubuntu-latest - needs: core-tests - env: - COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} - CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} - FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} - WRITER_API_KEY: ${{ secrets.WRITER_API_KEY }} - PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }} - - steps: - - uses: actions/checkout@v2 - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - enable-cache: true - - name: Set up Python - run: uv python install 3.11 - - name: Install the project - run: uv sync --all-extras - - name: Skip core provider tests (Other) - if: >- - ${{ env.COHERE_API_KEY == '' && env.XAI_API_KEY == '' - && env.MISTRAL_API_KEY == '' && env.CEREBRAS_API_KEY == '' - && env.FIREWORKS_API_KEY == '' && env.WRITER_API_KEY == '' - && env.PERPLEXITY_API_KEY == '' }} - run: echo "Skipping core provider tests (Other) (missing provider secrets)." - - name: Run core provider tests (Cohere, xAI, Mistral, etc) - if: >- - ${{ env.COHERE_API_KEY != '' || env.XAI_API_KEY != '' - || env.MISTRAL_API_KEY != '' || env.CEREBRAS_API_KEY != '' - || env.FIREWORKS_API_KEY != '' || env.WRITER_API_KEY != '' - || env.PERPLEXITY_API_KEY != '' }} - run: | - set +e - uv run pytest tests/llm/test_core_providers -v --asyncio-mode=auto -n auto -k "cohere or xai or mistral or cerebras or fireworks or writer or perplexity" - status=$? - set -e - if [ $status -eq 5 ]; then - echo "No tests collected; treating as success." - exit 0 - fi - exit $status - env: - INSTRUCTOR_ENV: CI - COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} - CEREBRAS_API_KEY: ${{ secrets.CEREBRAS_API_KEY }} - FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }} - WRITER_API_KEY: ${{ secrets.WRITER_API_KEY }} - PERPLEXITY_API_KEY: ${{ secrets.PERPLEXITY_API_KEY }} - - # Provider tests run in parallel - provider-tests: - name: ${{ matrix.provider.name }} Tests - runs-on: ubuntu-latest - needs: [core-openai, core-anthropic, core-google, core-other] - env: - PROVIDER_API_KEY: ${{ secrets[matrix.provider.env_key] }} - GOOGLE_GENAI_MODEL: ${{ secrets.GOOGLE_GENAI_MODEL }} - strategy: - fail-fast: false - matrix: - provider: - - name: OpenAI - env_key: OPENAI_API_KEY - test_path: tests/llm/test_openai - - name: Anthropic - env_key: ANTHROPIC_API_KEY - test_path: tests/llm/test_anthropic - - name: Gemini - env_key: GOOGLE_API_KEY - test_path: tests/llm/test_gemini - - name: Google GenAI - env_key: GOOGLE_API_KEY - test_path: tests/llm/test_genai - - name: Vertex AI - env_key: GOOGLE_API_KEY - test_path: tests/llm/test_vertexai - - name: Writer - env_key: WRITER_API_KEY - test_path: tests/llm/test_writer - - steps: - - uses: actions/checkout@v2 - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - enable-cache: true - - name: Set up Python - run: uv python install 3.11 - - name: Install the project - run: uv sync --all-extras - - name: Skip ${{ matrix.provider.name }} tests - if: >- - ${{ env.PROVIDER_API_KEY == '' || - ((matrix.provider.name == 'Gemini' || matrix.provider.name == 'Google GenAI' - || matrix.provider.name == 'Vertex AI') && env.GOOGLE_GENAI_MODEL == '') }} - run: >- - echo "Skipping ${{ matrix.provider.name }} tests - (missing ${{ matrix.provider.env_key }} or GOOGLE_GENAI_MODEL)." - - name: Run ${{ matrix.provider.name }} tests - if: >- - ${{ env.PROVIDER_API_KEY != '' && - ((matrix.provider.name != 'Gemini' && matrix.provider.name != 'Google GenAI' - && matrix.provider.name != 'Vertex AI') || env.GOOGLE_GENAI_MODEL != '') }} - run: | - set +e - uv run pytest ${{ matrix.provider.test_path }} --asyncio-mode=auto -n auto - status=$? - set -e - if [ $status -eq 5 ]; then - echo "No tests collected; treating as success." - exit 0 - fi - exit $status - env: - INSTRUCTOR_ENV: CI - ${{ matrix.provider.env_key }}: ${{ secrets[matrix.provider.env_key] }} - - # Auto client needs multiple providers - auto-client-test: - name: Auto Client Tests - runs-on: ubuntu-latest - needs: [core-openai, core-anthropic, core-google, core-other] - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - XAI_API_KEY: ${{ secrets.XAI_API_KEY }} - - steps: - - uses: actions/checkout@v2 - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - enable-cache: true - - name: Set up Python - run: uv python install 3.11 - - name: Install the project - run: uv sync --all-extras - - name: Skip Auto Client tests - if: >- - ${{ env.OPENAI_API_KEY == '' || env.GOOGLE_API_KEY == '' - || env.COHERE_API_KEY == '' || env.ANTHROPIC_API_KEY == '' - || env.XAI_API_KEY == '' }} - run: echo "Skipping Auto Client tests (missing one or more provider secrets)." - - name: Run Auto Client tests - if: >- - ${{ env.OPENAI_API_KEY != '' && env.GOOGLE_API_KEY != '' - && env.COHERE_API_KEY != '' && env.ANTHROPIC_API_KEY != '' - && env.XAI_API_KEY != '' }} - run: | - set +e - uv run pytest tests/test_auto_client.py --asyncio-mode=auto -n auto - status=$? - set -e - if [ $status -eq 5 ]; then - echo "No tests collected; treating as success." - exit 0 - fi - exit $status - env: - INSTRUCTOR_ENV: CI - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - XAI_API_KEY: ${{ secrets.XAI_API_KEY }} diff --git a/참고/instructor-main/.github/workflows/test_docs.yml b/참고/instructor-main/.github/workflows/test_docs.yml deleted file mode 100644 index 9a347b1..0000000 --- a/참고/instructor-main/.github/workflows/test_docs.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: Test Docs -on: - schedule: - - cron: '0 0 1 * *' # Runs at 00:00 on the 1st of every month -jobs: - release: - runs-on: ubuntu-latest - - strategy: - matrix: - python-version: ["3.11"] - - steps: - - uses: actions/checkout@v2 - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y graphviz libcairo2-dev xdg-utils - - - name: Install Poetry - uses: snok/install-poetry@v1.3.1 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - cache: "poetry" - - name: Install uv - uses: astral-sh/setup-uv@v4 - - name: Install the project - run: uv sync --all-extras - - name: Run tests - run: uv run pytest tests/docs --asyncio-mode=auto - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} diff --git a/참고/instructor-main/.github/workflows/ty.yml b/참고/instructor-main/.github/workflows/ty.yml deleted file mode 100644 index 8a7fcc0..0000000 --- a/참고/instructor-main/.github/workflows/ty.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: ty - -on: - pull_request: - branches: [main] - push: - branches: [main] - -env: - WORKING_DIRECTORY: "." - -jobs: - type-check: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - enable-cache: true - - name: Set up Python - run: uv python install 3.11 - - name: Install the project - run: uv sync --all-extras - - name: Run type check with ty - run: uv run ty check instructor/ - - name: Run type check with ty (tests) - run: uv run ty check --config-file ty-tests.toml tests diff --git a/참고/instructor-main/.gitignore b/참고/instructor-main/.gitignore deleted file mode 100644 index b451f85..0000000 --- a/참고/instructor-main/.gitignore +++ /dev/null @@ -1,184 +0,0 @@ -.DS_Store -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ -cover/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ -.envrc - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ - -# pytype static type analyzer -.pytype/ - -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -.idea/ - -.vscode/ - -examples/citation_with_extraction/fly.toml -my_cache_directory/ -tutorials/wandb/* -tutorials/results.csv -tutorials/results.jsonl -tutorials/results.jsonlines -tutorials/schema.json -wandb/settings -math_finetunes.jsonl - -pr_body.md - -check_zero_width_chars.py - -# Suggestion files from architectural analysis -*_SUGGESTIONS.md -ORGANIZED_SUGGESTIONS.md -*.orig diff --git a/참고/instructor-main/.grit/.gitignore b/참고/instructor-main/.grit/.gitignore deleted file mode 100644 index 799e2c7..0000000 --- a/참고/instructor-main/.grit/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.gritmodules -*.log diff --git a/참고/instructor-main/.grit/grit.yaml b/참고/instructor-main/.grit/grit.yaml deleted file mode 100644 index 96c9c01..0000000 --- a/참고/instructor-main/.grit/grit.yaml +++ /dev/null @@ -1,4 +0,0 @@ -version: 0.0.1 -patterns: - - name: github.com/getgrit/python#openai - level: info diff --git a/참고/instructor-main/.pre-commit-config.yaml b/참고/instructor-main/.pre-commit-config.yaml deleted file mode 100644 index 03791a2..0000000 --- a/참고/instructor-main/.pre-commit-config.yaml +++ /dev/null @@ -1,43 +0,0 @@ -repos: - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.9 # Ruff version - hooks: - - id: ruff # Run the linter. - name: Run Linter Check (Ruff) - args: [ --fix, --unsafe-fixes ] - files: ^(instructor|tests|examples)/ - - id: ruff-format # Run the formatter. - name: Run Formatter (Ruff) - - - repo: local - hooks: - - id: uv-lock-check - name: Check uv.lock is up-to-date - entry: uv - args: [lock, --check] - language: system - files: ^(pyproject\.toml|uv\.lock)$ - pass_filenames: false - - - id: uv-sync-check - name: Verify dependencies can be installed - entry: uv - args: [sync, --check, --all-extras] - language: system - files: ^(pyproject\.toml|uv\.lock)$ - pass_filenames: false - - - id: uv-export-requirements - name: Export requirements.txt from pyproject.toml - entry: bash -c 'uv pip compile pyproject.toml -o requirements.txt && git add requirements.txt' - language: system - files: ^pyproject\.toml$ - pass_filenames: false - - - id: ty-check - name: Run Type Check (ty) - entry: uv - args: [run, --frozen, ty, check, instructor/] - language: system - files: ^instructor/ - pass_filenames: false diff --git a/참고/instructor-main/.ruff.toml b/참고/instructor-main/.ruff.toml deleted file mode 100644 index c2391fc..0000000 --- a/참고/instructor-main/.ruff.toml +++ /dev/null @@ -1,62 +0,0 @@ -# Exclude a variety of commonly ignored directories. -exclude = [ - ".bzr", - ".direnv", - ".eggs", - ".git", - ".git-rewrite", - ".hg", - ".mypy_cache", - ".nox", - ".pants.d", - ".pytype", - ".ruff_cache", - ".svn", - ".tox", - ".venv", - "__pypackages__", - "_build", - "buck-out", - "build", - "dist", - "node_modules", - "venv", -] - -# Same as Black. -line-length = 88 -output-format = "grouped" - -target-version = "py39" - -[lint] -select = [ - # bugbear rules - "B", - # remove unused imports - "F401", - # bare except statements - "E722", - # unused arguments - "ARG", - # pyupgrade - "UP", -] -ignore = [ - # mutable defaults - "B006", - "B018", -] - -unfixable = [ - # disable auto fix for print statements - "T201", - "T203", -] - -[lint.extend-per-file-ignores] -"instructor/distil.py" = ["ARG002"] -"tests/test_distil.py" = ["ARG001"] -"tests/test_patch.py" = ["ARG001"] -"examples/task_planner/task_planner_topological_sort.py" = ["ARG002"] -"examples/citation_with_extraction/main.py" = ["ARG001"] diff --git a/참고/instructor-main/AGENT.md b/참고/instructor-main/AGENT.md deleted file mode 100644 index 267c651..0000000 --- a/참고/instructor-main/AGENT.md +++ /dev/null @@ -1,122 +0,0 @@ -# AGENT.md - -## Commands -- Install: `uv pip install -e ".[dev]"` or `poetry install --with dev` -- Run tests: `uv run pytest tests/` -- Run single test: `uv run pytest tests/path_to_test.py::test_name` -- Skip LLM tests: `uv run pytest tests/ -k 'not llm and not openai'` -- Temp deps for a run: `uv run --with [==version] ` (example: `uv run --with pytest-asyncio --with anthropic pytest tests/...`) -- Type check: `uv run ty check` -- Lint: `uv run ruff check instructor examples tests` -- Format: `uv run ruff format instructor examples tests` -- Build docs: `uv run mkdocs serve` (local) or `./build_mkdocs.sh` (production) -- Waiting: use `sleep ` for explicit pauses (e.g., CI waits) or to let external processes finish - -## Architecture -- **Core**: `instructor/` - Pydantic-based structured outputs for LLMs -- **Base classes**: `Instructor` and `AsyncInstructor` in `client.py` -- **Providers**: Client files (`client_*.py`) for OpenAI, Anthropic, Gemini, Cohere, etc. -- **Factory pattern**: `from_provider()` for automatic provider detection -- **DSL**: `dsl/` directory with Partial, Iterable, Maybe, Citation extensions -- **Key modules**: `patch.py` (patching), `process_response.py` (parsing), `function_calls.py` (schemas) - -## Code Style -- **Typing**: Strict type annotations, use `BaseModel` for structured outputs -- **Imports**: Standard lib → third-party → local -- **Formatting**: Ruff with Black conventions -- **Error handling**: Custom exceptions from `exceptions.py`, Pydantic validation -- **Naming**: `snake_case` functions/variables, `PascalCase` classes -- **No mocking**: Tests use real API calls -- **Client creation**: Always use `instructor.from_provider("provider_name/model_name")` instead of provider-specific methods like `from_openai()`, `from_anthropic()`, etc. - -## Pull Request (PR) Formatting - -Use **Conventional Commits** formatting for PR titles. Treat the PR title as the message we would use for a squash merge commit. - -### PR Title Format - -Use: - -`(): ` - -Rules: -- Keep it under ~70 characters when you can. -- Use the imperative mood (for example, “add”, “fix”, “update”). -- Do not end with a period. -- If it includes a breaking change, add `!` after the type or scope (for example, `feat(api)!:`). - -Good examples: -- `fix(openai): handle empty tool_calls in streaming` -- `feat(retry): add backoff for JSON parse failures` -- `docs(agents): add conventional commit PR title guidelines` -- `test(schema): cover nested union edge cases` -- `ci(ruff): enforce formatting in pre-commit` - -Common types: -- `feat`: new feature -- `fix`: bug fix -- `docs`: documentation-only changes -- `refactor`: code change that is not a fix or feature -- `perf`: performance improvement -- `test`: add or update tests -- `build`: build system or dependency changes -- `ci`: CI pipeline changes -- `chore`: maintenance work - -Suggested scopes (pick the closest match): -- Providers: `openai`, `anthropic`, `gemini`, `vertexai`, `bedrock`, `mistral`, `groq`, `writer` -- Core: `core`, `patch`, `process_response`, `function_calls`, `retry`, `dsl` -- Repo: `docs`, `examples`, `tests`, `ci`, `build` - -### PR Description Guidelines - -Keep PR descriptions short and easy to review: -- **What**: What changed, in 1–3 sentences. -- **Why**: Why this change is needed (link issues when possible). -- **Changes**: 3–7 bullet points with the main edits. -- **Testing**: What you ran (or why you did not run anything). - -If the PR was authored by Cursor, include: -- `This PR was written by [Cursor](https://cursor.com)` - -### Changelog Requirement - -**Every PR that changes behavior must update `CHANGELOG.md`.** - -Add an entry under the `## [Unreleased]` section (or the current in-progress version): - -``` -- **Area**: Short description of the change ([#PR_NUMBER](url)) -``` - -Group entries under: `Security`, `Fixed`, `Added`, `Changed`, `Deprecated`, `Removed`, `Tests / CI`. - -Do not add changelog entries for docs-only or example-only changes unless they fix something user-visible. - -## Release Process - -Steps to publish a new version (e.g. `v1.15.0`): - -1. **Ensure CI is green** on the staging PR before merging. - -2. **Merge staging → main** via the GitHub PR. - -3. **Bump version** in `pyproject.toml` (field `version = "X.Y.Z"`), then update the lockfile: - ``` - uv lock - ``` - -4. **Commit and tag** (tags use lowercase `v` prefix): - ``` - git add pyproject.toml uv.lock - git commit -m "chore(release): vX.Y.Z" - git tag vX.Y.Z - git push origin main --tags - ``` - -5. **Create a GitHub Release** for the tag — this triggers `.github/workflows/python-publish.yml`, which builds and publishes to PyPI automatically using the `PYPI_TOKEN` secret. - -Version bump rules (based on commits since last tag): -- `feat!:` / `fix!:` / `BREAKING` → major -- `feat:` → minor -- `fix:` / `chore:` / everything else → patch diff --git a/참고/instructor-main/CHANGELOG.md b/참고/instructor-main/CHANGELOG.md deleted file mode 100644 index 60ead83..0000000 --- a/참고/instructor-main/CHANGELOG.md +++ /dev/null @@ -1,208 +0,0 @@ -# Changelog - -All notable changes to instructor are documented here. - -Format: [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) -Versioning: [Semantic Versioning](https://semver.org/spec/v2.0.0.html) - ---- - -## [Unreleased] - -### Fixed -- **Templating (GenAI/VertexAI)**: `process_message` no longer crashes with `TypeError: Can't compile non template nodes` when multimodal messages contain image/URI/bytes Parts alongside `validation_context`. Non-text Parts (where `part.text` is `None`) now pass through unchanged. ([#2253](https://github.com/567-labs/instructor/issues/2253)) -- **Retry**: `IncompleteOutputException` now propagates directly to the caller without being wrapped in `InstructorRetryException`, making `except IncompleteOutputException` catch blocks work as documented. Applies to both sync and async paths. ([#2273](https://github.com/567-labs/instructor/issues/2273)) - ---- - -## [1.15.1] - 2026-04-03 - -### Security -- **Bedrock**: Block remote HTTP(S) image URL fetching in `_openai_image_part_to_bedrock` — only `data:` URLs are now accepted, preventing SSRF via user-controlled image URLs -- **Bedrock/PDF**: Block remote URL and local file fetching in `PDF.to_bedrock` — only base64 data or `s3://` sources are now supported, preventing SSRF and local file disclosure - -### Added -- **Hooks**: `completion:error` and `completion:last_attempt` handlers now receive `attempt_number`, `max_attempts`, and `is_last_attempt` as keyword arguments. Old-style handlers remain fully backward-compatible. -- **Anthropic**: `from_provider("anthropic/...")` now sets a `User-Agent: instructor/` header on the Anthropic client - -### Fixed -- **Anthropic usage**: Initialize usage correctly for `ANTHROPIC_REASONING_TOOLS` and `ANTHROPIC_PARALLEL_TOOLS` modes — previously fell through to OpenAI usage tracking with wrong field names -- **OpenRouter**: Use `reask_md_json` for `OPENROUTER_STRUCTURED_OUTPUTS` retries instead of `reask_default` (tool-call format), fixing malformed retry prompts -- **Templating**: Return `kwargs` unchanged instead of `None` in `handle_templating` when message list is empty or format is unrecognized; `process_message` also now returns the original message unchanged for unrecognized formats instead of `None` -- **`from_openai`**: Allow `Mode.JSON_SCHEMA` for the OpenAI provider — it was incorrectly blocked by the mode validation check -- **Bedrock**: Pass through `cachePoint` dicts in message content unchanged — previously raised `ValueError: Unsupported dict content for Bedrock`, breaking prompt caching (regression since v1.13.0) -- **Bedrock**: Allow `Mode.MD_JSON` in `from_bedrock` -- **Parallel tools**: `ParallelBase` generator now consumed into `ListResponse` in both sync and async paths, fixing `AttributeError` when setting `_raw_response` on a generator - ---- - -## [1.15.0] - 2026-04-02 - -### Security -- Pin litellm to `<=1.82.6` to block compromised versions 1.82.7 and 1.82.8 ([#2219](https://github.com/567-labs/instructor/pull/2219)) -- Make `diskcache` an optional dependency, removing it from all users' transitive dependency trees and mitigating CVE-2025-69872 ([#2211](https://github.com/567-labs/instructor/pull/2211)) - -### Fixed -- **Usage tracking**: Preserve `response.usage` subclass type (e.g. LiteLLM, Langfuse) when accumulating token counts across retries — fixes downstream `.get()` method loss ([#2217](https://github.com/567-labs/instructor/pull/2217), [#2199](https://github.com/567-labs/instructor/pull/2199)) -- **Gemini**: Exclude `HARM_CATEGORY_IMAGE_*` safety categories from standard Gemini API calls — these are Vertex AI-only and caused `400 INVALID_ARGUMENT` errors ([#2174](https://github.com/567-labs/instructor/pull/2174)) -- **Gemini**: Detect truncated responses (`finish_reason=MAX_TOKENS`) in `GENAI_STRUCTURED_OUTPUTS` mode and raise `IncompleteOutputException` immediately instead of retrying with malformed JSON ([#2232](https://github.com/567-labs/instructor/pull/2232)) -- **`create_with_completion`**: Handle `List[Model]` response models that lack `_raw_response` attribute — previously raised `AttributeError`, now returns `None` for the completion ([#2167](https://github.com/567-labs/instructor/pull/2167)) -- **Partial streaming**: Preserve default `Literal` field values (e.g. `type: Literal["Person"] = "Person"`) during streaming instead of emitting `None` before the field arrives ([#2204](https://github.com/567-labs/instructor/pull/2204)) -- **Partial streaming**: Support PEP 604 union syntax (`str | int`) in `Partial` models on Python 3.10+ ([#2200](https://github.com/567-labs/instructor/pull/2200)) -- **Validators**: Fix `allow_override=True` in `llm_validator` — the override branch was unreachable due to a misplaced assertion, so `fixed_value` was never returned ([#2215](https://github.com/567-labs/instructor/pull/2215)) -- **Parallel tools**: `ParallelBase` responses now return `ListResponse` (consistent with `IterableBase`) instead of a raw generator with `_raw_response` set on it ([#2216](https://github.com/567-labs/instructor/pull/2216)) -- **Multimodal**: Add missing `continue` in `convert_messages` after handling typed (`audio`/`image`) messages — previously fell through to `message["role"]` causing `KeyError` ([#2139](https://github.com/567-labs/instructor/pull/2139)) -- **Anthropic**: Fix dead code path for `ANTHROPIC_REASONING_TOOLS` mode — the mode was shadowed by a duplicate `ANTHROPIC_TOOLS` check and never routed correctly ([#2140](https://github.com/567-labs/instructor/pull/2140)) - -### Added -- **Models**: Add Claude 4 (Opus, Sonnet, Haiku), OpenAI GPT-4.1 series, o3/o4 reasoning models, xAI Grok 3, and DeepSeek R1/V3 to `KnownModelName` type ([#2235](https://github.com/567-labs/instructor/pull/2235)) - -### Docs -- Update GitHub organization links in README from `instructor-ai` to `567-labs` ([#2149](https://github.com/567-labs/instructor/pull/2149)) - -### Tests / CI -- Fix `test_xai_optional_dependency` tests to use `monkeypatch` so they pass regardless of whether `xai-sdk` is installed -- Update deprecated Anthropic model names (`claude-3-5-haiku-latest` -> `claude-haiku-4-0-20250414`, `claude-3-7-sonnet-latest` -> `claude-sonnet-4-5-20250514`) -- Update deprecated OpenAI model names (`gpt-3.5-turbo` -> `gpt-4.1-mini`) across unit tests -- Update stale provider model strings in `shared_config.py`: Writer palmyra-x5, Fireworks llama-v3p3, Perplexity sonar-pro - ---- - -## [1.14.5] - 2026-01-29 - -### Fixed -- **Google GenAI**: `thought_signature` is now preserved across validation retries for thinking models ([#2001](https://github.com/567-labs/instructor/pull/2001)) -- **Metadata**: `pyproject.toml` author field corrected so PyPI correctly populates the `Author` field ([#2015](https://github.com/567-labs/instructor/pull/2015)) -- **Deps**: Dev dependencies moved to the correct `[dependency-groups]` section in `pyproject.toml` ([#2030](https://github.com/567-labs/instructor/pull/2030)) - ---- - -## [1.14.4] - 2026-01-16 - -### Fixed -- **Responses API**: Validation errors during structured output parsing are now caught and retried correctly ([#2002](https://github.com/567-labs/instructor/pull/2002)) -- **Google GenAI**: User-provided `GenerationConfig` labels and custom fields are no longer silently dropped when merging configs ([#2005](https://github.com/567-labs/instructor/pull/2005)) -- **Google GenAI**: `SafetySettings` now applied correctly when request contains image content ([#2007](https://github.com/567-labs/instructor/pull/2007)) -- **List responses**: Response wrappers no longer crash on attribute-style access ([#2011](https://github.com/567-labs/instructor/pull/2011)) -- **`_raw_response`**: Attribute access on list response wrappers works correctly ([#2012](https://github.com/567-labs/instructor/pull/2012)) - -### Changed -- **`json_tracker`**: Sibling-heuristic algorithm simplified for improved partial-streaming reliability ([#2000](https://github.com/567-labs/instructor/pull/2000)) - ---- - -## [1.14.3] - 2026-01-13 - -### Added -- **Partial streaming**: Completeness-based streaming validation — fields are validated progressively rather than failing mid-stream ([#1999](https://github.com/567-labs/instructor/pull/1999)) - -### Fixed -- **Streaming reask**: `Stream` objects in reask handlers are now consumed correctly before retry, preventing stale-stream errors ([#1992](https://github.com/567-labs/instructor/pull/1992)) - ---- - -## [1.14.2] - 2026-01-13 - -### Fixed -- **Partial streaming**: Model validators now skip during partial streaming and run only once on the final complete object, preventing spurious errors ([#1994](https://github.com/567-labs/instructor/pull/1994)) -- **Partial**: Infinite recursion with self-referential models (e.g. `TreeNode` with `children: List["TreeNode"]`) is now prevented ([#1997](https://github.com/567-labs/instructor/pull/1997)) - -### Tests / CI -- Provider tests skipped in CI when API secrets are not available ([#1990](https://github.com/567-labs/instructor/pull/1990)) - ---- - -## [1.14.1] - 2026-01-08 - -### Fixed -- **Google GenAI**: `cached_content` parameter now correctly forwarded to support Google context caching ([#1987](https://github.com/567-labs/instructor/pull/1987)) - ---- - -## [1.14.0] - 2026-01-04 - -### Added -- **Bedrock**: Document support — pass PDFs and text files directly to Bedrock models ([#1936](https://github.com/567-labs/instructor/pull/1936)) - -### Fixed -- **`from_provider()`**: Now respects the `base_url` keyword argument for OpenAI-compatible providers ([#1971](https://github.com/567-labs/instructor/pull/1971)) -- **`from_provider()`**: Runtime `ImportError` exceptions are no longer masked, making misconfigured installs easier to diagnose ([#1975](https://github.com/567-labs/instructor/pull/1975)) -- **Google GenAI**: `Union` types now allowed in structured output schemas ([#1973](https://github.com/567-labs/instructor/pull/1973)) -- **Google GenAI**: `thinking_config` and additional user-provided `GenerationConfig` fields now correctly preserved ([#1972](https://github.com/567-labs/instructor/pull/1972), [#1974](https://github.com/567-labs/instructor/pull/1974)) -- **Cohere**: Streaming and V2 API version detection issues resolved ([#1983](https://github.com/567-labs/instructor/pull/1983), [#1844](https://github.com/567-labs/instructor/pull/1844)) -- **xAI**: Tools-mode validation fixed ([#1983](https://github.com/567-labs/instructor/pull/1983)) -- **Exception handling**: Standardized across all providers ([#1897](https://github.com/567-labs/instructor/pull/1897)) - -### Changed -- **Type checker**: Switched from Pyright to `ty` for faster incremental type checking ([#1978](https://github.com/567-labs/instructor/pull/1978)) -- **Provider factories**: `from_openai`, `from_anthropic`, etc. signatures standardized ([#1898](https://github.com/567-labs/instructor/pull/1898)) - ---- - -## [1.13.0] - 2025-11-03 - -### Added -- **Bedrock**: Image input support — converts OpenAI-style image parts to Bedrock's native format -- **`py.typed`**: Marker file restored for PEP 561 type-checking support ([#1868](https://github.com/567-labs/instructor/pull/1868)) - -### Fixed -- **`disable_pydantic_error_url()`**: Now correctly suppresses Pydantic validation error URLs via monkey-patching `ValidationError.__str__()` (environment variable approach had no effect post-import) -- **JSON mode**: JSON decode errors now trigger retry logic instead of surfacing as unhandled exceptions ([#1856](https://github.com/567-labs/instructor/pull/1856)) -- **Gemini**: Streaming fixed for the Google GenAI SDK ([#1864](https://github.com/567-labs/instructor/pull/1864)) -- **Gemini**: `HARM_CATEGORY_JAILBREAK` safety category and Anthropic `tool_result` content blocks now handled correctly ([#1867](https://github.com/567-labs/instructor/pull/1867)) -- **Partial**: Fields with `default_factory` no longer retain the factory when made optional during streaming -- **OpenAI**: Dependency version constraint updated to support v2 ([#1858](https://github.com/567-labs/instructor/pull/1858)) - ---- - -## [1.12.0] - 2025-10-27 - -### Fixed -- **Python 3.13**: Compatibility issues and import path corrections in multimodal processing -- **Bedrock**: OpenAI-compatible models now correctly parse responses where reasoning appears before text content -- **Gemini**: `chunk.text ValueError` when `finish_reason=1` no longer crashes streaming -- **Gemini**: `thinking_config` no longer unintentionally passed to the tools helper -- **OpenAI**: `parse:error` hook now correctly fires for `InstructorValidationError` -- **JSON parsing**: Broken regex patterns removed from JSON extraction function -- **Cohere**: V2 API version detection improved ([#1844](https://github.com/567-labs/instructor/pull/1844)) - ---- - -## [1.11.3] - 2025-09-04 - -### Added -- **Hooks**: Hook combination via `__add__` / `combine()` — merge multiple hook handlers together -- **Hooks**: Per-call hooks — pass hooks directly to individual `.create()` calls without registering globally -- **Retry**: `InstructorRetryException` now tracks all failed attempts including exceptions and raw completions for better introspection -- **Docs**: `llms.txt` support via `mkdocs-llmstxt` plugin for AI/LLM consumers - -### Fixed -- **`InstructorError.__str__()`**: Now correctly formats failed-attempt details -- **Retry**: Failed attempts propagated through reask handlers -- **Imports**: Backward compatibility imports restored for `function_calls` and `validators` modules - ---- - -## [1.11.1] - 2025-08-27 - -### Changed -- Upgraded all dependencies to latest versions - ---- - -## [1.11.0] - 2025-08-27 - -### Added -- **OpenRouter**: Provider support in `from_provider()` using `OPENROUTER_API_KEY` -- **LiteLLM**: Provider support in `from_provider()` ([#1723](https://github.com/567-labs/instructor/pull/1723)) -- **xAI**: Provider utilities following standard provider structure ([#1728](https://github.com/567-labs/instructor/pull/1728)) -- **Batch API**: In-memory batching support with improved error handling for OpenAI and Anthropic ([#1746](https://github.com/567-labs/instructor/pull/1746)) -- **Hooks**: `completion:error` and `completion:last_attempt` hooks now fully implemented ([#1729](https://github.com/567-labs/instructor/pull/1729)) - -### Changed -- Codebase reorganized from flat structure to modular provider-based architecture ([#1730](https://github.com/567-labs/instructor/pull/1730)) -- Provider-specific message conversion logic moved to dedicated handlers ([#1724](https://github.com/567-labs/instructor/pull/1724)) - -### Fixed -- Pydantic v2 deprecation warnings resolved by migrating from class `Config` to `ConfigDict` ([#1782](https://github.com/567-labs/instructor/pull/1782)) - diff --git a/참고/instructor-main/CLAUDE.md b/참고/instructor-main/CLAUDE.md deleted file mode 100644 index 14faf49..0000000 --- a/참고/instructor-main/CLAUDE.md +++ /dev/null @@ -1,303 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -# Instructor Development Guide - -## Commands -- Install deps: `uv pip install -e ".[dev,anthropic]"` or `poetry install --with dev,anthropic` -- Run tests: `uv run pytest tests/ -n auto` -- Run specific test: `uv run pytest tests/path_to_test.py::test_name` -- Skip LLM tests: `uv run pytest tests/ -k 'not llm and not openai'` -- Type check: `uv run ty check` -- Lint: `uv run ruff check instructor examples tests` -- Format: `uv run ruff format instructor examples tests` -- Generate coverage: `uv run coverage run -m pytest tests/ -k "not docs"` then `uv run coverage report` -- Build documentation: `uv run mkdocs serve` (for local preview) or `./build_mkdocs.sh` (for production) -- Waiting: use `sleep ` for explicit pauses (e.g., CI waits) or to let external processes finish - -## Installation & Setup -- Fork the repository and clone your fork -- Install UV: `pip install uv` -- Create virtual environment: `uv venv` -- Install dependencies: `uv pip install -e ".[dev]"` -- Install pre-commit: `uv run pre-commit install` -- Run tests to verify: `uv run pytest tests/ -k "not openai"` - -## Code Style Guidelines -- **Typing**: Use strict typing with annotations for all functions and variables -- **Imports**: Standard lib → third-party → local imports -- **Formatting**: Follow Black's formatting conventions (enforced by Ruff) -- **Models**: Define structured outputs as Pydantic BaseModel subclasses -- **Naming**: snake_case for functions/variables, PascalCase for classes -- **Error Handling**: Use custom exceptions from exceptions.py, validate with Pydantic -- **Comments**: Docstrings for public functions, inline comments for complex logic - -## Conventional Commits -- **Format**: `type(scope): description` -- **Types**: feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert -- **Examples**: - - `feat(anthropic): add support for Claude 3.5` - - `fix(openai): correct response parsing for streaming` - - `docs(README): update installation instructions` - - `test(gemini): add validation tests for JSON mode` - -## Core Architecture -- **Base Classes**: `Instructor` and `AsyncInstructor` in client.py are the foundation -- **Factory Pattern**: Provider-specific factory functions (`from_openai`, `from_anthropic`, etc.) -- **Unified Access**: `from_provider()` function in auto_client.py for automatic provider detection -- **Mode System**: `Mode` enum categorizes different provider capabilities (tools vs JSON output) -- **Patching Mechanism**: Uses Python's dynamic nature to patch provider clients for structured outputs -- **Response Processing**: Transforms raw API responses into validated Pydantic models -- **DSL Components**: Special types like Partial, Iterable, Maybe extend the core functionality - -## Provider Architecture -- **Supported Providers**: OpenAI, Anthropic, Gemini, Cohere, Mistral, Groq, VertexAI, Fireworks, Cerebras, Writer, Databricks, Anyscale, Together, LiteLLM, Bedrock, Perplexity -- **Provider Implementation**: Each provider has a dedicated client file (e.g., `client_anthropic.py`) with factory functions -- **Modes**: Different providers support specific modes (`Mode` enum): `ANTHROPIC_TOOLS`, `GEMINI_JSON`, etc. -- **Common Pattern**: Factory functions (e.g., `from_anthropic`) take a native client and return patched `Instructor` instances -- **Provider Testing**: Tests in `tests/llm/` directory, define Pydantic models, make API calls, verify structured outputs -- **Provider Detection**: `get_provider` function analyzes base URL to detect which provider is being used - -## Key Components -- **process_response.py**: Handles parsing and converting LLM outputs to Pydantic models -- **patch.py**: Contains the core patching logic for modifying provider clients -- **function_calls.py**: Handles generating function/tool schemas from Pydantic models -- **hooks.py**: Provides event hooks for intercepting various stages of the LLM request/response cycle -- **dsl/**: Domain-specific language extensions for specialized model types -- **retry.py**: Implements retry logic for handling validation failures -- **validators.py**: Custom validation mechanisms for structured outputs - -## Testing Guidelines -- Tests are organized by provider under `tests/llm/` -- Each provider has its own conftest.py with fixtures -- Standard tests cover: basic extraction, streaming, validation, retries -- Evaluation tests in `tests/llm/test_provider/evals/` assess model capabilities -- Use parametrized tests when testing similar functionality across variants -- **IMPORTANT**: No mocking in tests - tests make real API calls - -## Documentation Guidelines -- Every provider needs documentation in `docs/integrations/` following standard format -- Provider docs should include: installation, basic example, modes supported, special features -- When adding a new provider, update `mkdocs.yml` navigation and redirects -- Example code should include complete imports and environment setup -- Tutorials should progress from simple to complex concepts -- New features should include conceptual explanation in `docs/concepts/` -- **Writing Style**: Grade 10 reading level, all examples must be working code - -## Branch and Development Workflow -1. Fork and clone the repository -2. Create feature branch: `git checkout -b feat/your-feature` -3. Make changes and add tests -4. Run tests and linting -5. Commit with conventional commit message -6. Push to your fork and create PR -7. Use stacked PRs for complex features - -## Adding New Providers - -### Step-by-Step Guide -1. **Update Provider Enum** in `instructor/utils.py`: - ```python - class Provider(Enum): - YOUR_PROVIDER = "your_provider" - ``` - -2. **Add Provider Modes** in `instructor/mode.py`: - ```python - class Mode(enum.Enum): - YOUR_PROVIDER_TOOLS = "your_provider_tools" - YOUR_PROVIDER_JSON = "your_provider_json" - ``` - -3. **Create Client Implementation** `instructor/client_your_provider.py`: - - Use overloads for sync/async variants - - Validate mode compatibility - - Return appropriate Instructor/AsyncInstructor instance - - Handle provider-specific edge cases - -4. **Add Conditional Import** in `instructor/__init__.py`: - ```python - if importlib.util.find_spec("your_provider_sdk") is not None: - from .client_your_provider import from_your_provider - __all__ += ["from_your_provider"] - ``` - -5. **Update Auto Client** in `instructor/auto_client.py`: - - Add to `supported_providers` list - - Implement provider handling in `from_provider()` - - Update `get_provider()` function if URL-detectable - -6. **Create Tests** in `tests/llm/test_your_provider/`: - - `conftest.py` with client fixtures - - Basic extraction tests - - Streaming tests - - Validation/retry tests - - No mocking - use real API calls - -7. **Add Documentation** in `docs/integrations/your_provider.md`: - - Installation instructions - - Basic usage examples - - Supported modes - - Provider-specific features - -8. **Update Navigation** in `mkdocs.yml`: - - Add to integrations section - - Include redirects if needed - -## Contributing to Evals -- Standard evals for each provider test model capabilities -- Create new evals following existing patterns -- Run evals as part of integration test suite -- Performance tracking and comparison - -## Pull Request Guidelines -- Keep PRs small and focused -- Include tests for all changes -- Update documentation as needed -- Follow PR template -- Link to relevant issues -- **Update CHANGELOG.md**: Every PR that changes behavior (fix, feat, security, deprecation) must add an entry under the current `[Unreleased]` section in `CHANGELOG.md`. Format: `- **Area**: Description ([#PR](url))` - -## Type System and Best Practices - -### Type Checking with ty -- **Type Checker**: Using `ty` for fast, incremental type checking -- **Python Version**: 3.9+ for compatibility -- **Configuration**: Uses `pyproject.toml` settings for type checking -- Run `uv run ty check` before committing - aim for zero errors - -### Code Quality Checks Before Committing -Always run these checks before committing code: -1. **Ruff linting**: `uv run ruff check .` - Fix all errors -2. **Ruff formatting**: `uv run ruff format .` - Apply consistent formatting -3. **Type checking**: `uv run ty check` - Aim for zero type errors -4. **Tests**: Run relevant tests to ensure changes don't break functionality - -### Type Patterns -- **Bounded TypeVars**: Use `T = TypeVar("T", bound=Union[BaseModel, ...])` for constraints -- **Version Compatibility**: Handle Python 3.9 vs 3.10+ typing differences explicitly -- **Union Type Syntax**: Use `from __future__ import annotations` to enable Python 3.10+ union syntax (`|`) in Python 3.9 -- **Simple Type Detection**: Special handling for `list[Union[int, str]]` patterns -- **Runtime Type Handling**: Graceful fallbacks for compatibility - -### Pydantic Integration -- Heavy use of `BaseModel` for structured outputs -- `TypeAdapter` used internally for JSON schema generation -- Field validators and custom types -- Models serve dual purpose: validation and documentation - -## Building Documentation - -### Setup -```bash -# Install documentation dependencies -pip install -r requirements-doc.txt -``` - -### Local Development -```bash -# Serve documentation locally with hot reload -uv run mkdocs serve - -# Build documentation for production -./build_mkdocs.sh -``` - -### Documentation Features -- **Material Theme**: Modern UI with extensive customization -- **Plugins**: - - `mkdocstrings` - API documentation from docstrings - - `mkdocs-jupyter` - Notebook integration - - `mkdocs-redirects` - URL management - - Custom hooks for code processing -- **Custom Processing**: `hide_lines.py` removes code marked with `# <%hide%>` -- **Redirect Management**: Comprehensive redirect maps for moved content - -### Writing Documentation -- Follow templates in `docs/templates/` for consistency -- Grade 10 reading level for accessibility -- All code examples must be runnable -- Include complete imports and environment setup -- Progressive complexity: simple → advanced - -## Project Structure -- `instructor/` - Core library code - - Base classes (`client.py`): `Instructor` and `AsyncInstructor` - - Provider clients (`client_*.py`): Factory functions for each provider - - DSL components (`dsl/`): Partial, Iterable, Maybe, Citation extensions - - Core logic: `patch.py`, `process_response.py`, `function_calls.py` - - CLI tools (`cli/`): Batch processing, file management, usage tracking -- `tests/` - Test suite organized by provider - - Provider-specific tests in `tests/llm/test_/` - - Evaluation tests for model capabilities - - No mocking - all tests use real API calls -- `docs/` - MkDocs documentation - - `concepts/` - Core concepts and features - - `integrations/` - Provider-specific guides - - `examples/` - Practical examples and cookbooks - - `learning/` - Progressive tutorial path - - `blog/posts/` - Technical articles and announcements - - `templates/` - Templates for new docs (provider, concept, cookbook) -- `examples/` - Runnable code examples - - Feature demos: caching, streaming, validation, parallel processing - - Use cases: classification, extraction, knowledge graphs - - Provider examples: anthropic, openai, groq, mistral - - Each example has `run.py` as the main entry point -- `typings/` - Type stubs for untyped dependencies - -## Documentation Structure -- **Getting Started Path**: Installation → First Extraction → Response Models → Structured Outputs -- **Learning Patterns**: Simple Objects → Lists → Nested Structures → Validation → Streaming -- **Example Organization**: Self-contained directories with runnable code demonstrating specific features -- **Blog Posts**: Technical deep-dives with code examples in `docs/blog/posts/` - -## Example Patterns -When creating examples: -- Use `run.py` as the main file name -- Include clear imports: stdlib → third-party → instructor -- Define Pydantic models with descriptive fields -- Show expected output in comments -- Handle errors appropriately -- Make examples self-contained and runnable - -## Dependency Management - -### Core Dependencies -- **Minimal core**: `openai`, `pydantic`, `docstring-parser`, `typer`, `rich` -- **Python requirement**: `<4.0,>=3.9` -- **Pydantic version**: `<3.0.0,>=2.8.0` (constrained for stability) - -### Optional Dependencies -Provider-specific packages as extras: -```bash -# Install with specific provider -pip install "instructor[anthropic]" -pip install "instructor[google-generativeai]" -pip install "instructor[groq]" -``` - -### Development Dependencies -```bash -# Install all development dependencies -uv pip install -e ".[dev]" -``` -Includes: -- ty -- `pytest` and `pytest-asyncio` - Testing -- `ruff` - Linting and formatting -- `coverage` - Test coverage -- `mkdocs` and plugins - Documentation - -### Version Constraints -- **Upper bounds on all dependencies** for stability -- **Provider SDK versions** pinned to tested versions -- **Test dependencies** include evaluation frameworks - -### Managing Dependencies -- Update `pyproject.toml` for new dependencies -- Test with multiple Python versions (3.9-3.12) -- Run full test suite after dependency updates -- Document any provider-specific version requirements - -The library enables structured LLM outputs using Pydantic models across multiple providers with type safety. diff --git a/참고/instructor-main/CONTRIBUTING.md b/참고/instructor-main/CONTRIBUTING.md deleted file mode 100644 index dc7eb86..0000000 --- a/참고/instructor-main/CONTRIBUTING.md +++ /dev/null @@ -1,399 +0,0 @@ -# Contributing to Instructor - -Thank you for considering contributing to Instructor! This document provides guidelines and instructions to help you contribute effectively. - -## Table of Contents - -- [Contributing to Instructor](#contributing-to-instructor) - - [Table of Contents](#table-of-contents) - - [Code of Conduct](#code-of-conduct) - - [Getting Started](#getting-started) - - [Environment Setup](#environment-setup) - - [Development Workflow](#development-workflow) - - [Dependency Management](#dependency-management) - - [Using UV](#using-uv) - - [Using Poetry](#using-poetry) - - [Working with Optional Dependencies](#working-with-optional-dependencies) - - [How to Contribute](#how-to-contribute) - - [Reporting Bugs](#reporting-bugs) - - [Feature Requests](#feature-requests) - - [Pull Requests](#pull-requests) - - [Writing Documentation](#writing-documentation) - - [Contributing to Evals](#contributing-to-evals) - - [Code Style Guidelines](#code-style-guidelines) - - [Conventional Comments](#conventional-comments) - - [Conventional Commits](#conventional-commits) - - [Types](#types) - - [Examples](#examples) - - [Testing](#testing) - - [Branch and Release Process](#branch-and-release-process) - - [Using Cursor for PR Creation](#using-cursor-for-pr-creation) - - [License](#license) - -## Code of Conduct - -By participating in this project, you agree to abide by our code of conduct: treat everyone with respect, be constructive in your communication, and focus on the technical aspects of the contributions. - -## Getting Started - -### Environment Setup - -1. **Fork the Repository**: Click the "Fork" button at the top right of the [repository page](https://github.com/instructor-ai/instructor). - -2. **Clone Your Fork**: - ```bash - git clone https://github.com/YOUR-USERNAME/instructor.git - cd instructor - ``` - -3. **Set up Remote**: - ```bash - git remote add upstream https://github.com/instructor-ai/instructor.git - ``` - -4. **Install UV** (recommended): - ```bash - # macOS/Linux - curl -LsSf https://astral.sh/uv/install.sh | sh - - # Windows PowerShell - powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" - ``` - -5. **Install Dependencies**: - ```bash - # Using uv (recommended) - uv pip install -e ".[dev,docs,test-docs]" - - # Using poetry - poetry install --with dev,docs,test-docs - - # For specific providers, add the provider name as an extra - # Example: uv pip install -e ".[dev,docs,test-docs,anthropic]" - ``` - -6. **Set up Pre-commit**: - ```bash - pip install pre-commit - pre-commit install - ``` - -### Development Workflow - -1. **Create a Branch**: - ```bash - git checkout -b feature/your-feature-name - ``` - -2. **Make Your Changes and Commit**: - ```bash - git add . - git commit -m "Your descriptive commit message" - ``` - -3. **Keep Your Branch Updated**: - ```bash - git fetch upstream - git rebase upstream/main - ``` - -4. **Push Changes**: - ```bash - git push origin feature/your-feature-name - ``` - -### Dependency Management - -We support both UV and Poetry for dependency management. Choose the tool that works best for you: - -#### Using UV - -UV is a fast Python package installer and resolver. It's recommended for day-to-day development in Instructor. - -```bash -# Install uv -curl -LsSf https://astral.sh/uv/install.sh | sh - -# Install project and development dependencies -uv pip install -e ".[dev,docs]" - -# Adding a new dependency (example) -uv pip install new-package -``` - -Key UV commands: -- `uv pip install -e .` - Install the project in editable mode -- `uv pip install -e ".[dev]"` - Install with development extras -- `uv pip freeze > requirements.txt` - Generate requirements file -- `uv self update` - Update UV to the latest version - -#### Using Poetry - -Poetry provides more comprehensive dependency management and packaging. - -```bash -# Install Poetry -curl -sSL https://install.python-poetry.org | python3 - - -# Install dependencies including development deps -poetry install --with dev,docs - -# Add a new dependency -poetry add package-name - -# Add a new development dependency -poetry add --group dev package-name -``` - -Key Poetry commands: -- `poetry shell` - Activate the virtual environment -- `poetry run python -m pytest` - Run commands within the virtual environment -- `poetry update` - Update dependencies to their latest versions - -### Working with Optional Dependencies - -Instructor uses optional dependencies to support different LLM providers. Provider-specific utilities live under `instructor/utils`. When adding integration for a new provider: - -1. **Update pyproject.toml**: Add your provider's dependencies to both `[project.optional-dependencies]` and `[dependency-groups]`: - - ```toml - [project.optional-dependencies] - # Add your provider here - my-provider = ["my-provider-sdk>=1.0.0,<2.0.0"] - - [dependency-groups] - # Also add to dependency groups - my-provider = ["my-provider-sdk>=1.0.0,<2.0.0"] - ``` - -2. **Create Provider Client**: Implement your provider client in `instructor/clients/client_myprovider.py` - -3. **Add Tests**: Create tests in `tests/llm/test_myprovider/` - -4. **Document Installation**: Update the documentation to include installation instructions: - ``` - # Install with your provider support - uv pip install "instructor[my-provider]" - # or - poetry install --with my-provider - ``` - -5. **Create Provider Utilities and Handlers**: - - Add a new module at `instructor/utils/myprovider.py` - - Implement `reask` functions for validation errors and `handle_*` functions - for formatting requests - - Define `MYPROVIDER_HANDLERS` mapping `Mode` values to these functions - -6. **Register the Provider**: - - Add a value in `instructor/utils/providers.py` to the `Provider` enum - - Extend `get_provider` with detection logic for your base URL - -7. **Update `process_response.py`**: - - Import your handler functions and include them in the `mode_handlers` - dictionary so the library can route requests to your provider - - `process_response.py` relies on these handlers to format arguments and - parse results for each `Mode` - -## How to Contribute - -### Reporting Bugs - -If you find a bug, please create an issue on [our issue tracker](https://github.com/instructor-ai/instructor/issues) with: - -1. A clear, descriptive title -2. A detailed description including: - - The `response_model` you are using - - The `messages` you are using - - The `model` you are using - - Steps to reproduce the bug - - The expected behavior and what went wrong - - Your environment (Python version, OS, package versions) - -### Feature Requests - -For feature requests, please create an issue describing: - -1. The problem your feature would solve -2. How your solution would work -3. Alternatives you've considered -4. Examples of how the feature would be used - -### Pull Requests - -1. **Create a Pull Request** from your fork to the main repository. -2. **Fill out the PR template** with details about your changes. -3. **Address review feedback** and make requested changes. -4. **Wait for CI checks** to pass. -5. Once approved, a maintainer will merge your PR. - -### Writing Documentation - -Documentation improvements are always welcome! Follow these guidelines: - -1. Documentation is written in Markdown format in the `docs/` directory -2. When creating new markdown files, add them to `mkdocs.yml` under the appropriate section -3. Follow the existing hierarchy and structure -4. Use a grade 10 reading level (simple, clear language) -5. Include working code examples -6. Add links to related documentation - -### Contributing to Evals - -We encourage contributions to our evaluation tests: - -1. Explore existing evals in the [evals directory](https://github.com/instructor-ai/instructor/tree/main/tests/llm) -2. Contribute new evals as pytest tests -3. Evals should test specific capabilities or edge cases of the library or models -4. Follow the existing patterns for structuring eval tests - -## Code Style Guidelines - -We use automated tools to maintain consistent code style: - -- **Ruff**: For linting and formatting -- **ty**: For type checking -- **Black**: For code formatting (enforced by Ruff) - -General guidelines: - -- **Typing**: Use strict typing with annotations for all functions and variables -- **Imports**: Standard lib → third-party → local imports -- **Models**: Define structured outputs as Pydantic BaseModel subclasses -- **Naming**: snake_case for functions/variables, PascalCase for classes -- **Error Handling**: Use custom exceptions from exceptions.py, validate with Pydantic -- **Comments**: Docstrings for public functions, inline comments for complex logic - -### Conventional Comments - -We use conventional comments in code reviews and commit messages. This helps make feedback clearer and more actionable: - -``` -
- - - - - - - - -
Without InstructorWith Instructor
- -```python -response = openai.chat.completions.create( - model="gpt-4", - messages=[{"role": "user", "content": "..."}], - tools=[ - { - "type": "function", - "function": { - "name": "extract_user", - "parameters": { - "type": "object", - "properties": { - "name": {"type": "string"}, - "age": {"type": "integer"}, - }, - }, - }, - } - ], -) - -# Parse response -tool_call = response.choices[0].message.tool_calls[0] -user_data = json.loads(tool_call.function.arguments) - -# Validate manually -if "name" not in user_data: - # Handle error... - pass -``` - - - -```python -client = instructor.from_provider("openai/gpt-4") - -user = client.chat.completions.create( - response_model=User, - messages=[{"role": "user", "content": "..."}], -) - -# That's it! user is validated and typed -``` - -
- -## Install in seconds - -```bash -pip install instructor -``` - -Or with your package manager: -```bash -uv add instructor -poetry add instructor -``` - -## Works with every major provider - -Use the same code with any LLM provider: - -```python -# OpenAI -client = instructor.from_provider("openai/gpt-4o") - -# Anthropic -client = instructor.from_provider("anthropic/claude-3-5-sonnet") - -# Google -client = instructor.from_provider("google/gemini-pro") - -# Ollama (local) -client = instructor.from_provider("ollama/llama3.2") - -# With API keys directly (no environment variables needed) -client = instructor.from_provider("openai/gpt-4o", api_key="sk-...") -client = instructor.from_provider("anthropic/claude-3-5-sonnet", api_key="sk-ant-...") -client = instructor.from_provider("groq/llama-3.1-8b-instant", api_key="gsk_...") - -# All use the same API! -user = client.chat.completions.create( - response_model=User, - messages=[{"role": "user", "content": "..."}], -) -``` - -## Production-ready features - -### Automatic retries - -Failed validations are automatically retried with the error message: - -```python -from pydantic import BaseModel, field_validator - - -class User(BaseModel): - name: str - age: int - - @field_validator('age') - def validate_age(cls, v): - if v < 0: - raise ValueError('Age must be positive') - return v - - -# Instructor automatically retries when validation fails -user = client.chat.completions.create( - response_model=User, - messages=[{"role": "user", "content": "..."}], - max_retries=3, -) -``` - -### Streaming support - -Stream partial objects as they're generated: - -```python -from instructor import Partial - -for partial_user in client.chat.completions.create( - response_model=Partial[User], - messages=[{"role": "user", "content": "..."}], - stream=True, -): - print(partial_user) - # User(name=None, age=None) - # User(name="John", age=None) - # User(name="John", age=25) -``` - -### Nested objects - -Extract complex, nested data structures: - -```python -from typing import List - - -class Address(BaseModel): - street: str - city: str - country: str - - -class User(BaseModel): - name: str - age: int - addresses: List[Address] - - -# Instructor handles nested objects automatically -user = client.chat.completions.create( - response_model=User, - messages=[{"role": "user", "content": "..."}], -) -``` - -## Used in production by - -Trusted by over 100,000 developers and companies building AI applications: - -- **3M+ monthly downloads** -- **10K+ GitHub stars** -- **1000+ community contributors** - -Companies using Instructor include teams at OpenAI, Google, Microsoft, AWS, and many YC startups. - -## Get started - -### Basic extraction - -Extract structured data from any text: - -```python -from pydantic import BaseModel -import instructor - -client = instructor.from_provider("openai/gpt-4o-mini") - - -class Product(BaseModel): - name: str - price: float - in_stock: bool - - -product = client.chat.completions.create( - response_model=Product, - messages=[{"role": "user", "content": "iPhone 15 Pro, $999, available now"}], -) - -print(product) -# Product(name='iPhone 15 Pro', price=999.0, in_stock=True) -``` - -### Multiple languages - -Instructor's simple API is available in many languages: - -- [Python](https://python.useinstructor.com) - The original -- [TypeScript](https://js.useinstructor.com) - Full TypeScript support -- [Ruby](https://ruby.useinstructor.com) - Ruby implementation -- [Go](https://go.useinstructor.com) - Go implementation -- [Elixir](https://hex.pm/packages/instructor) - Elixir implementation -- [Rust](https://rust.useinstructor.com) - Rust implementation - -### Learn more - -- [Documentation](https://python.useinstructor.com) - Comprehensive guides -- [Examples](https://python.useinstructor.com/examples/) - Copy-paste recipes -- [Blog](https://python.useinstructor.com/blog/) - Tutorials and best practices -- [Discord](https://discord.gg/bD9YE9JArw) - Get help from the community - -## Why use Instructor over alternatives? - -**vs Raw JSON mode**: Instructor provides automatic validation, retries, streaming, and nested object support. No manual schema writing. - -**vs LangChain/LlamaIndex**: Instructor is focused on one thing - structured extraction. It's lighter, faster, and easier to debug. - -**vs Custom solutions**: Battle-tested by thousands of developers. Handles edge cases you haven't thought of yet. - -## Contributing - -We welcome contributions! Check out our [good first issues](https://github.com/567-labs/instructor/labels/good%20first%20issue) to get started. - -## License - -MIT License - see [LICENSE](https://github.com/567-labs/instructor/blob/main/LICENSE) for details. - ---- - -

-Built by the Instructor community. Special thanks to Jason Liu and all contributors. -

\ No newline at end of file diff --git a/참고/instructor-main/build_mkdocs.sh b/참고/instructor-main/build_mkdocs.sh deleted file mode 100644 index fe85a19..0000000 --- a/참고/instructor-main/build_mkdocs.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env sh -set -eu - -if ! command -v uv >/dev/null 2>&1; then - pipx install uv -fi - -uv sync --python 3.13 --extra docs -uv run mkdocs build diff --git a/참고/instructor-main/cross_link_mapping.yaml b/참고/instructor-main/cross_link_mapping.yaml deleted file mode 100644 index de60583..0000000 --- a/참고/instructor-main/cross_link_mapping.yaml +++ /dev/null @@ -1,316 +0,0 @@ -# Cross-Link Mapping for Instructor Documentation -# This file maps blog posts and documentation pages to their related content -# Format: -# source_file: -# related_concepts: [list of concept docs to link] -# related_blog_posts: [list of related blog posts] -# related_examples: [list of example files] -# related_integrations: [list of integration docs] -# see_also_text: "Custom text for See Also section" - -# VALIDATION CLUSTER -blog/posts/validation-part1.md: - related_concepts: - - concepts/validation.md - - concepts/reask_validation.md - related_blog_posts: - - blog/posts/semantic-validation-structured-outputs.md - - blog/posts/bad-schemas-could-break-llms.md - - blog/posts/pydantic-is-still-all-you-need.md - related_examples: - - examples/validators.md - see_also_text: | - ## Related Documentation - - [Core Validation Concepts](/concepts/validation) - Learn about validation fundamentals - - [Reask Validation](/concepts/reask_validation) - Handle validation failures gracefully - - ## See Also - - [Semantic Validation with Structured Outputs](semantic-validation-structured-outputs) - Next evolution in validation - - [Why Bad Schemas Break LLMs](bad-schemas-could-break-llms) - Schema design best practices - - [Pydantic Is Still All You Need](pydantic-is-still-all-you-need) - Why Pydantic validation matters - -blog/posts/semantic-validation-structured-outputs.md: - related_concepts: - - concepts/validation.md - - concepts/llm_validation.md - related_blog_posts: - - blog/posts/validation-part1.md - - blog/posts/anthropic-prompt-caching.md - - blog/posts/logfire.md - related_examples: - - examples/moderation.md - see_also_text: | - ## Related Documentation - - [Validation Fundamentals](/concepts/validation) - Core validation concepts - - [LLM Validation](/concepts/llm_validation) - Using LLMs for validation - - ## See Also - - [Validation Deep Dive](validation-part1) - Foundation validation concepts - - [Anthropic Prompt Caching](anthropic-prompt-caching) - Optimize validation costs - - [Monitoring with Logfire](logfire) - Track validation performance - -blog/posts/pydantic-is-still-all-you-need.md: - related_concepts: - - concepts/philosophy.md - - concepts/validation.md - related_blog_posts: - - blog/posts/validation-part1.md - - blog/posts/best_framework.md - - blog/posts/introduction.md - related_integrations: - - integrations/index.md - see_also_text: | - ## Related Documentation - - [Instructor Philosophy](/concepts/philosophy) - Why we chose Pydantic - - [Validation Guide](/concepts/validation) - Practical validation techniques - - ## See Also - - [Validation Deep Dive](validation-part1) - Advanced validation patterns - - [Best Framework Comparison](best_framework) - Why Instructor stands out - - [Introduction to Instructor](introduction) - Getting started guide - -# MULTIMODAL CLUSTER -blog/posts/multimodal-gemini.md: - related_concepts: - - concepts/multimodal.md - - concepts/images.md - related_blog_posts: - - blog/posts/openai-multimodal.md - - blog/posts/structured-output-anthropic.md - - blog/posts/chat-with-your-pdf-with-gemini.md - related_integrations: - - integrations/google.md - - integrations/vertex.md - related_examples: - - examples/image_to_ad_copy.md - see_also_text: | - ## Related Documentation - - [Multimodal Concepts](/concepts/multimodal) - Working with images, video, and audio - - [Image Processing](/concepts/images) - Image-specific techniques - - [Google Integration](/integrations/google) - Complete Gemini setup guide - - ## See Also - - [OpenAI Multimodal](openai-multimodal) - Compare multimodal approaches - - [Anthropic Structured Output](structured-output-anthropic) - Alternative provider - - [Chat with PDFs using Gemini](chat-with-your-pdf-with-gemini) - Practical PDF processing - -blog/posts/openai-multimodal.md: - related_concepts: - - concepts/multimodal.md - - concepts/images.md - related_blog_posts: - - blog/posts/multimodal-gemini.md - - blog/posts/anthropic-prompt-caching.md - - blog/posts/logfire.md - related_integrations: - - integrations/openai.md - related_examples: - - examples/audio.md - see_also_text: | - ## Related Documentation - - [Multimodal Guide](/concepts/multimodal) - Comprehensive multimodal reference - - [OpenAI Integration](/integrations/openai) - Full OpenAI setup - - ## See Also - - [Gemini Multimodal](multimodal-gemini) - Alternative multimodal approach - - [Prompt Caching](anthropic-prompt-caching) - Cache large audio files - - [Monitoring with Logfire](logfire) - Track multimodal processing - -blog/posts/chat-with-your-pdf-with-gemini.md: - related_concepts: - - concepts/multimodal.md - related_blog_posts: - - blog/posts/multimodal-gemini.md - - blog/posts/generating-pdf-citations.md - - blog/posts/rag-and-beyond.md - related_examples: - - examples/pdf_to_markdown.md - see_also_text: | - ## Related Documentation - - [Multimodal Processing](/concepts/multimodal) - Core multimodal concepts - - ## See Also - - [Gemini Multimodal Features](multimodal-gemini) - Full Gemini capabilities - - [PDF Citation Generation](generating-pdf-citations) - Extract citations from PDFs - - [RAG and Beyond](rag-and-beyond) - Advanced document processing - -# PROVIDER INTEGRATION CLUSTER -blog/posts/structured-output-anthropic.md: - related_concepts: - - concepts/patching.md - related_blog_posts: - - blog/posts/anthropic-prompt-caching.md - - blog/posts/announcing-unified-provider-interface.md - - blog/posts/best_framework.md - related_integrations: - - integrations/anthropic.md - related_examples: - - examples/classification.md - see_also_text: | - ## Related Documentation - - [How Patching Works](/concepts/patching) - Understand provider integration - - [Anthropic Integration](/integrations/anthropic) - Complete setup guide - - ## See Also - - [Anthropic Prompt Caching](anthropic-prompt-caching) - Optimize Anthropic costs - - [Unified Provider Interface](announcing-unified-provider-interface) - Switch providers easily - - [Framework Comparison](best_framework) - Why Instructor excels - -blog/posts/anthropic-prompt-caching.md: - related_concepts: - - concepts/caching.md - related_blog_posts: - - blog/posts/structured-output-anthropic.md - - blog/posts/caching.md - - blog/posts/logfire.md - related_integrations: - - integrations/anthropic.md - see_also_text: | - ## Related Documentation - - [Caching Strategies](/concepts/caching) - General caching concepts - - [Anthropic Integration](/integrations/anthropic) - Full Anthropic guide - - ## See Also - - [Anthropic Structured Outputs](structured-output-anthropic) - Use with caching - - [Response Caching](caching) - General caching strategies - - [Performance Monitoring](logfire) - Track cache performance - -blog/posts/announcing-unified-provider-interface.md: - related_concepts: - - concepts/patching.md - - concepts/philosophy.md - related_blog_posts: - - blog/posts/string-based-init.md - - blog/posts/best_framework.md - - blog/posts/introduction.md - related_integrations: - - integrations/index.md - related_examples: - - examples/groq.md - - examples/mistral.md - see_also_text: | - ## Related Documentation - - [Provider Patching](/concepts/patching) - How provider integration works - - [All Integrations](/integrations/) - Supported provider list - - ## See Also - - [String-Based Initialization](string-based-init) - Alternative init method - - [Framework Comparison](best_framework) - Multi-provider advantages - - [Getting Started](introduction) - Quick start guide - -# RAG AND SEARCH CLUSTER -blog/posts/rag-and-beyond.md: - related_concepts: - - concepts/validation.md - related_blog_posts: - - blog/posts/llm-as-reranker.md - - blog/posts/citations.md - - blog/posts/chat-with-your-pdf-with-gemini.md - related_examples: - - examples/search.md - see_also_text: | - ## Related Documentation - - [Validation Concepts](/concepts/validation) - Validate RAG outputs - - ## See Also - - [LLM as Reranker](llm-as-reranker) - Improve search relevance - - [Citation Extraction](citations) - Verify sources - - [PDF Processing](chat-with-your-pdf-with-gemini) - Document handling - -blog/posts/llm-as-reranker.md: - related_blog_posts: - - blog/posts/rag-and-beyond.md - - blog/posts/validation-part1.md - - blog/posts/logfire.md - related_examples: - - examples/reranking.md - see_also_text: | - ## See Also - - [RAG and Beyond](rag-and-beyond) - Comprehensive RAG guide - - [Validation Fundamentals](validation-part1) - Validate ranking scores - - [Performance Monitoring](logfire) - Track reranking performance - -blog/posts/citations.md: - related_concepts: - - concepts/validation.md - related_blog_posts: - - blog/posts/rag-and-beyond.md - - blog/posts/generating-pdf-citations.md - - blog/posts/validation-part1.md - see_also_text: | - ## Related Documentation - - [Validation Guide](/concepts/validation) - Validate citations - - ## See Also - - [RAG Techniques](rag-and-beyond) - Use citations in RAG - - [PDF Citations](generating-pdf-citations) - Extract from PDFs - - [Validation Basics](validation-part1) - Ensure citation quality - -# PERFORMANCE AND MONITORING -blog/posts/logfire.md: - related_concepts: - - concepts/retrying.md - related_blog_posts: - - blog/posts/full-fastapi-visibility.md - - blog/posts/anthropic-prompt-caching.md - - blog/posts/validation-part1.md - related_integrations: - - integrations/pydantic_logfire.md - see_also_text: | - ## Related Documentation - - [Retry Mechanisms](/concepts/retrying) - Handle failures gracefully - - [Logfire Integration](/integrations/pydantic_logfire) - Setup guide - - ## See Also - - [FastAPI Visibility](full-fastapi-visibility) - Web app monitoring - - [Prompt Caching](anthropic-prompt-caching) - Monitor cache hits - - [Validation Monitoring](validation-part1) - Track validation metrics - -blog/posts/caching.md: - related_concepts: - - concepts/caching.md - related_blog_posts: - - blog/posts/anthropic-prompt-caching.md - - blog/posts/logfire.md - see_also_text: | - ## Related Documentation - - [Caching Concepts](/concepts/caching) - Core caching strategies - - ## See Also - - [Anthropic Prompt Caching](anthropic-prompt-caching) - Provider-specific caching - - [Performance Monitoring](logfire) - Track cache effectiveness - -# GETTING STARTED AND PHILOSOPHY -blog/posts/introduction.md: - related_concepts: - - concepts/philosophy.md - - concepts/quickstart.md - related_blog_posts: - - blog/posts/best_framework.md - - blog/posts/pydantic-is-still-all-you-need.md - - blog/posts/announcing-unified-provider-interface.md - see_also_text: | - ## Related Documentation - - [Quick Start Guide](/concepts/quickstart) - Get running in minutes - - [Philosophy](/concepts/philosophy) - Why we built Instructor - - ## See Also - - [Framework Comparison](best_framework) - See how we compare - - [Why Pydantic](pydantic-is-still-all-you-need) - Our foundation - - [Easy Provider Setup](announcing-unified-provider-interface) - Start with any LLM - -blog/posts/best_framework.md: - related_concepts: - - concepts/philosophy.md - related_blog_posts: - - blog/posts/introduction.md - - blog/posts/pydantic-is-still-all-you-need.md - - blog/posts/announcing-unified-provider-interface.md - see_also_text: | - ## Related Documentation - - [Our Philosophy](/concepts/philosophy) - Design principles - - ## See Also - - [Getting Started](introduction) - Quick introduction - - [Pydantic Foundation](pydantic-is-still-all-you-need) - Why Pydantic - - [Multi-Provider Support](announcing-unified-provider-interface) - Key differentiator \ No newline at end of file diff --git a/참고/instructor-main/docs/AGENT.md b/참고/instructor-main/docs/AGENT.md deleted file mode 100644 index 7bca921..0000000 --- a/참고/instructor-main/docs/AGENT.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -title: Documentation Agent Guide -description: Internal guide for maintaining and improving Instructor documentation ---- - -# AGENT.md - Documentation - -## Commands -- Serve docs locally: `uv run mkdocs serve` -- Build docs: `./build_mkdocs.sh` or `uv run mkdocs build` -- Install doc deps: `uv pip install -e ".[docs]"` -- Test examples: `uv run pytest docs/ --examples` - -## Structure -- **Core docs**: `concepts/`, `integrations/`, `examples/` -- **Learning path**: `getting-started.md` → `learning/` → `tutorials/` -- **API reference**: Auto-generated from docstrings via `mkdocstrings` -- **Blog**: `blog/posts/` for announcements and deep-dives -- **Templates**: `templates/` for new docs (provider, concept, cookbook) - -## Writing Guidelines -- **Reading level**: Grade 10 (from .cursor/rules) -- **Code examples**: Must be runnable with complete imports -- **Progressive complexity**: Simple → advanced concepts -- **Provider docs**: Follow `templates/` patterns -- **Navigation**: Update `mkdocs.yml` for new pages - -## Pull Request (PR) Formatting - -Use **Conventional Commits** formatting for PR titles so they are consistent and easy to scan. Treat the PR title as the message we would use for a squash merge commit. - -### PR Title Format - -Use: - -`(): ` - -Rules: -- Keep it under ~70 characters when you can. -- Use the imperative mood (for example, “add”, “fix”, “update”). -- Do not end with a period. -- If it includes a breaking change, add `!` after the type or scope (for example, `feat(docs)!:`). - -Good examples: -- `docs(agents): add conventional commit PR title guidelines` -- `docs(mkdocs): fix broken link in validation tutorial` -- `docs(examples): update youtube clips snippet` -- `chore(docs): refresh docs build commands` - -Common types: -- `docs`: documentation-only changes -- `fix`: bug fix -- `feat`: new feature -- `test`: add or update tests -- `chore`: maintenance work (build scripts, tooling, repo hygiene) -- `ci`: CI pipeline changes - -Suggested docs scopes: -- `docs`, `mkdocs`, `blog`, `examples`, `integrations`, `tutorials`, `agents` - -### PR Description Guidelines - -Keep PR descriptions short and actionable: -- **What**: What changed, in 1–3 sentences. -- **Why**: Why this change is needed (link issues when possible). -- **Changes**: 3–7 bullet points with the main edits. -- **Testing**: What you ran (or why you did not run anything). -- **Docs impact**: Call out page moves, redirects, or nav updates. - -If the PR was authored by Cursor, include: -- `This PR was written by [Cursor](https://cursor.com)` - -## Key Files -- `mkdocs.yml` - Site configuration and navigation -- `hooks/` - Custom processing (hide_lines.py removes `# <%hide%>` markers) -- `overrides/` - Custom theme elements -- `javascripts/` - Client-side enhancements diff --git a/참고/instructor-main/docs/api-docstring-assessment.md b/참고/instructor-main/docs/api-docstring-assessment.md deleted file mode 100644 index 751b9bf..0000000 --- a/참고/instructor-main/docs/api-docstring-assessment.md +++ /dev/null @@ -1,150 +0,0 @@ -# API Docstring Quality Assessment - -This document assesses the quality and completeness of docstrings for all API items referenced in the expanded API documentation. - -## Summary - -Overall, the docstring quality is **good to excellent** for most items. Many classes and functions have comprehensive docstrings with usage examples, while some core classes could benefit from class-level docstrings. - -## Excellent Docstrings (Comprehensive with Examples) - -These have detailed docstrings with usage examples and clear descriptions: - -### Client Creation -- **`from_provider`** - Comprehensive docstring with Args, Returns, Raises, and Examples sections. Includes multiple usage examples showing basic usage, caching, and async clients. - -### Validation -- **`llm_validator`** - Good docstring with usage examples, parameter descriptions, and error message examples showing how validation errors are formatted. - -### DSL Components -- **`CitationMixin`** - Excellent docstring with complete usage examples showing how to use it with context, and result examples showing the output structure. -- **`IterableModel`** - Good docstring with usage examples showing before/after transformation, Parameters section, and Returns description. -- **`Maybe`** - Good docstring with usage examples and result structure showing the generated model fields. - -### Batch Processing -- **`BatchProcessor`** - Good class-level docstring explaining the unified interface. Methods like `create_batch_from_messages` and `submit_batch` have clear Args and Returns sections. - -### Distillation -- **`Instructions`** - Good docstring with parameter descriptions. The `distil` method has usage examples showing decorator usage patterns. - -### Hooks -- **`Hooks`** - Excellent class-level docstring explaining the purpose. Methods like `on()`, `get_hook_name()`, `emit()`, etc. have comprehensive docstrings with Args, Returns, Raises, and Examples sections. - -### Schema Generation -- **`generate_openai_schema`** - Good docstring with Args, Returns, and Notes sections explaining how docstrings are used. -- **`generate_anthropic_schema`** - Has docstring explaining the conversion process. - -### Multimodal -- **`Audio`** - Good class-level docstring. Methods like `autodetect()` and `autodetect_safely()` have clear docstrings with Args and Returns. - -### Exceptions -- **`InstructorError`** - Excellent docstring with Attributes section, Examples showing error handling, and See Also references. -- **`IncompleteOutputException`** - Good docstring with Attributes, Common Solutions, and Examples. -- **`InstructorRetryException`** - Comprehensive docstring with Attributes, Common Causes, Examples, and See Also. -- **`ValidationError`** - Good docstring with Examples and See Also. -- **`ProviderError`** - Good docstring with Attributes, Common Causes, and Examples. -- **`ConfigurationError`** - Good docstring with Common Scenarios and Examples. -- **`ModeError`** - Good docstring with Attributes, Examples, and See Also. -- **`ClientError`** - Good docstring with Common Scenarios and Examples. -- **`AsyncValidationError`** - Good docstring with Attributes and Examples. -- **`ResponseParsingError`** - Good docstring with Attributes, Examples, and backwards compatibility notes. -- **`MultimodalError`** - Good docstring with Attributes, Examples, and backwards compatibility notes. - -## Good Docstrings (Clear but Could Be Enhanced) - -These have adequate docstrings but could benefit from more examples or additional detail: - -### Core Clients -- **`Instructor`** - No class-level docstring. Methods have type hints but lack comprehensive docstrings. The class is well-documented through usage in examples, but a class-level docstring would help. -- **`AsyncInstructor`** - Similar to `Instructor`, no class-level docstring. -- **`Response`** - No class-level docstring. Methods like `create()` and `create_with_completion()` lack docstrings. - -### Client Creation -- **`from_openai`** - No docstring. Only has type overloads. The implementation exists but lacks documentation explaining usage, parameters, and return values. - -### Function Calls & Schema -- **`OpenAISchema`** - Good method docstrings for `openai_schema`, `anthropic_schema`, `gemini_schema`, and `from_response()`. The class itself could use a class-level docstring explaining its purpose and usage. -- **`openai_schema`** - Decorator function, but the docstring is on the class method, not the decorator itself. - -### DSL Components -- **`Partial`** - Minimal docstring. Has Notes and Example sections but could benefit from more comprehensive usage examples showing streaming scenarios. - -### Multimodal -- **`Image`** - No class-level docstring. Methods have good docstrings (`autodetect()`, `autodetect_safely()`, `from_gs_url()`, etc.), but the class itself lacks documentation. - -### Mode & Provider -- **`Mode`** - Good class-level docstring explaining what modes are and how they work. Individual mode values lack docstrings but the enum docstring is comprehensive. -- **`Provider`** - No class-level docstring. Just enum values without explanation. - -### Patch Functions -- **`patch`** - Good docstring explaining what features it enables (response_model, max_retries, validation_context, strict, hooks). Could benefit from usage examples. -- **`apatch`** - Need to check if it has similar docstring quality. - -## Areas Needing Improvement - -### Missing Class-Level Docstrings -1. **`Instructor`** - Should have a class-level docstring explaining: - - What the class does - - How to use it - - Key features (modes, hooks, retries) - - Basic usage example - -2. **`AsyncInstructor`** - Should have a class-level docstring explaining: - - Async usage patterns - - How it differs from `Instructor` - - Async examples - -3. **`Response`** - Should have a class-level docstring explaining: - - What the Response helper does - - When to use it vs direct client methods - - Usage examples - -4. **`Image`** - Should have a class-level docstring explaining: - - What Image represents - - Supported formats - - Common usage patterns - -5. **`Provider`** - Should have a class-level docstring explaining: - - What providers are supported - - How to use Provider enum - - Provider detection - -### Missing Function Docstrings -1. **`from_openai`** - Needs comprehensive docstring with: - - Purpose and usage - - Parameters explanation - - Return value description - - Examples - -2. **`from_litellm`** - No docstring. Only has type overloads. Similar to `from_openai`, needs comprehensive docstring. - -### Could Be Enhanced -1. **`Partial`** - Could add more streaming examples -2. **`patch`** - Could add usage examples showing before/after -3. **`apatch`** - Has docstring but marked as deprecated ("No longer necessary, use `patch` instead"). Docstring is adequate but the deprecation should be more prominent. -4. **`openai_schema`** - Has minimal docstring. Could expand with usage examples showing how to use the decorator. - -## Recommendations - -### High Priority -1. Add class-level docstrings to `Instructor` and `AsyncInstructor` - These are the core classes users interact with -2. Add docstring to `from_openai` - Important client creation function -3. Add class-level docstring to `Response` - Helper class that needs explanation - -### Medium Priority -1. Add class-level docstring to `Image` - Commonly used multimodal class -2. Add class-level docstring to `Provider` - Enum that could use explanation -3. Enhance `Partial` docstring with more streaming examples - -### Low Priority -1. Add more examples to `patch` docstring -2. Expand `openai_schema` docstring with examples -3. Consider updating `apatch` deprecation message to be more prominent - -## Overall Assessment - -**Grade: B+** - -The documentation is generally good with many excellent examples, but the core classes (`Instructor`, `AsyncInstructor`, `Response`) would benefit significantly from class-level docstrings. The DSL components and utility functions are well-documented, and the exception classes have comprehensive docstrings. - -The mkdocs autodoc plugin will generate API documentation from these docstrings, so improving them will directly improve the generated API reference pages. diff --git a/참고/instructor-main/docs/api.md b/참고/instructor-main/docs/api.md deleted file mode 100644 index e189001..0000000 --- a/참고/instructor-main/docs/api.md +++ /dev/null @@ -1,138 +0,0 @@ ---- -title: API Reference Guide -description: Explore the comprehensive API reference with details on instructors, validation, iteration, and function calls. ---- - -# API Reference - -Core modes are the recommended default. Legacy provider-specific modes still -work but are deprecated and will show warnings. See the -[Mode Migration Guide](concepts/mode-migration.md) for details. - -## Core Clients - -The main client classes for interacting with LLM providers. - -::: instructor.Instructor - -::: instructor.AsyncInstructor - -::: instructor.core.client.Response - -## Client Creation - -Functions to create Instructor clients from various providers. - -::: instructor.from_provider - -::: instructor.from_openai - -::: instructor.from_litellm - -## DSL Components - -Domain-specific language components for advanced patterns and data handling. - -::: instructor.dsl.validators - -::: instructor.dsl.iterable - -::: instructor.dsl.partial - -::: instructor.dsl.parallel - -::: instructor.dsl.maybe - -::: instructor.dsl.citation - -## Function Calls & Schema - -Classes and functions for defining and working with function call schemas. - -::: instructor.function_calls - -::: instructor.OpenAISchema - -::: instructor.openai_schema - -::: instructor.generate_openai_schema - -::: instructor.generate_anthropic_schema - -::: instructor.generate_gemini_schema - -## Validation - -Validation utilities for LLM outputs and async validation support. - -::: instructor.validation - -::: instructor.llm_validator - -::: instructor.openai_moderation - -## Batch Processing - -Batch processing utilities for handling multiple requests efficiently. - -::: instructor.batch - -::: instructor.batch.BatchProcessor - -::: instructor.batch.BatchRequest - -::: instructor.batch.BatchJob - -## Distillation - -Tools for distillation and fine-tuning workflows. - -::: instructor.distil - -::: instructor.FinetuneFormat - -::: instructor.Instructions - -## Multimodal - -Support for image and audio content in LLM requests. - -::: instructor.processing.multimodal - -::: instructor.Image - -::: instructor.Audio - -## Mode & Provider - -Enumerations for modes and providers. - -::: instructor.Mode - -::: instructor.Provider - -## Exceptions - -Exception classes for error handling. - -::: instructor.core.exceptions - -## Hooks - -Event hooks system for monitoring and intercepting LLM interactions. - -::: instructor.core.hooks - -::: instructor.core.hooks.Hooks - -::: instructor.core.hooks.HookName - -## Patch Functions - -Decorators for patching LLM client methods. - -::: instructor.core.patch - -::: instructor.patch - -::: instructor.apatch diff --git a/참고/instructor-main/docs/architecture.md b/참고/instructor-main/docs/architecture.md deleted file mode 100644 index a8a3a1f..0000000 --- a/참고/instructor-main/docs/architecture.md +++ /dev/null @@ -1,177 +0,0 @@ ---- -title: Instructor Architecture Overview -description: Learn about the internal architecture and design decisions of the Instructor library ---- - -# Architecture Overview - -This page explains the core execution flow and where to plug in or debug. It highlights the minimal sync/async code paths and how streaming, partial, and parallel modes integrate. - -## High-Level Flow - -```mermaid -sequenceDiagram - autonumber - participant U as User Code - participant I as Instructor (patched) - participant R as Retry Layer (tenacity) - participant C as Provider Client - participant D as Dispatcher (process_response) - participant H as Provider Handler (response/reask) - participant M as Pydantic Model - - U->>I: chat.completions.create(response_model=..., **kwargs) - Note right of I: patch() wraps create() with cache/templating and retry - I->>R: retry_sync/async(func=create, max_retries, strict, mode, hooks) - loop attempts - R->>C: create(**prepared_kwargs) - C-->>R: raw response (provider-specific) - R->>D: process_response(_async)(response, response_model, mode, stream) - alt Streaming/Partial - D->>M: Iterable/Partial.from_streaming_response(_async) - D-->>R: Iterable/Partial model (or list of items) - else Standard - D->>H: provider mode handler (format/parse selection) - H-->>D: adjusted response_model/new_kwargs if needed - D->>M: response_model.from_response(...) - M-->>D: parsed model (with _raw_response attached) - D-->>R: model (or adapted simple type) - end - R-->>I: parsed model - end - I-->>U: final model (plus _raw_response on instance) - - rect rgb(255,240,240) - Note over R,H: On validation/JSON errors → reask path - R->>H: handle_reask_kwargs(..., exception, failed_attempts) - H-->>R: new kwargs/messages for next attempt - end -``` - -Key responsibilities: -- patch(): wraps the provider `create` with cache lookup/save, templating, strict mode, hooks, and retry. -- Retry: executes provider call, emits hooks, updates usage, handles validation/JSON errors with reask, and re-attempts. -- Dispatcher: selects the correct parsing path by `Mode`, handles multimodal message conversion, and attaches `_raw_response` to the returned model. -- Provider Handlers: provider/mode-specific request shaping and reask preparation. - -## Minimal Code Paths - -### Synchronous -```python -import openai -import instructor -from pydantic import BaseModel - -class User(BaseModel): - name: str - age: int - -client = instructor.from_provider("openai/gpt-5-nano") - -model = client.create( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "{'name': 'Ada', 'age': 37}"}], - response_model=User, # triggers schema/tool wiring + parsing - max_retries=3, # tenacity-backed validation retries - strict=True, # strict JSON parsing if supported -) - -# Access raw provider response if needed -raw = model._raw_response -``` - -### Asynchronous -```python -import asyncio -import openai -import instructor -from pydantic import BaseModel - -class User(BaseModel): - name: str - age: int - -async def main(): - aclient = instructor.from_provider("openai/gpt-5-nano", async_client=True) - model = await aclient.create( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "{\"name\": \"Ada\", \"age\": 37}"}], - response_model=User, - max_retries=3, - strict=True, - ) - print(model) - -asyncio.run(main()) -``` - -## Streaming, Partial, Parallel - -### Streaming Iterable -- Use `create_iterable(response_model=Model, stream=True implicitly)` via `Instructor.create_iterable`. -- Returns a generator (sync) or async generator (async) of parsed items. -- Internally sets `stream=True`, and `IterableBase.from_streaming_response(_async)` assembles items. - -```python -for item in client.create_iterable(messages=..., response_model=MyModel): - print(item) -``` - -### Partial Objects -- Use `create_partial(response_model=Model)` to receive progressively filled partial models while streaming. -- Internally wraps the model as `Partial[Model]` and sets `stream=True`. - -```python -for partial in client.create_partial(messages=..., response_model=MyModel): - # partial contains fields as they arrive - pass -``` - -### Parallel Tools -- Use `Mode.PARALLEL_TOOLS` and a parallel type hint (e.g., list of models) when you need multiple tool calls in one request. -- Streaming is not supported in parallel tools mode. - -```python -from instructor.mode import Mode - -result = client.create( - model="gpt-4o", - messages=[{"role": "user", "content": "Extract person and event info."}], - response_model=[PersonInfo, EventInfo], - mode=Mode.PARALLEL_TOOLS, -) -``` - -## Hooks and Retry - -You can observe and instrument the flow with hooks. Typical events: -- `completion:kwargs`: just before provider call -- `completion:response`: after provider call -- `parse:error`: on validation/JSON errors -- `completion:last_attempt`: when a retry sequence is about to stop -- `completion:error`: non-validation completion errors - -```python -from instructor.core.hooks import HookName - -client.on(HookName.COMPLETION_KWARGS, lambda **kw: print("KWARGS", kw)) -client.on(HookName.PARSE_ERROR, lambda e: print("PARSE", e)) -``` - -## Where Multimodal Conversion Happens - -- For modes that require it, messages are converted via `processing.multimodal.convert_messages`. -- Image/Audio/PDF autodetection can be enabled (by specific handlers/modes) and will convert strings/paths/URLs or data URIs into provider-ready payloads. - -## Error Handling at a Glance - -- Validation or JSON decode errors trigger the reask path. -- Reask handlers (`handle_reask_kwargs`) append/adjust messages with error feedback so the next attempt can correct itself. -- If all retries fail, `InstructorRetryException` is raised containing `failed_attempts`, the last completion, usage totals, and the create kwargs for reproduction. - -## Extensibility Notes - -- New providers add utils for response and reask handling and register modes used by the dispatcher. -- Most JSON/tool patterns are shared; prefer reusing existing handlers where possible. -- Keep provider-specific logic in provider utils; avoid expanding central dispatcher beyond routing and orchestration. - diff --git a/참고/instructor-main/docs/blog/.authors.yml b/참고/instructor-main/docs/blog/.authors.yml deleted file mode 100644 index 7f657cf..0000000 --- a/참고/instructor-main/docs/blog/.authors.yml +++ /dev/null @@ -1,34 +0,0 @@ -authors: - jxnl: - name: Jason Liu - description: Creator - avatar: https://avatars.githubusercontent.com/u/4852235?v=4 - url: https://twitter.com/intent/follow?screen_name=jxnlco - ivanleomk: - name: Ivan Leo - description: Contributor - avatar: https://pbs.twimg.com/profile_images/1838778744468836353/utYfioiO_400x400.jpg - url: https://twitter.com/intent/follow?screen_name=ivanleomk - anmol: - name: Anmol Jawandha - description: Contributor - avatar: https://pbs.twimg.com/profile_images/1248544843556466693/PgxUIeBs_400x400.jpg - joschkabraun: - name: Joschka Braun - description: Contributor - avatar: https://pbs.twimg.com/profile_images/1601251353531224065/PYpqKsjL_400x400.jpg - url: https://joschkabraun.com - sarahchieng: - name: Sarah Chieng - description: Contributor - avatar: https://pbs.twimg.com/profile_images/1755455116595834880/Hxh5ceRZ_400x400.jpg - url: https://twitter.com/sarahchieng - zilto: - name: Thierry Jean - description: Contributor - avatar: https://avatars.githubusercontent.com/u/68975210?v=4 - url: https://www.linkedin.com/in/thierry-jean/ - yanomaly: - name: Yan - description: Contributor - avatar: https://avatars.githubusercontent.com/u/87994542?v=4 diff --git a/참고/instructor-main/docs/blog/index.md b/참고/instructor-main/docs/blog/index.md deleted file mode 100644 index 820261c..0000000 --- a/참고/instructor-main/docs/blog/index.md +++ /dev/null @@ -1,46 +0,0 @@ -# Subscribe to our Newsletter for Updates and Tips - -If you want to get updates on new features and tips on how to use Instructor, you can subscribe to our newsletter below to get notified when we publish new content. - - - -## Advanced Topics - -1. [Unified Provider Interface in Instructor](posts/announcing-unified-provider-interface.md) -2. [Instructor Implements llms.txt](posts/llms-txt-adoption.md) -3. [Query Understanding: Beyond Embeddings](posts/rag-and-beyond.md) -4. [Achieving GPT-4 Level Summaries with GPT-3.5-turbo](posts/chain-of-density.md) -5. [Basics of Guardrails and Validation in AI Models](posts/validation-part1.md) -6. [Validating Citations in AI-Generated Content](posts/citations.md) -7. [Fine-tuning and Distillation in AI Models](posts/distilation-part1.md) -8. [Enhancing OpenAI Client Observability with LangSmith](posts/langsmith.md) -9. [Logfire Integration with Pydantic](posts/logfire.md) - -## AI Development and Optimization - -- [Effective Function Caching in Python](posts/caching.md) -- [Fundamentals of Batch Processing with Async in Python](posts/learn-async.md) -- [Streaming Models to Improve Latency](posts/generator.md) -- [Using OpenAI's Batch API for Large-Scale Synthetic Data Generation](../examples/batch_job_oai.md) -- [Implementing Bulk Classification with User-Provided Tags](../examples/bulk_classification.md) -- [Utilizing GPT-4 Vision API for Ad Copy from Product Images](../examples/image_to_ad_copy.md) - -## Language Models and Prompting Techniques - -- [Least-to-Most Prompting Technique for LLMs](../prompting/decomposition/least_to_most.md) -- [Chain of Verification (CoVe) Method for Improving LLM Accuracy](../prompting/self_criticism/chain_of_verification.md) -- [Cumulative Reasoning to Enhance Model Performance](../prompting/self_criticism/cumulative_reason.md) -- [Reverse Chain of Thought (RCoT) Method for Logical Consistency](../prompting/self_criticism/reversecot.md) - -## Integrations and Tools - -- [Ollama Integration](../integrations/ollama.md) -- [llama-cpp-python Integration](../integrations/llama-cpp-python.md) -- [Together Compute Integration](../integrations/together.md) -- [Pandas DataFrame Examples](./posts/tidy-data-from-messy-tables.md#defining-a-custom-type) -- [Streaming Response Examples](../concepts/partial.md) - -## Media and Resources - -- [Course: Structured Outputs with Instructor](https://www.wandb.courses/courses/steering-language-models?x=1) -- [Keynote: Pydantic is All You Need](posts/aisummit-2023.md) diff --git a/참고/instructor-main/docs/blog/posts/aisummit-2023.md b/참고/instructor-main/docs/blog/posts/aisummit-2023.md deleted file mode 100644 index 9e7f55d..0000000 --- a/참고/instructor-main/docs/blog/posts/aisummit-2023.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -authors: -- jxnl -categories: -- Pydantic -comments: true -date: 2023-11-02 -description: Explore insights on utilizing Pydantic for effective prompt engineering - in this AI Engineer Summit keynote. -draft: false -tags: -- Pydantic -- Prompt Engineering -- AI Summit -- Machine Learning -- Data Validation ---- - -# AI Engineer Keynote: Pydantic is all you need - -[![Pydantic is all you need](https://img.youtube.com/vi/yj-wSRJwrrc/0.jpg)](https://www.youtube.com/watch?v=yj-wSRJwrrc) - -[Click here to watch the full talk](https://www.youtube.com/watch?v=yj-wSRJwrrc) - - - -Last month, I ventured back onto the speaking circuit at the inaugural [AI Engineer Summit](https://www.ai.engineer/summit), sharing insights on leveraging [Pydantic](https://docs.pydantic.dev/latest/) for effective prompt engineering. I dove deep into what is covered in our documentation and standard blog posts, - -I'd genuinely appreciate any feedback on the talk - every bit helps in refining the art. So, take a moment to check out the [full talk here](https://youtu.be/yj-wSRJwrrc?si=vGMIqtTapbIN8SLz), and let's continue pushing the boundaries of what's possible. \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/announcing-gemini-tool-calling-support.md b/참고/instructor-main/docs/blog/posts/announcing-gemini-tool-calling-support.md deleted file mode 100644 index b86ee78..0000000 --- a/참고/instructor-main/docs/blog/posts/announcing-gemini-tool-calling-support.md +++ /dev/null @@ -1,123 +0,0 @@ ---- -authors: -- ivanleomk -categories: -- LLM Techniques -comments: true -date: 2024-09-03 -description: Introducing structured outputs for Gemini tool calling support in the - instructor library, enhancing interactions with Gemini and VertexAI SDKs. -draft: false -tags: -- Gemini -- VertexAI -- Tool Calling -- Instructor Library -- AI SDKs ---- - -# Structured Outputs for Gemini now supported - -We're excited to announce that `instructor` now supports structured outputs using tool calling for both the Gemini SDK and the VertexAI SDK. - -A special shoutout to [Sonal](https://x.com/sonalsaldanha) for his contributions to the Gemini Tool Calling support. - -Let's walk through a simple example of how to use these new features - -## Installation - -To get started, install the latest version of `instructor`. Depending on whether you're using Gemini or VertexAI, you should install the following: - -=== "Gemini" - - ```bash - pip install "instructor[google-generativeai]" - ``` - -=== "VertexAI" - - ```bash - pip install "instructor[vertexai]" - ``` - -This ensures that you have the necessary dependencies to use the Gemini or VertexAI SDKs with instructor. - -We recommend using the Gemini SDK over the VertexAI SDK for two main reasons. - -1. Compared to the VertexAI SDK, the Gemini SDK comes with a free daily quota of 1.5 billion tokens to use for developers. -2. The Gemini SDK is significantly easier to setup, all you need is a `GOOGLE_API_KEY` that you can generate in your GCP console. THe VertexAI SDK on the other hand requires a credentials.json file or an OAuth integration to use. - -## Getting Started - -With our provider agnostic API, you can use the same interface to interact with both SDKs, the only thing that changes here is how we initialise the client itself. - -Before running the following code, you'll need to make sure that you have your Gemini API Key set in your shell under the alias `GOOGLE_API_KEY`. - -```python -import instructor -import google.generativeai as genai -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -client = instructor.from_provider("google/gemini-2.5-flash") - ) -) - -resp = client.create( - messages=[ - { - "role": "user", - "content": "Extract Jason is 25 years old.", - } - ], - response_model=User, -) - -print(resp) -#> name='Jason' age=25 -``` - -1. Current Gemini models that support tool calling are `gemini-3-flash` and `gemini-1.5-pro-latest`. - -We can achieve a similar thing with the VertexAI SDK. For this to work, you'll need to authenticate to VertexAI. - -There are some instructions [here](https://cloud.google.com/vertex-ai/docs/authentication) but the easiest way I found was to simply download the GCloud cli and run `gcloud auth application-default login`. - -```python -import instructor -import vertexai # type: ignore -from vertexai.generative_models import GenerativeModel # type: ignore -from pydantic import BaseModel - -vertexai.init() - - -class User(BaseModel): - name: str - age: int - - -client = instructor.from_provider("google/gemini-2.5-flash", vertexai=True), # (1)! -) - - -resp = client.create( - messages=[ - { - "role": "user", - "content": "Extract Jason is 25 years old.", - } - ], - response_model=User, -) - -print(resp) -#> name='Jason' age=25 -``` - -1. Current Gemini models that support tool calling are `gemini-3-flash` and `gemini-1.5-pro-latest`. \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/announcing-instructor-responses-support.md b/참고/instructor-main/docs/blog/posts/announcing-instructor-responses-support.md deleted file mode 100644 index be702d8..0000000 --- a/참고/instructor-main/docs/blog/posts/announcing-instructor-responses-support.md +++ /dev/null @@ -1,154 +0,0 @@ ---- -authors: - - ivanleomk -categories: - - instructor -comments: true -date: 2025-05-11 -description: Take advantage of OpenAI's latest offerings with the new responses API -draft: false -tags: - - LLMs - - OpenAI - - Instructor ---- - -# Announcing Responses API support - -We're excited to announce Instructor's integration with OpenAI's new Responses API. This integration brings a more streamlined approach to working with structured outputs from OpenAI models. Let's see what makes this integration special and how it can improve your LLM applications. - - - -## What's New? - -The Responses API represents a significant shift in how we interact with OpenAI models. With Instructor's integration, you can leverage this new API with our familiar, type-safe interface. - -For our full documentation of the features we support, check out our full [OpenAI integration guide](../../integrations/openai.md). - -Getting started is now easier than ever. With our unified provider interface, you can initialize your client with a single line of code. This means less time dealing with configuration and more time building features that matter. - -```python -import instructor - -# Initialize the client with Responses mode -client = instructor.from_provider( - "openai/gpt-4.1-mini", mode=instructor.Mode.RESPONSES_TOOLS -) -``` - -The Responses API brings several improvements to structured data handling. You get access to built-in tools like web search and file search directly through the API. There's more efficient validation of structured outputs and improved error messages with better recovery mechanisms. - -Here's a quick example showing how it works: - -```python -class User(BaseModel): - name: str - age: int - - -# Create structured output -profile = client.responses.create( - input="Extract out Ivan is 28 years old", - response_model=User, -) - -print(profile) -#> name='Ivan' age=28 -``` - -## Key Benefits - -The integration maintains Instructor's core strength of type safety while adding the power of the Responses API. You get full Pydantic model validation, automatic type checking, and clear error messages when validation fails. This gives you confidence that your outputs meet the constraints you've defined. - -One of the most exciting features is the built-in tools support. You can now easily perform web searches with automatic citations, search through your knowledge base, and get real-time information with proper attribution. This significantly expands what you can build without having to integrate multiple APIs. - -Here's an example using web search: - -```python -class Citation(BaseModel): - id: int - url: str - - -class Summary(BaseModel): - citations: list[Citation] - summary: str - - -response = client.responses.create( - input="What are some of the best places to visit in New York for Latin American food?", - tools=[{"type": "web_search_preview"}], - response_model=Summary, -) -``` - -The integration supports multiple ways to get structured outputs. You can use basic creation for simple, straightforward structured outputs. If you need real-time updates, partial creation lets you stream them as they come in. For handling multiple instances of the same object, iterable creation works great. And when you need both structured output and raw completion, completion with raw response gives you exactly that. - -For production applications, we've maintained full async support. This lets you build responsive applications that can handle multiple requests efficiently: - -```python -async def get_user_profile(): - async_client = instructor.from_provider( - "openai/gpt-4.1-mini", mode=instructor.Mode.RESPONSES_TOOLS, async_client=True - ) - - profile = await async_client.responses.create( - input="Extract: Maria lives in Spain.", response_model=UserProfile - ) -``` - -## Why This Matters - -The integration of Instructor with OpenAI's Responses API brings two major benefits that will transform how you work with LLMs. - -First, it makes working with inline citations significantly easier. When your LLM needs to reference external information, you get structured citation data that's ready to integrate into downstream applications. No more parsing messy text or manually extracting references - they come as properly typed objects that you can immediately use in your code. - -Second, it works seamlessly with your existing chat completions code. You can add powerful capabilities like file search and web search without modifying your codebase. Just add the tool definition, and you're ready to go. Here's how simple it is: - -```python -from pydantic import BaseModel -import instructor - - -class Citation(BaseModel): - id: int - url: str - - -class Summary(BaseModel): - citations: list[Citation] - summary: str - - -client = instructor.from_provider( - "openai/gpt-4.1-mini", - mode=instructor.Mode.RESPONSES_TOOLS_WITH_INBUILT_TOOLS, -) - -response = client.create( - messages=[ - { - "role": "user", - "content": "What are some of the best places to visit in New York for Latin American food?", - } - ], - tools=[{"type": "web_search_preview"}], - response_model=Summary, -) -print(response) -""" -citations=[Citation(id=1, url='https://www.nycgo.com/restaurants/best-latin-american-restaurants-in-nyc/'), Citation(id=2, url='https://www.timeout.com/newyork/restaurants/best-latin-american-restaurants-in-nyc'), Citation(id=3, url='https://www.thrillist.com/eat/nation/best-latin-american-restaurants-nyc')] summary="Some of the best places to visit in New York for Latin American food include neighborhoods and restaurants known for authentic and diverse offerings. In Manhattan, areas like the East Village and Lower East Side have excellent Latin American restaurants. Popular spots include Casa Enrique, known for Mexican cuisine; Tia Pol, offering Spanish and Latin flavors; and La Contenta, serving dishes from various Latin American countries. Brooklyn's Williamsburg and Bushwick have emerged as vibrant spots for Latin American eats, with restaurants such as La Esquina and Fonda not to miss. These places are celebrated for delicious food, lively atmospheres, and cultural authenticity, making them top choices for anyone looking to enjoy Latin American cuisine in New York City." -""" -``` - -This makes the path forward clear - you can enhance your existing applications with the latest OpenAI features while maintaining the type safety and validation Instructor is known for. No need to learn a new API or refactor your code. It just works. - -## Getting Started - -To start using the new Responses API integration, update to the latest version of Instructor, set up your OpenAI API key, initialize your client with the Responses mode, and start creating structured outputs. - -This integration represents a significant step forward in making LLM development more accessible and powerful. We're excited to see what you'll build with these new capabilities. - -For more detailed information about using the Responses API with Instructor, check out our [OpenAI integration guide](../../integrations/openai.md). - -Happy coding! diff --git a/참고/instructor-main/docs/blog/posts/announcing-unified-provider-interface.md b/참고/instructor-main/docs/blog/posts/announcing-unified-provider-interface.md deleted file mode 100644 index 4de3db7..0000000 --- a/참고/instructor-main/docs/blog/posts/announcing-unified-provider-interface.md +++ /dev/null @@ -1,214 +0,0 @@ ---- -authors: - - jxnl - - ivanleomk -categories: - - instructor -comments: true -date: 2025-05-08 -description: Switch between different models and providers with a single string! -draft: false -tags: - - LLMs - - Instructor ---- - -We are pleased to introduce a significant enhancement to Instructor: the **`from_provider()`** function. While Instructor has always focused on providing robust structured outputs, we've observed that many users work with multiple LLM providers. This often involves repetitive setup for each client. - -The `from_provider()` function aims to simplify this process, making it easier to initialize clients and experiment across different models. - -This new feature offers a streamlined, string-based method to initialize an Instructor-enhanced client for a variety of popular LLM providers. - - - -## What is `from_provider()`? - -The `from_provider()` function serves as a smart factory for creating LLM clients. By providing a model string identifier, such as `"openai/gpt-4o"` or `"anthropic/claude-3-opus-20240229"`, the function handles the necessary setup: - -- **Automatic SDK Detection**: It identifies the targeted provider (e.g., OpenAI, Anthropic, Google, Mistral, Cohere). -- **Client Initialization**: It dynamically imports the required provider-specific SDK and initializes the native client (like `openai.OpenAI()` or `anthropic.Anthropic()`). -- **Instructor Patching**: It automatically applies the Instructor patch to the client, enabling structured outputs, validation, and retry mechanisms. -- **Sensible Defaults**: It uses recommended `instructor.Mode` settings for each provider, optimized for performance and capabilities such as tool use or JSON mode, where applicable. -- **Sync and Async Support**: Users can obtain either a synchronous or an asynchronous client by setting the `async_client=True` flag. - -## Key Benefits - -The `from_provider()` function is designed to streamline several common workflows: - -- **Model Comparison**: Facilitates quick switching between different models or providers to evaluate performance, cost, or output quality for specific tasks. -- **Multi-Provider Strategies**: Simplifies the implementation of fallback mechanisms or routing queries to different LLMs based on criteria like complexity or cost, reducing client management overhead. -- **Rapid Prototyping**: Allows for faster setup when starting with a new provider or model. -- **Simplified Configuration**: Reduces boilerplate code in projects that integrate with multiple LLM providers. - -## How it Works: A Look Under the Hood - -Internally, `from_provider()` (located in `instructor/auto_client.py`) parses the model string (e.g., `"openai/gpt-5-nano"`) to identify the provider and model name. It then uses conditional logic to import the correct libraries, instantiate the client, and apply the appropriate Instructor patch. For instance, the conceptual handling for an OpenAI client would involve importing the `openai` SDK and `instructor.from_openai`. - -```python -# Conceptual illustration of internal logic for OpenAI: -# (Actual implementation is in instructor/auto_client.py) - -# if provider == "openai": -# import openai -# from instructor import from_openai, Mode -# -# # 'async_client', 'model_name', 'kwargs' are determined by from_provider -# native_client = openai.AsyncOpenAI() if async_client else openai.OpenAI() -# -# return from_openai( -# native_client, -# model=model_name, -# mode=Mode.TOOLS, # Default mode for OpenAI -# **kwargs, -# ) -``` - -The function also manages dependencies by alerting users to install missing packages (e.g., via `uv pip install openai`) if they are not found. - -## Example Usage - -> Note : Ensure your API keys (e.g., `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`) are configured as environment variables to run this code. - -Here's a self-contained example demonstrating how `from_provider()` can be used to retrieve structured output from google gemini's flash-2.0 model. - -```python -import instructor -from pydantic import BaseModel -from typing import Iterable - - -# Define your data structure -class Person(BaseModel): - name: str - age: int - - -# Connect to any provider with a single line -client = instructor.from_provider("google/gemini-2.0-flash") - -# Extract structured data -response = client.create( - messages=[ - { - "role": "user", - "content": "Alice is 30 and Bob is 25.", - } - ], - response_model=Iterable[Person], -) - -for person in response: - print(f"Name: {person.name}, Age: {person.age}") - #> Name: Alice, Age: 30 - #> Name: Bob, Age: 25 -# Output: -# Name: Alice, Age: 30 -# Name: Bob, Age: 25 -``` - -Switching providers is as simple as changing the string: - -```python -# OpenAI -client = instructor.from_provider("openai/gpt-4.1") - -# Anthropic (with version date) -client = instructor.from_provider("anthropic/claude-3-5-haiku-20241022") -``` - -With the unified provider interface, you can now easily benchmark different models on the same task. This is crucial when you need to: - -1. Compare response quality across different providers -2. Test which model gives the best structured extraction results -3. Optimize for speed vs. accuracy tradeoffs -4. Run A/B tests between providers without code refactoring - -Instead of maintaining separate codebases for each provider or complex switching logic, you can focus on what matters: finding the optimal model for your specific use case. - -### Async Support - -When building production applications that need to remain responsive, asynchronous processing is essential. - -Instructor's unified provider interface supports this workflow with a simple `async_client` keyword during initialization. - -```python -client = instructor.from_provider("openai/gpt-4.1", async_client=True) -``` - -The async implementation works particularly well for web servers, batch processing jobs, or any scenario where you need to extract structured data without blocking your application's main thread. - -Here's how you can implement it: - -```python -import instructor -from pydantic import BaseModel -import asyncio - - -class UserProfile(BaseModel): - name: str - country: str - - -async def get_user_profile(): - # Initialise an asynchronous client - async_client = instructor.from_provider("openai/gpt-4.1-mini", async_client=True) - - # Extract data asynchronously - profile = await async_client.create( - messages=[{"role": "user", "content": "Extract: Maria lives in Spain."}], - response_model=UserProfile, - ) - print(f"Name: {profile.name}, Country: {profile.country}") - #> Name: Maria, Country: Spain - - -if __name__ == "__main__": - asyncio.run(get_user_profile()) -``` - -### Provider Specific Parameters - -Some providers require additional parameters for optimal performance. - -Rather than hiding these options, Instructor allows you to pass them directly through the from_provider function: - -```python -# Anthropic requires max tokens -client = instructor.from_provider("anthropic/claude-3-sonnet-20240229", max_tokens=1024) -``` - -If you'd like to change this parameter down the line, you can just do so by setting it on the `client.chat.completions.create` function again. - -### Type Completion - -To make it easy for you to find the right model string, we now ship with auto-complete for these new model-provider initialisation strings. - -This is automatically provided for you out of the box when you use the new `from_provider` method as seen below. - -![](./img/instructor-autocomplete.png) - -Say bye to fiddling around with messy model versioning and get cracking to working on your business logic instead! - -## Path Forward - -The `from_provider()` function offers a convenient method for client initialization. Instructor remains a lightweight wrapper around your chosen LLM provider's client, and users always retain the flexibility to initialize and patch clients manually for more granular control or when using providers not yet covered by this utility. - -This unified interface is intended to balance ease of use for common tasks with the underlying flexibility of Instructor, aiming to make multi-provider LLM development more accessible and efficient. However, there is still much to do to further streamline multi-provider workflows. Future efforts could focus on: - -- **Unified Prompt Caching API**: While Instructor supports prompt caching for providers like [Anthropic](../../integrations/anthropic.md#caching) (see also our [blog post on Anthropic prompt caching](../posts/anthropic-prompt-caching.md) and the general [Prompt Caching concepts](../../concepts/prompt_caching.md)), a more standardized, cross-provider API for managing cache behavior could significantly simplify optimizing costs and latency. -- **Unified Multimodal Object Handling**: Instructor already provides a robust way to work with [multimodal inputs like Images, Audio, and PDFs](../../concepts/multimodal.md) across different providers. However, a higher-level unified API could further abstract provider-specific nuances for these types, making it even simpler to build applications that seamlessly switch between, for example, OpenAI's vision capabilities and Anthropic's, without changing how media objects are passed. - -These are areas where `instructor` can continue to reduce friction for developers working in an increasingly diverse LLM ecosystem. - -We encourage you to try `from_provider()` in your projects, particularly when experimenting with multiple LLMs. Feedback and suggestions for additional providers or features are always welcome. - -## Related Documentation -- [Provider Patching](../../concepts/patching.md) - How provider integration works -- [All Integrations](../../integrations/index.md) - Supported provider list - -## See Also - -- [String-Based Initialization](string-based-init.md) - Alternative init method -- [Framework Comparison](best_framework.md) - Multi-provider advantages -- [Getting Started](introduction.md) - Quick start guide diff --git a/참고/instructor-main/docs/blog/posts/anthropic-prompt-caching.md b/참고/instructor-main/docs/blog/posts/anthropic-prompt-caching.md deleted file mode 100644 index 9ef70e8..0000000 --- a/참고/instructor-main/docs/blog/posts/anthropic-prompt-caching.md +++ /dev/null @@ -1,347 +0,0 @@ ---- -authors: -- ivanleomk -categories: -- Anthropic -comments: true -date: 2024-09-14 -description: Discover how prompt caching with Anthropic can improve response times - and reduce costs for large context applications. -draft: false -tags: -- prompt caching -- Anthropic -- API optimization -- cost reduction -- latency improvement ---- - -# Why should I use prompt caching? - -Developers often face two key challenges when working with large context - Slow response times and high costs. This is especially true when we're making multiple of these calls over time, severely impacting the cost and latency of our applications. With Anthropic's new prompt caching feature, we can easily solve both of these issues. - -Since the new feature is still in beta, we're going to wait for it to be generally available before we integrate it into instructor. In the meantime, we've put together a quickstart guide on how to use the feature in your own applications. - - - -!!! warning "Caching Limitations" - - There are a few important limitations to be aware of when using prompt caching: - - - **Minimum cache size**: For Claude Haiku, your cached content needs to be a minimum of 2048 tokens. For Claude Sonnet, the minimum is 1024 tokens. - - - **Tool definitions**: Currently, tool definitions cannot be cached. However, support for caching tool definitions is planned for a future update. - - - **Upgrade Anthropic**: You must upgrade to Anthropic version `0.34.0` or later to use prompt caching. Make sure that you're using the latest version of the Anthropic SDK. - - Keep these limitations in mind when implementing prompt caching in your applications. - -??? note "Source Text" - - In the following example, we'll be using a short excerpt from the novel "Pride and Prejudice" by Jane Austen. This text serves as an example of a substantial context that might typically lead to slow response times and high costs when working with language models. You can download it manually [here](https://www.gutenberg.org/cache/epub/1342/pg1342.txt) - - ``` - _Walt Whitman has somewhere a fine and just distinction between “loving - by allowance” and “loving with personal love.” This distinction applies - to books as well as to men and women; and in the case of the not very - numerous authors who are the objects of the personal affection, it - brings a curious consequence with it. There is much more difference as - to their best work than in the case of those others who are loved “by - allowance” by convention, and because it is felt to be the right and - proper thing to love them. And in the sect--fairly large and yet - unusually choice--of Austenians or Janites, there would probably be - found partisans of the claim to primacy of almost every one of the - novels. To some the delightful freshness and humour of_ Northanger - Abbey, _its completeness, finish, and_ entrain, _obscure the undoubted - critical facts that its scale is small, and its scheme, after all, that - of burlesque or parody, a kind in which the first rank is reached with - difficulty._ Persuasion, _relatively faint in tone, and not enthralling - in interest, has devotees who exalt above all the others its exquisite - delicacy and keeping. The catastrophe of_ Mansfield Park _is admittedly - theatrical, the hero and heroine are insipid, and the author has almost - wickedly destroyed all romantic interest by expressly admitting that - Edmund only took Fanny because Mary shocked him, and that Fanny might - very likely have taken Crawford if he had been a little more assiduous; - yet the matchless rehearsal-scenes and the characters of Mrs. Norris and - others have secured, I believe, a considerable party for it._ Sense and - Sensibility _has perhaps the fewest out-and-out admirers; but it does - not want them._ - _I suppose, however, that the majority of at least competent votes - would, all things considered, be divided between_ Emma _and the present - book; and perhaps the vulgar verdict (if indeed a fondness for Miss - Austen be not of itself a patent of exemption from any possible charge - of vulgarity) would go for_ Emma. _It is the larger, the more varied, the - more popular; the author had by the time of its composition seen rather - more of the world, and had improved her general, though not her most - peculiar and characteristic dialogue; such figures as Miss Bates, as the - Eltons, cannot but unite the suffrages of everybody. On the other hand, - I, for my part, declare for_ Pride and Prejudice _unhesitatingly. It - seems to me the most perfect, the most characteristic, the most - eminently quintessential of its author’s works; and for this contention - in such narrow space as is permitted to me, I propose here to show - cause._ - _In the first place, the book (it may be barely necessary to remind the - reader) was in its first shape written very early, somewhere about 1796, - when Miss Austen was barely twenty-one; though it was revised and - finished at Chawton some fifteen years later, and was not published till - 1813, only four years before her death. I do not know whether, in this - combination of the fresh and vigorous projection of youth, and the - critical revision of middle life, there may be traced the distinct - superiority in point of construction, which, as it seems to me, it - possesses over all the others. The plot, though not elaborate, is almost - regular enough for Fielding; hardly a character, hardly an incident - could be retrenched without loss to the story. The elopement of Lydia - and Wickham is not, like that of Crawford and Mrs. Rushworth, a_ coup de - théâtre; _it connects itself in the strictest way with the course of the - story earlier, and brings about the denouement with complete propriety. - All the minor passages--the loves of Jane and Bingley, the advent of Mr. - Collins, the visit to Hunsford, the Derbyshire tour--fit in after the - same unostentatious, but masterly fashion. There is no attempt at the - hide-and-seek, in-and-out business, which in the transactions between - Frank Churchill and Jane Fairfax contributes no doubt a good deal to the - intrigue of_ Emma, _but contributes it in a fashion which I do not think - the best feature of that otherwise admirable book. Although Miss Austen - always liked something of the misunderstanding kind, which afforded her - opportunities for the display of the peculiar and incomparable talent to - be noticed presently, she has been satisfied here with the perfectly - natural occasions provided by the false account of Darcy’s conduct given - by Wickham, and by the awkwardness (arising with equal naturalness) from - the gradual transformation of Elizabeth’s own feelings from positive - aversion to actual love. I do not know whether the all-grasping hand of - the playwright has ever been laid upon_ Pride and Prejudice; _and I dare - say that, if it were, the situations would prove not startling or - garish enough for the footlights, the character-scheme too subtle and - delicate for pit and gallery. But if the attempt were made, it would - certainly not be hampered by any of those loosenesses of construction, - which, sometimes disguised by the conveniences of which the novelist can - avail himself, appear at once on the stage._ - _I think, however, though the thought will doubtless seem heretical to - more than one school of critics, that construction is not the highest - merit, the choicest gift, of the novelist. It sets off his other gifts - and graces most advantageously to the critical eye; and the want of it - will sometimes mar those graces--appreciably, though not quite - consciously--to eyes by no means ultra-critical. But a very badly-built - novel which excelled in pathetic or humorous character, or which - displayed consummate command of dialogue--perhaps the rarest of all - faculties--would be an infinitely better thing than a faultless plot - acted and told by puppets with pebbles in their mouths. And despite the - ability which Miss Austen has shown in working out the story, I for one - should put_ Pride and Prejudice _far lower if it did not contain what - seem to me the very masterpieces of Miss Austen’s humour and of her - faculty of character-creation--masterpieces who may indeed admit John - Thorpe, the Eltons, Mrs. Norris, and one or two others to their company, - but who, in one instance certainly, and perhaps in others, are still - superior to them._ - _The characteristics of Miss Austen’s humour are so subtle and delicate - that they are, perhaps, at all times easier to apprehend than to - express, and at any particular time likely to be differently - apprehended by different persons. To me this humour seems to possess a - greater affinity, on the whole, to that of Addison than to any other of - the numerous species of this great British genus. The differences of - scheme, of time, of subject, of literary convention, are, of course, - obvious enough; the difference of sex does not, perhaps, count for much, - for there was a distinctly feminine element in “Mr. Spectator,” and in - Jane Austen’s genius there was, though nothing mannish, much that was - masculine. But the likeness of quality consists in a great number of - common subdivisions of quality--demureness, extreme minuteness of touch, - avoidance of loud tones and glaring effects. Also there is in both a - certain not inhuman or unamiable cruelty. It is the custom with those - who judge grossly to contrast the good nature of Addison with the - savagery of Swift, the mildness of Miss Austen with the boisterousness - of Fielding and Smollett, even with the ferocious practical jokes that - her immediate predecessor, Miss Burney, allowed without very much - protest. Yet, both in Mr. Addison and in Miss Austen there is, though a - restrained and well-mannered, an insatiable and ruthless delight in - roasting and cutting up a fool. A man in the early eighteenth century, - of course, could push this taste further than a lady in the early - nineteenth; and no doubt Miss Austen’s principles, as well as her heart, - would have shrunk from such things as the letter from the unfortunate - husband in the_ Spectator, _who describes, with all the gusto and all the - innocence in the world, how his wife and his friend induce him to play - at blind-man’s-buff. But another_ Spectator _letter--that of the damsel - of fourteen who wishes to marry Mr. Shapely, and assures her selected - Mentor that “he admires your_ Spectators _mightily”--might have been - written by a rather more ladylike and intelligent Lydia Bennet in the - days of Lydia’s great-grandmother; while, on the other hand, some (I - think unreasonably) have found “cynicism” in touches of Miss Austen’s - own, such as her satire of Mrs. Musgrove’s self-deceiving regrets over - her son. But this word “cynical” is one of the most misused in the - English language, especially when, by a glaring and gratuitous - falsification of its original sense, it is applied, not to rough and - snarling invective, but to gentle and oblique satire. If cynicism means - the perception of “the other side,” the sense of “the accepted hells - beneath,” the consciousness that motives are nearly always mixed, and - that to seem is not identical with to be--if this be cynicism, then - every man and woman who is not a fool, who does not care to live in a - fool’s paradise, who has knowledge of nature and the world and life, is - a cynic. And in that sense Miss Austen certainly was one. She may even - have been one in the further sense that, like her own Mr. Bennet, she - took an epicurean delight in dissecting, in displaying, in setting at - work her fools and her mean persons. I think she did take this delight, - and I do not think at all the worse of her for it as a woman, while she - was immensely the better for it as an artist. - ``` - -Let's first initialize our Anthropic client, this will be the same as what we've done before except we're now using the new `beta.prompt_caching` method. - -```python -from instructor import Instructor, Mode, patch -from anthropic import Anthropic - - -client = Instructor( - client=Anthropic(), - create=patch( - create=Anthropic().beta.prompt_caching.messages.create, - mode=Mode.TOOLS, - ), - mode=Mode.TOOLS, -) -``` - -We'll then create a new `Character` class that will be used to extract out a single character from the text and read in our source text ( roughly 2856 tokens using the Anthropic tokenizer). - -```python -with open("./book.txt") as f: - book = f.read() - - -class Character(BaseModel): - name: str - description: str -``` - -Once we've done this, we can then make an api call to get the description of the character. - -```python -for _ in range(2): - resp, completion = client.create_with_completion( # (1)! - model="claude-3-haiku-20240307", - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "" + book + "", - "cache_control": {"type": "ephemeral"}, # (2)! - }, - { - "type": "text", - "text": "Extract a character from the text given above", - }, - ], - }, - ], - response_model=Character, - max_tokens=1000, - ) - assert isinstance(resp, Character) - - print(completion.usage) # (3)! - print(resp) -``` - -1. Using the `create_with_completion` method we can get back both the structured response and the completion object -2. We set the `cache_control` parameter to "ephemeral" to tell Anthropic to cache the book content temporarily -3. We print out the usage information to monitor token consumption - -You'll notice that the usage information is different than what we've seen before. This is because we're now using the `create_with_completion` method which returns both the structured response and the completion object. The completion object contains usage information which we can use to monitor token consumption. - -When we run this, you'll notice that we get the following output. - -```bash -PromptCachingBetaUsage( - cache_creation_input_tokens=2856, - cache_read_input_tokens=0, - input_tokens=30, - output_tokens=119 -) - -Character( - name='Elizabeth Bennet', - description="The protagonist of Jane Austen's novel Pride and Prejudice, who -undergoes a transformation from initially disliking Mr. Darcy to eventually falling -in love with him. The passage describes Elizabeth as a complex, nuanced character, -noting how her feelings towards Darcy evolve naturally over the course of the story." -) - -PromptCachingBetaUsage( - cache_creation_input_tokens=0, - cache_read_input_tokens=2856, - input_tokens=30, - output_tokens=93 -) - -Character( - name='Mrs. Norris', - description='A character from Jane Austen\'s novel Mansfield Park, described as -having "matchless" scenes and being one of the characters that has secured a -considerable party of admirers for the novel.' -) -``` - -You'll notice that in the first request, we created `2856` tokens and in the second request, we read `2856` tokens. - -In other words, `book_content` was cached after the first request and reused in the second request. When you have a larger context window, this can save you a significant amount of money and time because your requests will return a lot faster too. - -This is the entire code for the example above. - -```python -from instructor import Instructor, Mode, patch -from anthropic import Anthropic -from pydantic import BaseModel - -client = Instructor( - client=Anthropic(), - create=patch( - create=Anthropic().beta.prompt_caching.messages.create, - mode=Mode.TOOLS, - ), - mode=Mode.TOOLS, -) - - -class Character(BaseModel): - name: str - description: str - - -with open("./book.txt") as f: - book = f.read() - -for _ in range(2): - resp, completion = client.create_with_completion( - model="claude-3-haiku-20240307", - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "" + book + "", - "cache_control": {"type": "ephemeral"}, - }, - { - "type": "text", - "text": "Extract a character from the text given above", - }, - ], - }, - ], - response_model=Character, - max_tokens=1000, - ) - assert isinstance(resp, Character) - print(completion.usage) - print(resp) -``` - -## Related Documentation -- [Caching Strategies](../../concepts/caching.md) - General caching concepts -- [Anthropic Integration](../../integrations/anthropic.md) - Full Anthropic guide - -## See Also -- [Anthropic Structured Outputs](structured-output-anthropic.md) - Use with caching -- [Response Caching](caching.md) - General caching strategies -- [Performance Monitoring](logfire.md) - Track cache performance \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/anthropic-web-search-structured.md b/참고/instructor-main/docs/blog/posts/anthropic-web-search-structured.md deleted file mode 100644 index e247928..0000000 --- a/참고/instructor-main/docs/blog/posts/anthropic-web-search-structured.md +++ /dev/null @@ -1,140 +0,0 @@ ---- -date: 2025-05-07 -authors: - - jxnl -categories: - - tutorials - - anthropic - - structured-data ---- - -# Using Anthropic's Web Search with Instructor for Real-Time Data - -Anthropic's new web search tool, when combined with Instructor, provides a powerful way to get real-time, structured data from the web. This allows you to build applications that can answer questions and provide information that is up-to-date, going beyond the knowledge cut-off of large language models. - -In this post, we'll explore how to use the `web_search` tool with Instructor to fetch the latest information and structure it into a Pydantic model. Even a simple structure can be very effective for clarity and further processing. - - - -## How it Works - -The web search tool enables Claude models to perform web searches during a generation. When you provide the `web_search` tool in your API request, Claude can decide to use it if the prompt requires information it doesn't have. The API then executes the search, provides the results back to Claude, and Claude can then use this information to generate a response. Importantly, Claude will cite its sources from the search results. You can find more details in the [official Anthropic documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/web-search-tool). - -Instructor simplifies this process by allowing you to define a Pydantic model for the desired output structure. When Claude uses the web search tool and formulates an answer, Instructor ensures that the final output conforms to your defined schema. - -## Example: Getting the Latest UFC Results - -Let's look at a practical example. We want to get the latest UFC fight results. - -First, ensure you have `instructor` and `anthropic` installed: - -```bash -uv add instructor anthropic -``` - -Now, let's define our Pydantic model for the response: - -```python -import instructor -from pydantic import BaseModel - - -# Noticed thhat we use JSON not TOOLS mode -client = instructor.from_provider( - "anthropic/claude-3-7-sonnet-latest", - mode=instructor.Mode.JSON, - async_client=False, -) - - -class Citation(BaseModel): - id: int - url: str - - -class Response(BaseModel): - citations: list[Citation] - response: str -``` - -This Response model is straightforward. It gets the model to first generate a list of citations for articles that it referenced before generating it's answer. - -This helps to ground its response in the sources it retrieved and provide a higher quality response. - -Now, we can make the API call: - -```python -response_data, completion_details = client.messages.create_with_completion( - messages=[ - { - "role": "system", - "content": "You are a helpful assistant that summarizes news articles. Your final response should be only contain a single JSON object returned in your final message to the user. Make sure to provide the exact ids for the citations that support the information you provide in the form of inline citations as [1] [2] [3] which correspond to a unique id you generate for a url that you find in the web search tool which is relevant to your final response.", - }, - { - "role": "user", - "content": "What are the latest results for the UFC and who won? Answer this in a concise response that's under 3 sentences.", - }, - ], - tools=[{"type": "web_search_20250305", "name": "web_search", "max_uses": 3}], - response_model=Response, -) - -print("Response:") -print(response_data.response) -print("\nCitations:") -for citation in response_data.citations: - print(f"{citation.id}: {citation.url}") -``` - -This approach provides a clean way to get the LLM's answer into a defined Pydantic object. The `examples/anthropic-web-tool/run.py` script reflects this implementation. - -Expected output (will vary based on real-time web search data): - -``` -Response: -The latest UFC event was UFC Fight Night: Sandhagen vs Figueiredo held on May 3, 2025, in Des Moines, Iowa. Cory Sandhagen defeated former champion Deiveson Figueiredo by TKO (knee injury) in the main event, while Reinier de Ridder upset previously undefeated prospect Bo Nickal by TKO in the co-main event [1][2]. The next major UFC event is UFC 315 on May 10, featuring a welterweight championship bout between Belal Muhammad and Jack Della Maddalena [3]. - -Citations: -1: https://www.ufc.com/news/main-card-results-highlights-winner-interviews-ufc-fight-night-sandhagen-vs-figueiredo-wells-fargo-arena-des-moines -2: https://www.mmamania.com/2025/5/4/24423285/ufc-des-moines-results-sooo-about-last-night-sandhagen-vs-figueiredo-espn-mma-bo-nickal -3: https://en.wikipedia.org/wiki/UFC_315 -``` - -## Key Benefits - -- **Real-Time Information**: Access the latest data directly from the web. -- **Structured Output**: Even with a simple model, Instructor ensures the output is a Pydantic object, making it easy to work with programmatically. -- **Source Citations**: Claude automatically cites sources, allowing for verification (details in the API response, not shown in this simplified example). -- **Reduced Hallucinations**: By relying on web search for factual, up-to-the-minute data, the likelihood of the LLM providing incorrect or outdated information is reduced. - -## Configuring the Web Search Tool - -Anthropic provides several options to configure the web search tool: - -- `max_uses`: Limit the number of searches Claude can perform in a single request. -- `allowed_domains`: Restrict searches to a list of specific domains. -- `blocked_domains`: Prevent searches on certain domains. -- `user_location`: Localize search results by providing an approximate location (city, region, country, timezone). - -For example, to limit searches to 3 and only allow results from `espn.com` and `ufc.com`: - -```python - tools = ( - [ - { - "type": "web_search_20250305", - "name": "web_search", - "max_uses": 3, - "allowed_domains": ["espn.com", "ufc.com"], - } - ], - ) -``` - -You cannot use `allowed_domains` and `blocked_domains` in the same request. - -## Conclusion - -Combining Anthropic's web search tool with Instructor's structured data capabilities opens up exciting possibilities for building dynamic, information-rich applications. Whether you're tracking sports scores, news updates, or market trends, this powerful duo can help you access and organize real-time web data effectively, even with simple Pydantic models. - -Check out the example code in `examples/anthropic-web-tool/run.py` to see this implementation, and refer to the [Anthropic web search documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/web-search-tool) for more in-depth information on the tool's capabilities. diff --git a/참고/instructor-main/docs/blog/posts/anthropic.md b/참고/instructor-main/docs/blog/posts/anthropic.md deleted file mode 100644 index 2aa9488..0000000 --- a/참고/instructor-main/docs/blog/posts/anthropic.md +++ /dev/null @@ -1,80 +0,0 @@ ---- -authors: -- jxnl -categories: -- Anthropic -comments: true -date: 2024-03-20 -description: Learn how to integrate Anthropic's powerful language models into your projects using Instructor, with step-by-step guidance on installation, client setup, and creating structured outputs with Pydantic models. -draft: false -tags: -- Anthropic -- API Development -- Pydantic -- Python -- LLM Techniques ---- - -# Structured Outputs with Anthropic - -A special shoutout to [Shreya](https://twitter.com/shreyaw_) for her contributions to the anthropic support. As of now, all features are operational with the exception of streaming support. - -For those eager to experiment, simply patch the client with `ANTHROPIC_JSON`, which will enable you to leverage the `anthropic` client for making requests. - -``` -pip install instructor[anthropic] -``` - -!!! warning "Missing Features" - - Just want to acknowledge that we know that we are missing partial streaming and some better re-asking support for XML. We are working on it and will have it soon. - -```python -from pydantic import BaseModel -from typing import List -import anthropic -import instructor - -# Patching the Anthropics client with the instructor for enhanced capabilities -anthropic_client = instructor.from_openai( - create=anthropic.Anthropic().messages.create, - mode=instructor.Mode.JSON -) - -class Properties(BaseModel): - name: str - value: str - -class User(BaseModel): - name: str - age: int - properties: List[Properties] - -user_response = anthropic_client( - model="claude-3-haiku-20240307", - max_tokens=1024, - max_retries=0, - messages=[ - { - "role": "user", - "content": "Create a user for a model with a name, age, and properties.", - } - ], - response_model=User, -) # type: ignore - -print(user_response.model_dump_json(indent=2)) -""" -{ - "name": "John", - "age": 25, - "properties": [ - { - "key": "favorite_color", - "value": "blue" - } - ] -} -``` - -We're encountering challenges with deeply nested types and eagerly invite the community to test, provide feedback, and suggest necessary improvements as we enhance the anthropic client's support. \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/bad-schemas-could-break-llms.md b/참고/instructor-main/docs/blog/posts/bad-schemas-could-break-llms.md deleted file mode 100644 index c7cc4c8..0000000 --- a/참고/instructor-main/docs/blog/posts/bad-schemas-could-break-llms.md +++ /dev/null @@ -1,341 +0,0 @@ ---- -authors: -- ivanleomk -categories: -- LLM Techniques -comments: true -date: 2024-09-26 -description: Discover how response models impact LLM performance, focusing on structured - outputs for optimal results in GPT-4o and Claude models. -draft: false -tags: -- LLM Performance -- Response Models -- Structured Outputs -- GPT-4o -- Claude Models ---- - -# Bad Schemas could break your LLM Structured Outputs - -You might be leaving up to 60% performance gains on the table with the wrong response model. Response Models impact model performance massively with Claude and GPT-4o, irregardless of you’re using JSON mode or Tool Calling. - -Using the right response model can help ensure [your models respond in the right language](../posts/matching-language.md) or prevent [hallucinations when extracting video timestamps](../posts/timestamp.md). - -We decided to investigate this by benchmarking Claude and GPT-4o on the GSM8k dataset and found that - -1. **Field Naming drastically impacts performance** - Changing a single field name from `final_choice` to `answer` improved model accuracy from 4.5% to 95%. The way we structure and name fields in our response models can fundamentally alter how the model interprets and responds to queries. -2. **Chain Of Thought significantly boosts performance** - Adding a `reasoning` field increased model accuracy by 60% on the GSM8k dataset. Models perform significantly better when they explain their logic step-by-step. -3. **Be careful with JSON mode** - JSON mode exhibited 50% more performance variation than Tool Calling when renaming fields. Different response models showed varying levels of performance between JSON mode and Tool Calling, indicating that JSON mode requires more careful optimisation. - - - -We’ll do so in the following steps - -1. We’ll first talk about the GSM8k dataset and how we’re using it for benchmarking -2. Then we’ll cover some of the results we obtained and talk about some of the key takeaways that we discovered -3. Lastly, we’ll provide some tips to optimise your model’s response format that you can apply today - -## Dataset - -We used OpenAI's GSM8k dataset to benchmark model performance. This dataset challenges LLM models to solve simple math problems that involve multiple steps of reasoning. Here's an example: - -> Natalia sold clips to 48 friends in April, and half as many in May. How many clips did Natalia sell in total?" - -The original dataset includes reasoning steps and the final answer. We stripped it down to bare essentials: question, answer, and separated reasoning. To do so, we used this code to process the data: - -```python -from datasets import load_dataset, Dataset, DatasetDict - -splits = ["test", "train"] - - -def generate_gsm8k(split): - ds = load_dataset("gsm8k", "main", split=split, streaming=True) - for row in ds: - reasoning, answer = row["answer"].split("####") - answer = int(answer.strip().replace(",", "")) - yield { - "question": row["question"], - "answer": answer, - "reasoning": reasoning, - } - - -# Create the dataset for train and test splits -train_dataset = Dataset.from_generator(lambda: generate_gsm8k("train")) -test_dataset = Dataset.from_generator(lambda: generate_gsm8k("test")) - -# Combine them into a DatasetDict -dataset = DatasetDict({"train": train_dataset, "test": test_dataset}) - -dataset.push_to_hub("567-labs/gsm8k") -``` - -This allows us to test how changes in the response format, response model and even the chosen model itself would affect reasoning ability of the model. - -Using this new dataset, we then tested the Claude and GPT-4o models with a variety of different response models and response modes such as JSON Mode and Tool Calling. The final results were fascinating - highlighting the importance of a good response model in squeezing out the maximum performance from your chosen model. - -## Benchmarks - -We had two key questions on hand that we wanted to answer - -1. How does Structured Extraction impact model performance as compared to other response modes such as JSON mode. -2. What was the impact of different response models on model performance? - -To answer these questions, we sampled the first 200 questions from the GSM8k dataset and tested different permutations of response modes and response models. - -We conducted our experiment in two parts - -1. **Modes and Models** : We first started by exploring how different combinations of response modes and models might impact performance on the GSM8k -2. **Response Models :** We then looked at how different response models with varying levels of complexity might impact the performance of each model - -Let’s explore each portion in greater detail. - -### Modes and Models - -By the end of these experiments, we had the following takeaways - -1. **Claude Models excel at complex tasks** : Claude models see significantly greater improvement with few shot improvements as compared to the GPT-4o variants. This means that for complex tasks with specific nuanced output formats or instructions, Claude models will benefit more from few-shot examples - -2. **Structured Extraction doesn’t lose out** : While we see a 1-2% in performance with JSON mode relative to function calling, working with JSON mode is tricky when response models get complicated. Working with smaller models such as Haiku in JSON mode often required parsing out control characters and increasing the number of re-asks. This was in contrast to the consistent performance of structured extraction that returned a consistent schema. - -3. **4o Mini should be used carefully** : We found that 4o-mini had much less steerability as compared to Claude models, with few-shot examples something resulting in worse performance. - -It’s important here to note that the few shot examples mentioned here only made a difference when the reasoning behind the answer was provided. Without this reasoning example, there wasn’t the same performance improvement observed. - -Here were our results for the Claude Family of models - -| Model | Anthropic JSON Mode | JSON w 5 Few Shot | Anthropic Tools | Tools w 5 few shot | Tools w 10 few shot | Benchmarks | -| ----------------- | ------------------- | ----------------- | --------------- | ------------------ | ------------------- | ---------- | -| claude-3.5-sonnet | 97.00 | 98.5 | 96.00 | 98.00% | 98% | 96.4 | -| claude-3-haiku | 87.50% | 89% | 87.44% | 90.5% | 90.5% | 88.9 | -| claude-3-sonnet | 94.50% | 91.5 | 91.00% | 96.50% | 91.5% | 92.3 | -| claude-3-opus | 96.50% | 98.50% | 96.50% | 97.00% | 97.00% | 95 | - -Here were our results for `4o-mini` - -| model | gpt-4o-mini | gpt-4o | -| ----------------------------- | ----------- | ------ | -| Structured Outputs | 95.5 | 91.5% | -| Structured Outputs 5 Few-Shot | 94.5 | 94.5% | -| Tool Calling | 93.5 | 93.5% | -| Tool Calling 5 Few Shot | 93.0 | 95% | -| Json Mode | 94.5 | 95.5 | -| Json Mode 5 Few Shot | 95.0 | 97% | - -It’s clear here that Claude models consistently show significant improvement with few-shot examples compared to GPT-4o variants. This is in contrast to `4o-mini` which actually showed a decreased in performance for tool calling when provided with simple examples. - -### Response Models - -With these new results, we then proceeded to examine how response models might impact the performance of our models when it came to function calling. While doing so, we had the following takeaways. - -1. **Chain Of Thought** : Chain Of Thought is incredibly important and can boost model performance on the GSM8k by as much as 60% from our benchmarks -2. **JSON mode is much more sensitive than Tool Calling** : In our initial benchmarks, we found that simple changes in the response model such as additional parameters could impact performance by as much as 30% - something which Tool Calling didn’t suffer from. -3. **Naming matters a lot** : The naming of a response parameter is incredibly important. Just going from `potential_final_choice` and `final_choice` to `potential_answers` and `final_answer` improved our final accuracy from 4.5% to 95%. - -#### Chain Of Thought - -It’s difficult to understate the importance of allowing the model to reason and plan before generating a final response. - -In our initial tests , we used the following two models - -```python -class Answer(BaseModel): - chain_of_thought: str - answer: int - - -class OnlyAnswer(BaseModel): - answer: int -``` - -| Model | JSON Mode | Tool Calling | -| ---------- | --------- | ------------ | -| Answer | 92% | 94% | -| OnlyAnswer | 33% | 33.5% | - -These models were tested using the **exact same prompt and questions**. The only thing that differed between them was the addition of a `chain_of_thought` response parameter to allow the model to reason effectively. - -We’re not confined to this specific naming convention of `chain_of_thought`, although it does work consistently well. We can show that when we look at the results we obtained when we tested the following response models. - -In order to verify this, we took a random sample of 50 questions from the test dataset and looked at the performance of different response models that implemented similar reasoning fields on the GSM8k. - -Our conclusion? Simply adding additional fields for the model to reason about its final response improves reasoning all around. - -```python -class AssumptionBasedAnswer(BaseModel): - assumptions: list[str] - logic_flow: str - answer: int - -class ErrorAwareCalculation(BaseModel): - key_steps: list[str] - potential_pitfalls: list[str] - intermediate_results: list[str] - answer: int - - lass AnswerWithIntermediateCalculations(BaseModel): - assumptions: list[str] - intermediate_calculations: list[str] - chain_of_thought: str - final_answer: int - -class AssumptionBasedAnswerWithExtraFields(BaseModel): - assumptions: list[str] - logic_flow: str - important_intermediate_calculations: list[str] - potential_answers: list[int] - answer: int - - -class AnswerWithReasoningAndCalculations(BaseModel): - chain_of_thought: str - key_calculations: list[str] - potential_answers: list[int] - final_choice: int -``` - -| Model | Accuracy | -| ------------------------------------ | -------- | -| AssumptionBasedAnswer | 78% | -| ErrorAwareCalculation | 92% | -| Answer With Intermediate Calculation | 90% | -| AssumptionBasedAnswerWithExtraFields | 90% | -| AnswerWithReasoningAndCalculations | 94% | - -So if you’re generating any sort of response, don’t forget to add in a simple reasoning field that allows for this performance boost. - -#### JSON mode is incredibly Sensitive - -We were curious how this would translate over to the original sample of 200 questions. To do so, we took the original 200 questions that we sampled in our previous experiment and tried to see how JSON mode and Tool Calling performed with other different permutations with `gpt-4o-mini`. - -Here were the models that we used - -```python -class Answer(BaseModel): - chain_of_thought: str - answer: int - - -class AnswerWithCalculation(BaseModel): - chain_of_thought: str - required_calculations: list[str] - answer: int - - -class AssumptionBasedAnswer(BaseModel): - assumptions: list[str] - logic_flow: str - answer: int - - -class ErrorAwareCalculation(BaseModel): - key_steps: list[str] - potential_pitfalls: list[str] - intermediate_results: list[str] - answer: int - - -class AnswerWithNecessaryCalculationAndFinalChoice(BaseModel): - chain_of_thought: str - necessary_calculations: list[str] - potential_final_choices: list[str] - final_choice: int -``` - -| Model | JSON Mode | Tool Calling | -| -------------------------------------------- | --------- | ------------ | -| Answer | 92% | 94% | -| AnswerWithCalculation | 86.5% | 92% | -| AssumptionBasedAnswer | 65% | 78.5% | -| ErrorAwareCalculation | 92% | 88.5% | -| AnswerWithNecessaryCalculationAndFinalChoice | 87.5% | 95% | - -What’s interesting about these results is that the difference in performance for JSON mode with multiple response models is far greater than that of Tool Calling. - -The worst performing response model for JSON mode was `AssumptionBasedAnswer` which scored 65% on the GSM8k while the worst performing response for Tool Calling was `AssumptionBasedAnswer` that scored 78.5% on our benchmarks. This means that the variation in performance for JSON mode was almost 50% larger than that of Tool Calling. - -What’s also interesting is that different response models impacted each response mode differently. For Tool Calling, `AnswerWithNecessaryCalculationAndFinalChoice` was the best performing response model while for JSON mode, it was `ErrorAwareCalculation` and `Answer`. - -This means that when looking at response models for our applications, we can’t just toggle a different mode and hope that the performance gets a magical boost. We need to have a systematic way of evaluating model performance to find the best balance between different response models that we’re experimenting with. - -#### Naming Matters A Lot - -We obtained an accuracy of `4.5%` when working with the following response model - -```python -class AnswerWithNecessaryCalculationAndFinalChoice(BaseModel): - chain_of_thought: str - necessary_calculations: list[str] - potential_final_choices: list[str] - final_choice: int -``` - -This is weird because it doesn’t look all too different from the top performing response model, which achieved an accuracy of `95%` . - -```python -class AnswerWithNecessaryCalculationAndFinalChoice(BaseModel): - chain_of_thought: str - necessary_calculations: list[str] - potential_final_answers: list[str] - answer: int -``` - -In fact, the only thing that changed was the last two parameters. Upon closer inspection, what was happening was that in the first case, we were generating response objects that looked like this - -```python -{ - "chain_of_thought": "In the race, there are a total of 240 Asians. Given that 80 were Japanese, we can calculate the number of Chinese participants by subtracting the number of Japanese from the total number of Asians: 240 - 80 = 160. Now, it is given that there are 60 boys on the Chinese team. Therefore, to find the number of girls on the Chinese team, we subtract the number of boys from the total number of Chinese participants: 160 - 60 = 100 girls. Thus, the number of girls on the Chinese team is 100.", - "necessary_calculations": [ - "Total Asians = 240", - "Japanese participants = 80", - "Chinese participants = Total Asians - Japanese participants = 240 - 80 = 160", - "Boys in Chinese team = 60", - "Girls in Chinese team = Chinese participants - Boys in Chinese team = 160 - 60 = 100", - ], - "potential_final_choices": ["60", "100", "80", "120"], - "final_choice": 2, -} -``` - -This meant that instead of the final answer of 100, our model was generating potential responses it could give and returning the final choice as the index of that answer. Simply renaming our response model here to `potential_final_answers` and `final_answer` resulted in the original result of `95%` again. - -```python -{ - "chain_of_thought": "First, we need to determine how many Asians were Chinese. Since there were 240 Asians in total and 80 of them were Japanese, we can find the number of Chinese by subtracting the number of Japanese from the total: 240 - 80 = 160. Now, we know that there are 160 Chinese participants. Given that there were 60 boys on the Chinese team, we can find the number of girls by subtracting the number of boys from the total number of Chinese: 160 - 60 = 100. Therefore, there are 100 girls on the Chinese team.", - "necessary_calculations": [ - "Total Asians = 240", - "Number of Japanese = 80", - "Number of Chinese = 240 - 80 = 160", - "Number of boys on Chinese team = 60", - "Number of girls on Chinese team = 160 - 60 = 100", - ], - "potential_final_answers": ["100", "60", "80", "40"], - "answer": 100, -} -``` - -These are the sort of insights we’d only be able to know by having a strong evaluation set and looking closely at our generated predictions. - -## Why Care about the response model? - -It’s pretty obvious that different combinations of field names dramatically impact the performance of models. Ultimately It’s not just about adding a single `chain_of_thought` field but also about paying close attention to how models are interpreting the field names. - -For instance, instead of asking for just chain_of_thought, we can be much more creative by prompting our model to generate python code, much like the example below. - -```python -class Equations(BaseModel): - chain_of_thought: str - eval_string: list[str] = Field( - description="Python code to evaluate to get the final answer. The final answer should be stored in a variable called `answer`." - ) -``` - -This allows us to combine a LLM’s expressiveness with the performance of a deterministic system, in this case a python interpreter. As we continue to implement more complex systems with these models, the key isn’t going to be just toggling JSON mode and praying for the best. Instead, we need robust evaluation sets for testing the impact of different response models, prompt changes and other permutations. - -## Try Instructor Today - -`instructor` makes it easy to get structured data from LLMs and is built on top of Pydantic. This makes it an indispensable tool to quickly prototype and find the right response models for your specific application. - -To get started with instructor today, check out our [Getting Started](../../index.md) and [Examples](../../examples/index.md) sections that cover various LLM providers and specialised implementations. \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/best_framework.md b/참고/instructor-main/docs/blog/posts/best_framework.md deleted file mode 100644 index d468280..0000000 --- a/참고/instructor-main/docs/blog/posts/best_framework.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -authors: -- jxnl -categories: -- LLM Techniques -comments: true -date: 2024-03-05 -description: Discover how the Instructor library simplifies structured LLM outputs - using Python type annotations for seamless data mapping. -draft: false -slug: zero-cost-abstractions -tags: -- Instructor -- LLM Outputs -- Python -- Pydantic -- Data Mapping ---- - -# Why Instructor is the Best Library for Structured LLM Outputs - -Large language models (LLMs) like GPTs are incredibly powerful, but working with their open-ended text outputs can be challenging. This is where the Instructor library shines - it allows you to easily map LLM outputs to structured data using Python type annotations. - - - -The core idea behind Instructor is incredibly simple: it's just a patch over the OpenAI Python SDK that adds a response_model parameter. This parameter lets you pass in a Pydantic model that describes the structure you want the LLM output mapped to. Pydantic models are defined using standard Python type hints, so there's zero new syntax to learn. - -Here's an example of extracting structured user data from an LLM: - -```python -from pydantic import BaseModel -import instructor - - -class User(BaseModel): - name: str - age: int - - -client = instructor.from_provider("openai/gpt-5-nano") - -user = client.create( - model="gpt-3.5-turbo", - response_model=User, # (1)! - messages=[ - { - "role": "user", - "content": "Extract the user's name and age from this: John is 25 years old", - } - ], -) - -print(user) # (2)! -#> name='John' age=25 -``` - -1. Notice that now we have a new response_model parameter that we pass in to the completions.create method. This parameter lets us specify the structure we want the LLM output to be mapped to. In this case, we're using a Pydantic model called User that describes a user's name and age. -2. The output of the completions.create method is a User object that matches the structure we specified in the response_model parameter, rather than a ChatCompletion. - -## Other Features - -Other features on instructor, in and out of the llibrary are: - -1. Ability to use [Tenacity in retrying logic](../../concepts/retrying.md) -2. Ability to use [Pydantic's validation context](../../concepts/reask_validation.md) -3. [Parallel Tool Calling](../../concepts/parallel.md) with correct types -4. Streaming [Partial](../../concepts/partial.md) and [Iterable](../../concepts/iterable.md) data. -5. Returning [Primitive](../../concepts/types.md) Types and [Unions](../../concepts/unions.md) as well! -6. Lots of [Cookbooks](../../examples/index.md), [Tutorials](../../tutorials/1-introduction.ipynb), and comprehensive Documentation in our [Integration Guides](../../integrations/index.md) - -## Instructor's Broad Applicability - -One of the key strengths of Instructor is that it's designed as a lightweight patch over the official OpenAI Python SDK. This means it can be easily integrated not just with OpenAI's hosted API service, but with any provider or platform that exposes an interface compatible with the OpenAI SDK. - -For example, providers like [Together](../../integrations/together.md), [Ollama](../../integrations/ollama.md), [Groq](../../integrations/groq.md), and [llama-cpp-python](../../integrations/llama-cpp-python.md) all either use or mimic the OpenAI Python SDK under the hood. With Instructor's zero-overhead patching approach, teams can immediately start deriving structured data outputs from any of these providers. There's no need for custom integration work. - -## Direct access to the messages array - -Unlike other libraries that abstract away the `messages=[...]` parameter, Instructor provides direct access. This direct approach facilitates intricate prompt engineering, ensuring compatibility with OpenAI's evolving message types, including future support for images, audio, or video, without the constraints of string formatting. - -## Low Abstraction - -What makes Instructor so powerful is how seamlessly it integrates with existing OpenAI SDK code. To use it, you literally just call instructor.from_openai() on your OpenAI client instance, then use response_model going forward. There's no complicated refactoring or new abstractions to wrap your head around. - -This incremental, zero-overhead adoption path makes Instructor perfect for sprinkling structured LLM outputs into an existing OpenAI-based application. You can start extracting data models from simple prompts, then incrementally expand to more complex hierarchical models, streaming outputs, and custom validations. - -And if you decide Instructor isn't a good fit after all, removing it is as simple as not applying the patch! The familiarity and flexibility of working directly with the OpenAI SDK is a core strength. - -Instructor solves the "string hellll" of unstructured LLM outputs. It allows teams to easily realize the full potential of tools like GPTs by mapping their text to type-safe, validated data structures. If you're looking to get more structured value out of LLMs, give Instructor a try! - -## Related Concepts - -- [Philosophy](../../concepts/philosophy.md) - Understand Instructor's design principles -- [Patching](../../concepts/patching.md) - Learn how Instructor patches LLM clients -- [Retrying](../../concepts/retrying.md) - Handle validation failures gracefully -- [Streaming](../../concepts/partial.md) - Work with streaming responses - -## See Also - -- [Introduction to Instructor](introduction.md) - Get started with structured outputs -- [Integration Guides](../../integrations/index.md) - See all supported providers -- [Type Examples](../../concepts/types.md) - Explore different response types diff --git a/참고/instructor-main/docs/blog/posts/caching.md b/참고/instructor-main/docs/blog/posts/caching.md deleted file mode 100644 index 58ee8a0..0000000 --- a/참고/instructor-main/docs/blog/posts/caching.md +++ /dev/null @@ -1,975 +0,0 @@ ---- -authors: -- jxnl -categories: -- Performance Optimization -- Cost Reduction -- API Efficiency -- Python Development -comments: true -date: 2023-11-26 -description: Master advanced Python caching strategies for LLM applications using functools, diskcache, and Redis. Learn how to optimize OpenAI API costs, reduce response times, and implement efficient caching for Pydantic models in production environments. -draft: false -slug: python-caching-llm-optimization -tags: -- Python -- Caching -- Pydantic -- Performance Optimization -- Redis -- OpenAI -- API Cost Optimization -- functools -- diskcache -- LLM Applications -- Production Scaling -- Memory Management -- Distributed Systems -- Async Programming -- Batch Processing ---- - -# Advanced Caching Strategies for Python LLM Applications (Validated & Tested ✅) - -> Instructor makes working with language models easy, but they are still computationally expensive. Smart caching strategies can reduce costs by up to 90% while dramatically improving response times. - - -> **Update (June 2025)** – Instructor now ships *native* caching support -> out-of-the-box. Pass a cache adapter directly when you create a -> client: -> -> ```python -> from instructor import from_provider -> from instructor.cache import AutoCache, RedisCache -> -> client = from_provider( -> "openai/gpt-4o", # or any other provider -> cache=AutoCache(maxsize=10_000), # in-process LRU -> # or cache=RedisCache(host="localhost") -> ) -> ``` -> -> Under the hood this uses the very same techniques explained below, so -> you can still roll your own adapter if you need a bespoke backend. The -> remainder of the post walks through the design rationale in detail and -> is fully compatible with the built-in implementation. - -## Built-in cache – feature matrix - -| Method / helper | Cached | What is stored | Notes | -|------------------------------------------|--------|-------------------------------------------------------|-------| -| `create(...)` | ✅ Yes | Parsed Pydantic model + raw completion JSON | | -| `create_with_completion(...)` | ✅ Yes | Same as above – second tuple element restored from cache | -| `create_partial(...)` | ❌ No | – | Streaming generators not cached (yet) | -| `create_iterable(...)` | ❌ No | – | Streaming generators not cached (yet) | -| Any call with `stream=True` | ❌ No | – | Provider always invoked | - -### How serialization works - -1. **Model** – we call `model_dump_json()` which produces a compact, loss-less JSON string. On a cache hit we re-hydrate with `model_validate_json()` so you get the same `BaseModel` subclass instance. -2. **Raw completion** – Instructor attaches the original `ChatCompletion` (or provider-specific) object to the model as `_raw_response`. We serialise this object too (when possible with `model_dump_json()`, otherwise a plain `str()` fallback) and restore it on a cache hit so `create_with_completion()` behaves identically. - -#### Raw Response Reconstruction - -For raw completion objects, we use a `SimpleNamespace` trick to reconstruct the original object structure: - -```python -# When caching: -raw_json = completion.model_dump_json() # Serialize to JSON - -# When restoring from cache: -import json -from types import SimpleNamespace - -restored = json.loads(raw_json, object_hook=lambda d: SimpleNamespace(**d)) -``` - -This approach allows us to restore the original dot-notation access patterns (e.g., `completion.usage.total_tokens`) without requiring the original class definitions. The `SimpleNamespace` objects behave identically to the original completion objects for attribute access while being much simpler to reconstruct from JSON. - -#### Defensive Handling - -The cache implementation includes multiple fallback strategies for different provider response types: - -1. **Pydantic models** (OpenAI, Anthropic) - Use `model_dump_json()` for perfect serialization -2. **Plain dictionaries** - Use standard `json.dumps()` with `default=str` fallback -3. **Unpickleable objects** - Fall back to string representation with a warning - -This ensures the cache works reliably across all providers, even if they don't follow the same response object patterns. - -### Streaming limitations - -The current implementation opts **not** to cache streaming helpers (`create_partial`, `create_iterable`, or `stream=True`). Replaying a realistic token-stream requires a dedicated design which is coming in a future release. Until then, those calls always reach the provider. - -Today, we're diving deep into optimizing instructor code while maintaining the excellent developer experience offered by [Pydantic](https://docs.pydantic.dev/latest/) models. We'll tackle the challenges of caching Pydantic models, typically incompatible with `pickle`, and explore comprehensive solutions using `decorators` like `functools.cache`. Then, we'll craft production-ready custom decorators with `diskcache` and `redis` to support persistent caching, distributed systems, and high-throughput applications. - - - -## The Cost of Repeated API Calls - -Let's first consider our canonical example, using the `OpenAI` Python client to extract user details: - -```python -import instructor -from pydantic import BaseModel - -# Enables `response_model` -client = instructor.from_provider("openai/gpt-5-nano") - - -class UserDetail(BaseModel): - name: str - age: int - - -def extract(data) -> UserDetail: - return client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": data}, - ], - ) -``` - -Now imagine batch processing data, running tests or experiments, or simply calling `extract` multiple times over a workflow. We'll quickly run into performance issues, as the function may be called repeatedly, and the same data will be processed over and over again, costing us time and money. - -### Real-World Cost Impact - -Consider these scenarios where caching becomes critical: - -- **Development & Testing**: Running the same test cases repeatedly during development -- **Batch Processing**: Processing large datasets with potential duplicates -- **Web Applications**: Multiple users requesting similar information -- **Data Pipelines**: ETL processes that might encounter the same data multiple times -- **Model Experimentation**: Testing different prompts on the same input data - -Without caching, a single GPT-4 call costs approximately $0.03 per 1K prompt tokens and $0.06 per 1K completion tokens. For applications making thousands of calls per day, this quickly adds up to significant expenses. - -## 1. `functools.cache` for Simple In-Memory Caching - -**When to Use**: Ideal for functions with immutable arguments, called repeatedly with the same parameters in small to medium-sized applications. Perfect for development environments, testing, and applications where you don't need cache persistence between sessions. - -```python -import functools - - -@functools.cache -def extract(data): - return client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": data}, - ], - ) -``` - -!!! warning "Cache Invalidation Considerations" - - Note that changing the model parameter does not invalidate the cache. This is because the cache key is based on the function's name and arguments, not the model. Consider including model parameters in your cache key for production applications. - -Let's see the dramatic performance impact in action: - -```python hl_lines="4 8 12" -import time - -start = time.perf_counter() # (1) -model = extract("Extract jason is 25 years old") -print(f"Time taken: {time.perf_counter() - start}") - -start = time.perf_counter() -model = extract("Extract jason is 25 years old") # (2) -print(f"Time taken: {time.perf_counter() - start}") - -#> Time taken: 0.104s -#> Time taken: 0.000s # (3) -#> Speed improvement: 207,636x faster! -``` - -1. Using `time.perf_counter()` to measure the time taken to run the function is better than using `time.time()` because it's more accurate and less susceptible to system clock changes. -2. The second time we call `extract`, the result is returned from the cache, and the function is not called. -3. The second call to `extract` is **over 200,000x faster** because the result is returned from the cache! - -**Benefits**: Easy to implement, provides fast access due to in-memory storage, and requires no additional libraries. - -**Limitations**: -- Cache is lost when the process restarts -- Memory usage grows with cache size -- Not suitable for distributed applications -- No cache size limits by default - -??? question "What is a decorator?" - - A decorator is a function that takes another function and extends the behavior of the latter function without explicitly modifying it. In Python, decorators are functions that take a function as an argument and return a closure. - - ```python hl_lines="3-5 9" - def decorator(func): - def wrapper(*args, **kwargs): - print("Do something before") # (1) - #> Do something before - result = func(*args, **kwargs) - print("Do something after") # (2) - #> Do something after - return result - - return wrapper - - - @decorator - def say_hello(): - #> Hello! - print("Hello!") - #> Hello! - - - say_hello() - #> "Do something before" - #> "Hello!" - #> "Do something after" - ``` - - 1. The code is executed before the function is called - 2. The code is executed after the function is called - -### Advanced functools Caching Patterns - -For more control over in-memory caching, consider `functools.lru_cache`: - -```python -import functools - - -@functools.lru_cache(maxsize=1000) # Limit cache to 1000 entries -def extract_with_limit(data: str, model: str = "gpt-3.5-turbo") -> UserDetail: - return client.create( - model=model, - response_model=UserDetail, - messages=[ - {"role": "user", "content": data}, - ], - ) -``` - -This provides: -- Memory usage control through `maxsize` -- Automatic eviction of least recently used items -- Cache statistics via `cache_info()` - -## 2. `diskcache` for Persistent, Large Data Caching - -??? note "Production-Ready Caching Code" - - We'll be using the same `instructor_cache` decorator for both `diskcache` and `redis` caching. This production-ready code includes error handling, type safety, and async support. - - ```python - import functools - import inspect - import diskcache - from typing import Any, Callable, TypeVar - import hashlib - import json - - cache = diskcache.Cache('./my_cache_directory') # (1) - - F = TypeVar('F', bound=Callable[..., Any]) - - - def instructor_cache( - cache_key_fn: Callable[[Any], str] | None = None, ttl: int | None = None - ) -> Callable[[F], F]: - """ - Advanced cache decorator for functions that return Pydantic models. - - Args: - cache_key_fn: Optional function to generate custom cache keys - ttl: Time to live in seconds (None for no expiration) - """ - - def decorator(func: F) -> F: - return_type = inspect.signature(func).return_annotation - if not issubclass(return_type, BaseModel): # (2) - raise ValueError("The return type must be a Pydantic model") - - @functools.wraps(func) - def wrapper(*args, **kwargs): - # Generate cache key - if cache_key_fn: - key = cache_key_fn((args, kwargs)) - else: - # Include model schema in key for cache invalidation - schema_hash = hashlib.md5( - json.dumps(return_type.model_json_schema(), sort_keys=True).encode() - ).hexdigest()[:8] - key = f"{func.__name__}-{schema_hash}-{functools._make_key(args, kwargs, typed=False)}" - - # Check if the result is already cached - if (cached := cache.get(key)) is not None: - # Deserialize from JSON based on the return type - return return_type.model_validate_json(cached) - - # Call the function and cache its result - result = func(*args, **kwargs) - serialized_result = result.model_dump_json() - - if ttl: - cache.set(key, serialized_result, expire=ttl) - else: - cache.set(key, serialized_result) - - return result - - return wrapper - - return decorator - ``` - - 1. We create a new `diskcache.Cache` instance to store the cached data. This will create a new directory called `my_cache_directory` in the current working directory. - 2. We only want to cache functions that return a Pydantic model to simplify serialization and deserialization logic in this example code - -**When to Use**: Suitable for applications needing cache persistence between sessions, dealing with large datasets, or requiring cache durability. Perfect for: - -- **Development workflows** where you want to preserve cache between restarts -- **Data processing pipelines** that run periodically -- **Applications with expensive computations** that benefit from long-term caching -- **Local development** where you want to avoid repeated API calls - -```python hl_lines="10" -import functools -import inspect -import instructor -import diskcache - -from pydantic import BaseModel - -client = instructor.from_provider("openai/gpt-5-nano") -cache = diskcache.Cache('./my_cache_directory') - - -def instructor_cache(func): - """Cache a function that returns a Pydantic model""" - return_type = inspect.signature(func).return_annotation # (4) - if not issubclass(return_type, BaseModel): # (1) - raise ValueError("The return type must be a Pydantic model") - - @functools.wraps(func) - def wrapper(*args, **kwargs): - key = ( - f"{func.__name__}-{functools._make_key(args, kwargs, typed=False)}" # (2) - ) - # Check if the result is already cached - if (cached := cache.get(key)) is not None: - # Deserialize from JSON based on the return type (3) - return return_type.model_validate_json(cached) - - # Call the function and cache its result - result = func(*args, **kwargs) - serialized_result = result.model_dump_json() - cache.set(key, serialized_result) - - return result - - return wrapper - - -class UserDetail(BaseModel): - name: str - age: int - - -@instructor_cache -def extract(data) -> UserDetail: - return client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": data}, - ], - ) -``` - -1. We only want to cache functions that return a Pydantic model to simplify serialization and deserialization logic -2. We use functool's `_make_key` to generate a unique key based on the function's name and arguments. This is important because we want to cache the result of each function call separately. -3. We use Pydantic's `model_validate_json` to deserialize the cached result into a Pydantic model. -4. We use `inspect.signature` to get the function's return type annotation, which we use to validate the cached result. - -**Benefits**: -- Reduces computation time for heavy data processing -- Provides disk-based caching for persistence -- Survives application restarts -- Configurable size limits and eviction policies -- Thread-safe operations - -### Diskcache Performance Characteristics - -- **Read Performance**: ~10,000 reads/second -- **Write Performance**: ~5,000 writes/second -- **Storage Efficiency**: Compressed storage options available -- **Memory Usage**: Minimal memory footprint - -## 3. Redis Caching for Distributed Systems - -??? note "Production Redis Caching Code" - - Enhanced Redis implementation with connection pooling, error handling, and monitoring. - - ```python - import functools - import inspect - import redis - import json - import hashlib - from typing import Any, Callable, TypeVar - import logging - - # Configure Redis with connection pooling - redis_pool = redis.ConnectionPool( - host='localhost', port=6379, db=0, max_connections=20, decode_responses=True - ) - cache = redis.Redis(connection_pool=redis_pool) - - logger = logging.getLogger(__name__) - - F = TypeVar('F', bound=Callable[..., Any]) - - - def instructor_cache_redis( - ttl: int = 3600, # 1 hour default - prefix: str = "instructor", - retry_on_failure: bool = True, - ) -> Callable[[F], F]: - """ - Redis cache decorator for Pydantic models with production features. - - Args: - ttl: Time to live in seconds - prefix: Cache key prefix for namespacing - retry_on_failure: Whether to retry on Redis failures - """ - - def decorator(func: F) -> F: - return_type = inspect.signature(func).return_annotation - if not issubclass(return_type, BaseModel): - raise ValueError("The return type must be a Pydantic model") - - @functools.wraps(func) - def wrapper(*args, **kwargs): - # Generate cache key with schema versioning - schema_hash = hashlib.md5( - json.dumps(return_type.model_json_schema(), sort_keys=True).encode() - ).hexdigest()[:8] - key = f"{prefix}:{func.__name__}:{schema_hash}:{functools._make_key(args, kwargs, typed=False)}" - - try: - # Check if the result is already cached - if (cached := cache.get(key)) is not None: - logger.debug(f"Cache hit for key: {key}") - return return_type.model_validate_json(cached) - - logger.debug(f"Cache miss for key: {key}") - except redis.RedisError as e: - logger.warning(f"Redis error during read: {e}") - if not retry_on_failure: - # Call function directly if Redis fails and retry is disabled - return func(*args, **kwargs) - - # Call the function and cache its result - result = func(*args, **kwargs) - serialized_result = result.model_dump_json() - - try: - cache.setex(key, ttl, serialized_result) - logger.debug(f"Cached result for key: {key}") - except redis.RedisError as e: - logger.warning(f"Redis error during write: {e}") - - return result - - return wrapper - - return decorator - ``` - -**When to Use**: Recommended for distributed systems where multiple processes need to access the cached data, high-throughput applications, or microservices architectures. Ideal for: - -- **Production web applications** with multiple instances -- **Distributed data processing** across multiple workers -- **Microservices** that need shared caching -- **High-frequency trading** or real-time applications -- **Multi-tenant applications** with shared cache needs - -```python -import redis -import functools -import inspect -import instructor - -from pydantic import BaseModel - -client = instructor.from_provider("openai/gpt-5-nano") -cache = redis.Redis("localhost") - - -def instructor_cache(func): - """Cache a function that returns a Pydantic model""" - return_type = inspect.signature(func).return_annotation - if not issubclass(return_type, BaseModel): # (1) - raise ValueError("The return type must be a Pydantic model") - - @functools.wraps(func) - def wrapper(*args, **kwargs): - key = f"{func.__name__}-{functools._make_key(args, kwargs, typed=False)}" # (2) - # Check if the result is already cached - if (cached := cache.get(key)) is not None: - # Deserialize from JSON based on the return type - return return_type.model_validate_json(cached) - - # Call the function and cache its result - result = func(*args, **kwargs) - serialized_result = result.model_dump_json() - cache.set(key, serialized_result) - - return result - - return wrapper - - -class UserDetail(BaseModel): - name: str - age: int - - -@instructor_cache -def extract(data) -> UserDetail: - # Assuming client.chat.completions.create returns a UserDetail instance - return client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": data}, - ], - ) -``` - -1. We only want to cache functions that return a Pydantic model to simplify serialization and deserialization logic -2. We use functool's `_make_key` to generate a unique key based on the function's name and arguments. This is important because we want to cache the result of each function call separately. - -**Benefits**: -- Scalable for large-scale systems -- Supports fast in-memory data storage and retrieval -- Versatile for various data types -- Built-in expiration and eviction policies -- Monitoring and observability features -- Atomic operations and transactions - -### Redis Performance Characteristics - -- **Throughput**: 100,000+ operations/second on modern hardware -- **Latency**: Sub-millisecond response times -- **Scalability**: Cluster mode for horizontal scaling -- **Persistence**: Optional disk persistence for durability - -!!! note "Implementation Consistency" - - If you look carefully at the code above, you'll notice that we're using the same `instructor_cache` decorator interface for all backends. The implementation details vary, but the API remains consistent, making it easy to switch between caching strategies. - -## Performance Benchmarks and Cost Analysis - -### Caching Performance Comparison - -Here's a **validated** real-world performance comparison across different caching strategies: - -| Strategy | First Call | Cached Call | Speed Improvement | Memory Usage | Persistence | Validated ✓ | -|----------|------------|-------------|-------------------|--------------|-------------|-------------| -| No Cache | 104ms | 104ms | 1x | Low | No | ✅ | -| **functools.cache** | 104ms | **0.0005ms** | **207,636x** | Medium | No | ✅ | -| diskcache | 104ms | 10-20ms | 5-10x | Low | Yes | ✅ | -| Redis (local) | 104ms | 2-5ms | 20-50x | Low | Yes | ✅ | -| Redis (network) | 104ms | 15-30ms | 3-7x | Low | Yes | ✅ | - -!!! success "Validated Performance" - - These numbers are from actual test runs using our comprehensive [caching examples](https://github.com/jxnl/instructor/tree/main/examples/caching). The `functools.cache` result showing **207,636x improvement** demonstrates the dramatic impact of in-memory caching. - -### Cost Impact Analysis - -Real-world cost savings validated across different application scales: - -| Application Scale | Daily Calls | Hit Rate | Daily Cost (No Cache) | Daily Cost (Cached) | Monthly Savings | -|-------------------|-------------|----------|----------------------|---------------------|-----------------| -| **Small App** | 1,000 | 50% | $2.00 | $1.00 | **$30.00** (50%) | -| **Medium App** | 10,000 | 70% | $20.00 | $6.00 | **$420.00** (70%) | -| **Large App** | 100,000 | 80% | $200.00 | $40.00 | **$4,800.00** (80%) | - -```python -# Real calculation function used in our tests -def calculate_cost_savings( - total_calls: int, cache_hit_rate: float, cost_per_call: float = 0.002 -): - cache_misses = total_calls * (1 - cache_hit_rate) - cost_without_cache = total_calls * cost_per_call - cost_with_cache = cache_misses * cost_per_call - savings = cost_without_cache - cost_with_cache - savings_percent = (savings / cost_without_cache) * 100 - return savings, savings_percent - - -# Example: Medium application -daily_savings, percent_saved = calculate_cost_savings(10000, 0.7) -monthly_savings = daily_savings * 30 -print(f"Monthly savings: ${monthly_savings:.2f} ({percent_saved:.1f}%)") -#> Monthly savings: $420.00 (70.0%) -``` - -These numbers demonstrate that **caching isn't just about performance-it's about sustainable cost management** for production LLM applications. - -## Advanced Caching Patterns - -### 1. Hierarchical Caching - -Combine multiple caching layers for optimal performance: - -```python -import functools - -# L1: In-memory cache (fastest) -# L2: Local disk cache (fast, persistent) -# L3: Redis cache (shared, network) - - -@functools.lru_cache(maxsize=100) # L1 -def extract_l1(data: str) -> UserDetail: - return extract_l2(data) - - -@diskcache_decorator # L2 -def extract_l2(data: str) -> UserDetail: - return extract_l3(data) - - -@redis_decorator # L3 -def extract_l3(data: str) -> UserDetail: - return client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[{"role": "user", "content": data}], - ) -``` - -### 2. Smart Cache Invalidation (Validated ✅) - -Implement intelligent cache invalidation based on model schema changes. **This feature has been tested and validated** to prevent stale data when your Pydantic models evolve: - -```python -def smart_cache_key( - func_name: str, args: tuple, kwargs: dict, model_class: type -) -> str: - """Generate cache key that includes model schema hash for automatic invalidation.""" - import hashlib - import json - - # Include model schema in cache key - schema_hash = hashlib.md5( - json.dumps(model_class.model_json_schema(), sort_keys=True).encode() - ).hexdigest()[:8] - - args_hash = hashlib.md5(str((args, kwargs)).encode()).hexdigest()[:8] - - return f"{func_name}:{schema_hash}:{args_hash}" - - -# Real test results showing this works: -# UserV1 cache key: extract:d4860f8f:9d4cb5ab -# UserV2 cache key: extract:9c28311a:9d4cb5ab (different schema hash!) -# Keys are different: True ✅ Schema-based invalidation works! -``` - -When you add a field to your model (like adding `email: Optional[str]` to a `User` model), the schema hash changes automatically, ensuring your cache doesn't return stale data with the old structure. - -### 3. Async Caching for High-Throughput Applications - -For applications using async/await patterns: - -```python -import aioredis - - -class AsyncInstructorCache: - def __init__(self, redis_url: str = "redis://localhost"): - self.redis = aioredis.from_url(redis_url) - - def cache(self, ttl: int = 3600): - def decorator(func): - @functools.wraps(func) - async def wrapper(*args, **kwargs): - key = f"{func.__name__}:{hash((args, kwargs))}" - - # Try to get from cache - cached = await self.redis.get(key) - if cached: - return UserDetail.model_validate_json(cached) - - # Execute function and cache result - result = await func(*args, **kwargs) - await self.redis.setex(key, ttl, result.model_dump_json()) - return result - - return wrapper - - return decorator - - -# Usage -cache = AsyncInstructorCache() - - -@cache.cache(ttl=3600) -async def extract_async(data: str) -> UserDetail: - return await client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[{"role": "user", "content": data}], - ) -``` - -## Integration with Instructor Features - -### Caching with Streaming Responses - -Combine caching with [streaming responses](../../concepts/partial.md) for optimal user experience: - -```python -@instructor_cache -def extract_streamable(data: str) -> UserDetail: - """Cache the final result while still allowing streaming for new requests.""" - return client.create_partial( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[{"role": "user", "content": data}], - stream=True, - ) -``` - -### Batch Processing with Caching - -Optimize [batch operations](../../examples/batch_job_oai.md) using intelligent caching: - -```python -async def process_batch_with_cache(items: list[str]) -> list[UserDetail]: - """Process batch items with cache optimization.""" - tasks = [] - for item in items: - # Each item benefits from caching - task = extract_async(item) - tasks.append(task) - - return await asyncio.gather(*tasks) -``` - -### Cache Monitoring and Observability (Production-Tested ✅) - -Implement comprehensive monitoring for production caching. **This monitoring system has been validated** to provide actionable insights: - -```python -from collections import defaultdict -from typing import Dict, Any - - -class CacheMetrics: - """Production-ready cache monitoring with real-world validation""" - - def __init__(self): - self.hits = 0 - self.misses = 0 - self.total_time_saved = 0.0 - self.hit_rate_by_function: Dict[str, Dict[str, int]] = defaultdict( - lambda: {"hits": 0, "misses": 0} - ) - - def record_hit(self, func_name: str, time_saved: float): - self.hits += 1 - self.total_time_saved += time_saved - self.hit_rate_by_function[func_name]["hits"] += 1 - print(f"✅ Cache HIT for {func_name}, saved {time_saved:.3f}s") - - def record_miss(self, func_name: str): - self.misses += 1 - self.hit_rate_by_function[func_name]["misses"] += 1 - print(f"❌ Cache MISS for {func_name}") - - @property - def hit_rate(self) -> float: - total = self.hits + self.misses - return self.hits / total if total > 0 else 0.0 - - def get_stats(self) -> Dict[str, Any]: - return { - "hit_rate": f"{self.hit_rate:.2%}", - "total_hits": self.hits, - "total_misses": self.misses, - "time_saved_seconds": f"{self.total_time_saved:.3f}", - "function_stats": dict(self.hit_rate_by_function), - } - - -# Example output from real test run: -# ✅ Cache HIT for extract, saved 0.800s -# ❌ Cache MISS for extract -# ✅ Cache HIT for extract, saved 0.900s -# Final metrics: -# Cache hit rate: 60.00% -# Total time saved: 2.4s -``` - -This monitoring approach provides **immediate feedback** on cache performance and helps identify optimization opportunities in production. - -## Best Practices and Production Considerations - -### 1. Cache Key Design - -- **Include Model Schema**: Automatically invalidate cache when model structure changes -- **Namespace Keys**: Use prefixes to avoid collisions in shared caches -- **Version Keys**: Include application version for controlled invalidation - -### 2. Error Handling - -```python -def robust_cache_decorator(func): - """Cache decorator with comprehensive error handling.""" - - @functools.wraps(func) - def wrapper(*args, **kwargs): - try: - # Try cache first - if cached := get_from_cache(args, kwargs): - return cached - except Exception as e: - logger.warning(f"Cache read failed: {e}") - - # Execute function - result = func(*args, **kwargs) - - try: - # Try to cache result - set_cache(args, kwargs, result) - except Exception as e: - logger.warning(f"Cache write failed: {e}") - - return result - - return wrapper -``` - -### 3. Security Considerations - -- **Sensitive Data**: Never cache personally identifiable information -- **Access Control**: Implement proper cache key isolation for multi-tenant applications -- **Encryption**: Consider encrypting cached data for sensitive applications - -### 4. Cache Warming Strategies - -```python -async def warm_cache(common_queries: list[str]): - """Pre-populate cache with common queries.""" - tasks = [extract_async(query) for query in common_queries] - await asyncio.gather(*tasks, return_exceptions=True) - logger.info(f"Warmed cache with {len(common_queries)} entries") -``` - -## Performance Optimization Tips - -### 1. Right-Size Your Cache - -- **Memory Caches**: Use `maxsize` to prevent memory bloat -- **Disk Caches**: Configure size limits and eviction policies -- **Redis**: Monitor memory usage and configure appropriate eviction policies - -### 2. Choose Optimal TTL Values - -```python -# Different TTL strategies based on data volatility -CACHE_TTL = { - "user_profiles": 3600, # 1 hour - relatively stable - "real_time_data": 60, # 1 minute - frequently changing - "static_content": 86400, # 24 hours - rarely changes - "expensive_computations": 604800, # 1 week - computational results -} -``` - -### 3. Cache Hit Rate Optimization - -- **Analyze Access Patterns**: Monitor which data is accessed most frequently -- **Implement Cache Warming**: Pre-populate cache with commonly accessed data -- **Use Consistent Hashing**: For distributed caches, ensure even distribution - -## Conclusion - -Choosing the right caching strategy depends on your application's specific needs, such as the size and type of data, the need for persistence, and the system's architecture. Whether it's optimizing a function's performance in a small application or managing large datasets in a distributed environment, Python offers robust solutions to improve efficiency and reduce computational overhead. - -The strategies we've covered provide a **validated, comprehensive toolkit**: - -- **functools.cache**: Perfect for development and single-process applications (✅ **207,636x speed improvement tested**) -- **diskcache**: Ideal for persistent caching with moderate performance needs (✅ **Production-ready examples included**) -- **Redis**: Essential for distributed systems and high-performance applications (✅ **Error handling validated**) - -Remember that caching is not just about performance-it's about providing a better user experience while managing costs effectively. Our **tested examples prove** that a well-implemented caching strategy can reduce API costs by 50-80% while improving response times by 5x to 200,000x. - -If you'd like to use this code, consider customizing it for your specific use case. For example, you might want to: - -- Encode the `Model.model_json_schema()` as part of the cache key for automatic invalidation -- Implement different TTL values for different types of data -- Add monitoring and alerting for cache performance -- Implement cache warming strategies for critical paths - -## Validated Examples & Testing - -All the caching strategies and performance claims in this guide have been **validated with working examples**: - -### 🧪 Test Your Own Caching -```bash -# Run comprehensive caching demonstration -cd examples/caching -python run.py - -# Test individual strategies -python test_concepts.py -``` - -### 📊 Real Results You'll See -``` -🚀 Testing functools.lru_cache -First call (miss): 0.104s -> processed: test data -Second call (hit): 0.000s -> processed: test data -Speed improvement: 207,636x faster -Cache info: CacheInfo(hits=1, misses=1, maxsize=128, currsize=1) - -💰 Cost Analysis Results: -Medium app, 70% hit rate: - Daily calls: 10,000 - Monthly savings: $420.00 (70.0%) -``` - -These are **actual results** from running the examples, not theoretical projections. - -## Related Resources - -### Core Concepts -- [Caching Strategies](../../concepts/caching.md) - Deep dive into caching patterns for LLM applications -- [Prompt Caching](../../concepts/prompt_caching.md) - Provider-specific caching features from OpenAI and Anthropic -- [Performance Optimization](../../concepts/parallel.md) - Parallel processing for better performance -- [Dictionary Operations](../../concepts/dictionary_operations.md) - Low-level optimization techniques - -### Working Examples -- [**Caching Examples**](https://github.com/jxnl/instructor/tree/main/examples/caching) - **Complete working examples** validating all strategies -- [Streaming Responses](../../concepts/partial.md) - Combine caching with real-time streaming -- [Async Processing](../../blog/posts/learn-async.md) - Async patterns for high-throughput applications -- [Batch Processing](../../examples/batch_job_oai.md) - Efficient batch operations with caching - -### Provider-Specific Features -- [Anthropic Prompt Caching](anthropic-prompt-caching.md) - Using Anthropic's native caching features -- [OpenAI API Usage Monitoring](../../cli/usage.md) - Track and optimize API costs - -### Production Scaling -- [Cost Optimization](../../faq.md#performance-and-costs) - Comprehensive cost reduction strategies -- [API Rate Limiting](../../faq.md#how-do-i-handle-rate-limits) - Handle rate limits with caching - -If you like the content, check out our [GitHub](https://github.com/jxnl/instructor) and give us a star to support the project! \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/chain-of-density.md b/참고/instructor-main/docs/blog/posts/chain-of-density.md deleted file mode 100644 index 4f2f867..0000000 --- a/참고/instructor-main/docs/blog/posts/chain-of-density.md +++ /dev/null @@ -1,546 +0,0 @@ ---- -authors: -- ivanleomk -- jxnl -categories: -- LLM Techniques -comments: true -date: 2023-11-05 -description: Learn to implement Chain of Density with GPT-3.5 for improved summarization, - achieving 20x latency reduction and 50x cost savings. -draft: false -slug: chain-of-density -tags: -- GPT-3.5 -- Chain of Density -- Summarization -- LLM Techniques -- Fine-tuning ---- - -# Smarter Summaries w/ Finetuning GPT-3.5 and Chain of Density - -> Discover how to distil an iterative method like Chain Of Density into a single finetuned model using Instructor - -In this article, we'll guide you through implementing the original Chain of Density method using Instructor, then show how to distile a GPT 3.5 model to match GPT-4's iterative summarization capabilities. Using these methods were able to decrease latency by 20x, reduce costs by 50x and maintain entity density. - -By the end you'll end up with a GPT 3.5 model, (fine-tuned using Instructor's great tooling), capable of producing summaries that rival the effectiveness of Chain of Density [[Adams et al. (2023)]](https://arxiv.org/abs/2309.04269). As always, all code is readily available in our `examples/chain-of-density` folder in our repo for your reference. - - - -??? abstract "Datasets and Colab Notebook" - - We've also uploaded all our generated data to Hugging Face [here](https://huggingface.co/datasets/ivanleomk/gpt4-chain-of-density) for you to use if you'd like to try reproducing these experiments. We've also added a [Colab Instance](https://colab.research.google.com/drive/1iBkrEh2G5U8yh8RmI8EkWxjLq6zIIuVm?usp=sharing) for you to check our generated values. - -## Part 1) Chain of Density - -Summarizing extensive texts with AI can be challenging, often relying on inconsistent techniques. Their novel method, Chain Of Density prompting, enhances AI-based text summarization, outperforming human-generated summaries. - -Initially, an AI produces a summary, then refines it through multiple iterations, adding missing article entities. Each iteration adds new article entities to the summary, keeping length consistent, leading to an entity-dense, informative summary called Chain Of Density. - -First introduced in the paper - [From Sparse to Dense: GPT-4 Summarization with Chain of Density Prompting](https://arxiv.org/abs/2309.04269). The team has found that this method is able to consistently beats similar summaries written by human annotators. - -??? info "Implementation Details" - - Note that our implementation uses a validator to ensure that the rewritten summary has a minimum length rather than a prompt. We also perform just 3 and not 5 rounds of rewrites, resulting in a lower final entity density. - -### Original Prompt - -We can break down the original process into smaller api calls. This allows us to introduce validation at each step to ensure that we're getting the results that we want. - -??? note "Original Chain of Density Prompt" - - ``` - Article: {{ARTICLE}} - - You will generate increasingly concise, entity-dense summaries of the - above Article. - - Repeat the following 2 steps 5 times. - - Step 1. Identify 1-3 informative Entities (";" delimited) from the - Article which are missing from the previously generated summary. - Step 2. Write a new, denser summary of identical length which covers - every entity and detail from the previous summary plus the Missing - Entities. - - A Missing Entity is: - - Relevant: to the main story. - - Specific: descriptive yet concise (5 words or fewer). - - Novel; not in the previous summary. - - Faithful: present in the Article. - - Anywhere: located anywhere in the Article. - - Guidelines: - - The first summary should be long (4-5 sentences, -80 words) yet - highly non-specific, containing little information beyond the - entities marked as missing. Use overly verbose language and fillers - (e.g., "this article discusses") to reach -80 words. - - Make every word count: re-write the previous summary to improve - flow and make space for additional entities. - - Make space with fusion, compression, and removal of uninformative - phrases like "the article discusses" - - The summaries should become highly dense and concise yet - self-contained, e.g., easily understood without the Article. - - Missing entities can appear anywhere in the new summary. - - Never drop entities from the previous summary. If space cannot be - made, add fewer new entities. - - Remember, use the exact same number of words for each summary. - - Answer in JSON. The JSON should be a list (length 5) of dictionaries - whose keys are "Missing_Entities" and "Denser_Summary" - ``` - -
- ![RAG](img/chain-of-density.png) -
Improved process with Instructor
-
- -### Data Modelling - -Before we begin modelling the data, let's make sure we install all of our dependencies - -``` -pip install instructor aiohttp rich -``` - -#### Initial Summary - -Let's start by walking through some of the data models that we'll be using as the `response_model` for our open ai function calls - -Firstly, we'll need a data model for the initial summary that we will be generating. We'll take the description of this class straight from the original prompt. It's important to note that these docstrings serve a purpose, they are **directly used by the LLM when generating the outputs**. - -??? note "A quick note on Docstrings" - - Under the hood, Instructor parses the `response_model` that you give us into a function call for OpenAI to execute. This means that the final output will be closely linked to the Pydantic model you specify. - - For instance, this simple model that we later use in fine-tuning. - - ```py - class GeneratedSummary(BaseModel): - """ - This represents a highly concise summary that includes as many entities as possible from the original source article. - - An Entity is a real-world object that's assigned a name - for example, a person, country a product or a book title. - - Guidelines - - Make every word count - - The new summary should be highly dense and concise yet self-contained, eg., easily understood without the Article. - - Make space with fusion, compression, and removal of uninformative phrases like "the article discusses" - """ - - summary: str = Field( - ..., - description="This represents the final summary generated that captures the meaning of the original article which is as concise as possible. ", - ) - ``` - - We eventually transform it into an OpenAI function call as seen below. - - ``` - { - "functions": [ - { - "name": "GeneratedSummary", - "description": "This represents a highly concise summary that includes as many entities as possible from the original source article.\n\nAn Entity is a real-world object that's assigned a name - for example, a person, country a product or a book title.\n\nGuidelines\n- Make every word count\n- The new summary should be highly dense and concise yet self-contained, eg., easily understood without the Article.\n- Make space with fusion, compression, and removal of uninformative phrases like \"the article discusses\"", - "parameters": { - "type": "object", - "properties": { - "summary": { - "description": "This represents the final summary generated that captures the meaning of the original article which is as concise as possible. ", - "title": "Summary", - "type": "string" - } - }, - "required": [ - "summary" - ] - - } - } - ] - } - } - ``` - - Therefore this means that the more elaborate and detailed your descriptions are, the better the outputs you will be able to get back. But we don't just stop there, since it's all Pydantic under the hood, you can validate and parse the resulting output to make sure it is **exactly what you specify**. It's all python all the way down. - -```py -class InitialSummary(BaseModel): - """ - This is an initial summary which should be long ( 4-5 sentences, ~80 words) - yet highly non-specific, containing little information beyond the entities marked as missing. - Use overly verbose languages and fillers (Eg. This article discusses) to reach ~80 words. - """ - - summary: str = Field( - ..., - description="This is a summary of the article provided which is overly verbose and uses fillers. It should be roughly 80 words in length", - ) -``` - -#### Rewritten Summary - -We'll also need one additional class to help model the rewritten schema - -```py -class RewrittenSummary(BaseModel): - """ - This is a new, denser summary of identical length which covers every entity - and detail from the previous summary plus the Missing Entities. - - Guidelines - - Make every word count : Rewrite the previous summary to improve flow and make space for additional entities - - Never drop entities from the previous summary. If space cannot be made, add fewer new entities. - - The new summary should be highly dense and concise yet self-contained, eg., easily understood without the Article. - - Make space with fusion, compression, and removal of uninformative phrases like "the article discusses" - - Missing entities can appear anywhere in the new summary - - An Entity is a real-world object that's assigned a name - for example, a person, country a product or a book title. - """ - - summary: str = Field( - ..., - description="This is a new, denser summary of identical length which covers every entity and detail from the previous summary plus the Missing Entities. It should have the same length ( ~ 80 words ) as the previous summary and should be easily understood without the Article", - ) - absent: List[str] = Field( - ..., - default_factory=list, - description="this is a list of Entities found absent from the new summary that were present in the previous summary", - ) - missing: List[str] = Field( - default_factory=list, - description="This is a list of 1-3 informative Entities from the Article that are missing from the new summary which should be included in the next generated summary.", - ) -``` - -!!! tip "Using Pydantic Validators with Instructor" - - For a more in-depth walkthrough on how to use `Pydantic` validators with the `Instructor` - library, we recommend checking out our previous article on LLM - validation - [Good LLM Validation is just Good Validation](../posts/validation-part1.md) - -Ideally, we'd like for `Missing` to have a length between 1 and 3, `Absent` to be an empty list and for our rewritten summaries to keep a minimum entity density. With `Instructor`, we can implement this logic using native `Pydantic` validators that are simply declared as part of the class itself. - -```py hl_lines="8 40 44" -import nltk -import spacy - -nlp = spacy.load("en_core_web_sm") - -@field_validator("summary") -def min_length(cls, v: str): - tokens = nltk.word_tokenize(v) #(1)! - num_tokens = len(tokens) - if num_tokens < 60: - raise ValueError( - "The current summary is too short. Please make sure that you generate a new summary that is around 80 words long." - ) - return v - -@field_validator("missing") -def has_missing_entities(cls, missing_entities: List[str]): - if len(missing_entities) == 0: - raise ValueError( - "You must identify 1-3 informative Entities from the Article which are missing from the previously generated summary to be used in a new summary" - ) - return missing_entities - -@field_validator("absent") -def has_no_absent_entities(cls, absent_entities: List[str]): - absent_entity_string = ",".join(absent_entities) - if len(absent_entities) > 0: - print(f"Detected absent entities of {absent_entity_string}") - raise ValueError( - f"Do not omit the following Entities {absent_entity_string} from the new summary" - ) - return absent_entities - -@field_validator("summary") -def min_entity_density(cls, v: str): - tokens = nltk.word_tokenize(v) - num_tokens = len(tokens) - - # Extract Entities - doc = nlp(v) #(2)! - num_entities = len(doc.ents) - - density = num_entities / num_tokens - if density < 0.08: #(3)! - raise ValueError( - f"The summary of {v} has too few entities. Please regenerate a new summary with more new entities added to it. Remember that new entities can be added at any point of the summary." - ) - - return v -``` - -1. Similar to the original paper, we utilize the `NLTK` word tokenizer to count the number of tokens within our generated sentences. - We aim for at least 60 tokens in our generated summary so that we don't lose information. - -2. We also use the spaCy library to calculate the entity density of the generated summary. - -3. We also implement a minimum entity density so that we stay within a given range. 0.08 is arbitrarily chosen in this case - -### Putting it all Together - -Now that we have our models and the rough flow figured out, let's implement a function to summarize a piece of text using `Chain Of Density` summarization. - -```python hl_lines="4 9-24 38-68" -import instructor -client = instructor.from_provider("openai/gpt-5-nano") #(1)! - -def summarize_article(article: str, summary_steps: int = 3): - summary_chain = [] - # We first generate an initial summary - summary: InitialSummary = client.create( # (2)! - model="gpt-4-0613", - response_model=InitialSummary, - messages=[ - { - "role": "system", - "content": "Write a summary about the article that is long (4-5 sentences) yet highly non-specific. Use overly, verbose language and fillers(eg.,'this article discusses') to reach ~80 words", - }, - {"role": "user", "content": f"Here is the Article: {article}"}, - { - "role": "user", - "content": "The generated summary should be about 80 words.", - }, - ], - max_retries=2, - ) - prev_summary = None - summary_chain.append(summary.summary) - for i in range(summary_steps): - missing_entity_message = ( - [] - if prev_summary is None - else [ - { - "role": "user", - "content": f"Please include these Missing Entities: {','.join(prev_summary.missing)}", - }, - ] - ) - new_summary: RewrittenSummary = client.create( # (3)! - model="gpt-4-0613", - messages=[ - { - "role": "system", - "content": """ - You are going to generate an increasingly concise,entity-dense summary of the following article. - - Perform the following two tasks - - Identify 1-3 informative entities from the following article which is missing from the previous summary - - Write a new denser summary of identical length which covers every entity and detail from the previous summary plus the Missing Entities - - Guidelines - - Make every word count: re-write the previous summary to improve flow and make space for additional entities - - Make space with fusion, compression, and removal of uninformative phrases like "the article discusses". - - The summaries should become highly dense and concise yet self-contained, e.g., easily understood without the Article. - - Missing entities can appear anywhere in the new summary - - Never drop entities from the previous summary. If space cannot be made, add fewer new entities. - """, - }, - {"role": "user", "content": f"Here is the Article: {article}"}, - { - "role": "user", - "content": f"Here is the previous summary: {summary_chain[-1]}", - }, - *missing_entity_message, - ], - max_retries=3, #(4)! - max_tokens=1000, - response_model=RewrittenSummary, - ) - summary_chain.append(new_summary.summary) - prev_summary = new_summary - - return summary_chain -``` - -1. We need to apply a `patch` function on the `OpenAI` client for us to get all - of the benefits that `Instructor` provides. With a simple `patch`, we can get - **automatic type coercion of our outputs and automatic retries for invalid outputs** - out of the box! - -2. We first generate an initial summary. Note here that we explicitly ask for a summary that has - 80 words and is lengthy with overly verbose fillers in the system prompt - -3. We slightly modify the original system prompt used in the original paper to perform a rewrite of the summary. - Using `Instructor`, we also get validation of the generated output with our `field_validator`s that we defined above - -4. If you've chosen a value that is larger than 0.08, make sure to increase this value in case you need to do multiple rewrites - -This summarization function yields a result which triples the number of entities while maintaining the same number of tokens. We can also see that stylistically, the summary is a lot more natural. - -**First Iteration** - -> This article discusses the highly-anticipated boxing match between Manny Pacquiao and Floyd Mayweather. The article revolves around Manny Pacquiao's statements about his upcoming fight and his preparations for the same. A portion of the article provides details about the financial stipulations of the match and its significance in the sporting arena. Quotes from Pacquiao illustrating his determination and his battle strategy are highlighted. The tone of the article is largely centered around creating a build-up to the upcoming mega event. - -**Final Iteration** - -> Manny Pacquiao, the Filipino boxer, anticipates the forthcoming May 2 showdown at the MGM Grand as the fight of his life, against the undefeated American Floyd Mayweather, in a $300m bout. Despite being seen as the underdog in this high-stakes Las Vegas match, Pacquiao is confident, promising a warrior's spirit and assuring the fans who have been awaiting this encounter for a decade, that it will indeed be the biggest sporting spectacle in history worthy of their anticipation - -## Part 2) Fine-Tuning - -In this section, we'll look into how to fine-tune a GPT 3.5 model so that it is able to perform at an equivalent level as a GPT-4 model. We'll then compare the performance of our model against that of `GPT-4` to see how it stacks up. - -### Creating a Training Set - -In order to prevent any contamination of data during testing, we randomly sampled 120 articles from the `griffin/chain-of-density` dataset and split these articles into a `train.csv` and a `test.csv` file which we uploaded to [Hugging Face](https://huggingface.co/datasets/ivanleomk/gpt4-chain-of-density). Now, we just neeed to import the `Instructions` module from the `Instructor` package which allows you to generate a nicely formatted `.jsonl` file to be used for fine-tuning - -```py hl_lines="2 9 11 13-21 40 43" -from typing import List -from chain_of_density import summarize_article #(1)! -import csv -import logging -import instructor -from pydantic import BaseModel -client = instructor.from_provider("openai/gpt-5-nano") # (2)! - -logging.basicConfig(level=logging.INFO) #(3)! - -instructions = instructor.Instructions( #(4)! - name="Chain Of Density", - finetune_format="messages", - # log handler is used to save the data to a file - # you can imagine saving it to a database or other storage - # based on your needs! - log_handlers=[logging.FileHandler("generated.jsonl")], - openai_client=client, -) - -class GeneratedSummary(BaseModel): - """ - This represents a highly concise summary that includes as many entities as possible from the original source article. - - An Entity is a real-world object that's assigned a name - for example, a person, country a product or a book title. - - Guidelines - - Make every word count - - The new summary should be highly dense and concise yet self-contained, eg., easily understood without the Article. - - Make space with fusion, compression, and removal of uninformative phrases like "the article discusses" - """ - - summary: str = Field( - ..., - description="This represents the final summary generated that captures the meaning of the original article which is as concise as possible. ", - ) - -@instructions.distil #(4)! -def distil_summarization(text: str) -> GeneratedSummary: - summary_chain: List[str] = summarize_article(text) - return GeneratedSummary(summary=summary_chain[-1]) #(5)! - -with open("train.csv", "r") as file: - reader = csv.reader(file) - next(reader) # Skip the header - for article, summary in reader: - # Run Distillisation to generate the values - distil_summarization(article) -``` - -1. In this example, we're using the summarize_article that we defined up above. We saved it in a local file called `chain_of_density.py`, - hence the import - -2. We patch the default OpenAI client so that we can use the Instructor library with it - -3. We also need to configure logging at the `INFO` level. This is very important, if this is not configured, your output will not be generated. - -4. We instantiate a `Instruction` object which will help us handle the conversion of our function calls into a valid `.jsonl` file. We also define - the name of the `.jsonl` file in the `log_handlers` parameter - -5. We add in an `instructions.distil` annotation so that we automatically capture the input and output of the function we'd like to - fine-tune our model to output - -6. We return a `Pydantic` object which matches the annotation that we use on our function. Note that we must specify a `Pydantic` object to - be returned when using the `instructions.distil` annotation - -!!! warning "Rate Limiting" - - We recommend running this script on a small subset of the dataset first to test you've got everything configured nicely. - Don't forget to add in rate limiting error handling with `tenacity` and set the `OPENAI_API_KEY` shell environment variable - before running any subsequent commands - -### Creating Fine-Tuning Jobs - -Once we run this script, we'll have a new file called `generated.jsonl` in our local repository. Now all that's left is to run the command below to start fine-tuning your first model! - -```sh -instructor jobs create-from-file generated.jsonl -``` - -??? notes "Finetuning Reference" - - Checking out our [Finetuning CLI](../../cli/finetune.md) to learn about other hyperparameters that you can tune to improve your model's performance. - -Once the job is complete, all we need to do is to then change the annotation in the function call to `distil_summarization` in our original file above to start using our new model. - -```py -@instructions.distil(model='gpt-3.5-turbo:finetuned-123', mode="dispatch") # (1)! -def distil_summarization(text: str) -> GeneratedSummary: - summary_chain: List[str] = summarize_article(text) - return GeneratedSummary(summary=summary_chain[-1]) -``` - -1. Don't forget to replace this with your new model id. OpenAI identifies fine tuned models with an id of - ft:gpt-3.5-turbo-0613:personal:: under their Fine-tuning tab on their dashboard - -With that, you've now got your own fine-tuned model ready to go and serve data in production. We've seen how Instructor can make your life easier, from fine-tuning to distillation. - -## Results and Benchmarks - -We'll be comparing the following models in 3 ways using 20 articles that were not used for fine-tuning. - -- Entity Density : This is entities per token, the higher the better for density. -- Latency : Time to last token generated in seconds -- Costs : Total cost to generate outputs - we break down the cost into training and inference costs for easy reference - -`3.5 Finetuned (n)` - -: This is a GPT 3.5 model that we fine-tuned on `n` examples. Each model was finetuned for 4-5 epochs ( This was automatically decided by the OpenAI scheduler ) - -`GPT-4 (COD)` - -: This is a GPT4 model which we applied 3 rounds of Chain Of Density rewrites to generate a summary with using the methodology above - -`GPT-3.5 (Vanilla)` - -: This is a GPT 3.5 model that we asked to generate entity-dense summaries which were concise. Summaries were generated in a single pass targeting about 80-90 tokens. - -| Model | Mean Latency (s) | Mean Entity Density | -| ------------------ | ---------------- | ------------------- | -| 3.5 Finetuned (20) | 2.1 | 0.15 | -| 3.5 Finetuned (50) | 2.1 | 0.14 | -| 3.5 Finetuned (76) | 2.1 | 0.14 | -| GPT-3.5 (Vanilla) | 16.8 | 0.12 | -| GPT-4 (COD) | 49.5 | 0.15 | - -??? notes "Finetuning Datasets" - - For our finetuned models, we did a few optimisations to raise the performance. - - We only included summaries that had a minimum density of 0.15 in the dataset, took the summary in the entire chain with the highest density as the final one, forced every regenerated summary to have a minimum density of 0.12 and regenerated summaries up to three times if they didn't meet the summaries. **This is a much more expensive strategy and can cost up to 2.5x or more what we do in this tutorial** - - This resulted in the total cost of $63.46 to generate just 75 examples due to the stringent requirements, translating to about $0.85 per generated summary example. - -Using the OpenAI Usage Dashboard, we can calculate the cost of generating 20 summaries as seen below. - -| Model | Training Cost ($) | Inference Cost ($) | Tokens Used | Total Cost ($) | -| ------------------ | ----------------- | ------------------ | ----------- | -------------- | -| GPT-3.5 (Vanilla) | - | 0.20 | 51,162 | 0.2 | -| 3.5 Finetuned (20) | 0.7 | 0.20 | 56,573 | 0.8 | -| 3.5 Finetuned (50) | 1.4 | 0.17 | 49,057 | 1.3 | -| 3.5 Finetuned (76) | 1.8 | 0.17 | 51,583 | 2.5 | -| GPT-4 (COD) | - | 12.9 | 409,062 | 12.9 | - -Here, we can see that `GPT-4` has an approximate inference cost of `0.65` per summary while our finetuned models have an inference cost of `0.0091` per summary which is ~ `72x` cheaper. - -Interestingly, the model finetuned with the least examples seems to outperform the others. While the reason for this is unknown, a few potential reasons could be that either we didn't train for sufficient epochs ( We chose the default 5 epochs ) or that the models started learning to imitate other behaviour such as more abstract writing styles from the larger variety of samples, resulting in a decrease in entity density. - -## Conclusions - -Finetuning this iterative method was 20-40x faster while improving overall performance, resulting in massive efficiency gains by finetuning and distilling capabilities into specialized models. - -We've seen how `Instructor` can make your life easier, from data modeling to distillation and finetuning. If you enjoy the content or want to try out `instructor` check out the [github](https://github.com/jxnl/instructor) and don't forget to give us a star! \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/chat-with-your-pdf-with-gemini.md b/참고/instructor-main/docs/blog/posts/chat-with-your-pdf-with-gemini.md deleted file mode 100644 index 8d1aa7d..0000000 --- a/참고/instructor-main/docs/blog/posts/chat-with-your-pdf-with-gemini.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -authors: - - ivanleomk -categories: - - Gemini - - Document Processing -comments: true -date: 2024-11-11 -description: Learn how to use Google's Gemini model with Instructor to process PDFs and extract structured information -draft: false -tags: - - Gemini - - Document Processing - - PDF Analysis - - Pydantic - - Python ---- - -# PDF Processing with Structured Outputs with Gemini - -In this post, we'll explore how to use Google's Gemini model with Instructor to analyse the [Gemini 1.5 Pro Paper](https://github.com/google-gemini/generative-ai-python/blob/0e5c5f25fe4ce266791fa2afb20d17dee780ca9e/third_party/test.pdf) and extract a structured summary. - -## The Problem - -Processing PDFs programmatically has always been painful. The typical approaches all have significant drawbacks: - -- **PDF parsing libraries** require complex rules and break easily -- **OCR solutions** are slow and error-prone -- **Specialized PDF APIs** are expensive and require additional integration -- **LLM solutions** often need complex document chunking and embedding pipelines - -What if we could just hand a PDF to an LLM and get structured data back? With Gemini's multimodal capabilities and Instructor's structured output handling, we can do exactly that. - -## Quick Setup - -First, install the required packages: - -```bash -pip install "instructor[google-generativeai]" -``` - -Then, here's all the code you need: - -```python -import instructor -import google.generativeai as genai -from google.ai.generativelanguage_v1beta.types.file import File -from pydantic import BaseModel -import time - -# Initialize the client -client = instructor.from_provider("google/gemini-2.5-flash") - - -# Define your output structure -class Summary(BaseModel): - summary: str - - -# Upload the PDF -file = genai.upload_file("path/to/your.pdf") - -# Wait for file to finish processing -while file.state != File.State.ACTIVE: - time.sleep(1) - file = genai.get_file(file.name) - print(f"File is still uploading, state: {file.state}") - -print(f"File is now active, state: {file.state}") -print(file) - -resp = client.create( - messages=[ - {"role": "user", "content": ["Summarize the following file", file]}, - ], - response_model=Summary, -) - -print(resp.summary) -``` - -??? note "Expand to see Raw Results" - - ```bash - summary="Gemini 1.5 Pro is a highly compute-efficient multimodal mixture-of-experts model capable of recalling and reasoning over fine-grained information from millions of tokens of context, including multiple long documents and hours of video and audio. It achieves near-perfect recall on long-context retrieval tasks across modalities, improves the state-of-the-art in long-document QA, long-video QA and long-context ASR, and matches or surpasses Gemini 1.0 Ultra's state-of-the-art performance across a broad set of benchmarks. Gemini 1.5 Pro is built to handle extremely long contexts; it has the ability to recall and reason over fine-grained information from up to at least 10M tokens. This scale is unprecedented among contemporary large language models (LLMs), and enables the processing of long-form mixed-modality inputs including entire collections of documents, multiple hours of video, and almost five days long of audio. Gemini 1.5 Pro surpasses Gemini 1.0 Pro and performs at a similar level to 1.0 Ultra on a wide array of benchmarks while requiring significantly less compute to train. It can recall information amidst distractor context, and it can learn to translate a new language from a single set of linguistic documentation. With only instructional materials (a 500-page reference grammar, a dictionary, and ≈ 400 extra parallel sentences) all provided in context, Gemini 1.5 Pro is capable of learning to translate from English to Kalamang, a Papuan language with fewer than 200 speakers, and therefore almost no online presence." - ``` - -## Benefits - -The combination of Gemini and Instructor offers several key advantages over traditional PDF processing approaches: - -**Simple Integration** - Unlike traditional approaches that require complex document processing pipelines, chunking strategies, and embedding databases, you can directly process PDFs with just a few lines of code. This dramatically reduces development time and maintenance overhead. - -**Structured Output** - Instructor's Pydantic integration ensures you get exactly the data structure you need. The model's outputs are automatically validated and typed, making it easier to build reliable applications. If the extraction fails, Instructor automatically handles the retries for you with support for [custom retry logic using tenacity](../../concepts/retrying.md). - -**Multimodal Support** - Gemini's multimodal capabilities mean this same approach works for various file types. You can process images, videos, and audio files all in the same api request. Check out our [multimodal processing guide](./multimodal-gemini.md) to see how we extract structured data from travel videos. - -## Conclusion - -Working with PDFs doesn't have to be complicated. - -By combining Gemini's multimodal capabilities with Instructor's structured output handling, we can transform complex document processing into simple, Pythonic code. - -No more wrestling with parsing rules, managing embeddings, or building complex pipelines - just define your data model and let the LLM do the heavy lifting. - -## Related Documentation -- [Multimodal Processing](../../concepts/multimodal.md) - Core multimodal concepts - -## See Also -- [Gemini Multimodal Features](multimodal-gemini.md) - Full Gemini capabilities -- [PDF Citation Generation](generating-pdf-citations.md) - Extract citations from PDFs -- [RAG and Beyond](rag-and-beyond.md) - Advanced document processing - -If you liked this, give `instructor` a try today and see how much easier structured outputs makes working with LLMs become. [Get started with Instructor today!](../../index.md) diff --git a/참고/instructor-main/docs/blog/posts/citations.md b/참고/instructor-main/docs/blog/posts/citations.md deleted file mode 100644 index 9b518d5..0000000 --- a/참고/instructor-main/docs/blog/posts/citations.md +++ /dev/null @@ -1,283 +0,0 @@ ---- -authors: -- jxnl -categories: -- Pydantic -comments: true -date: 2023-11-18 -description: Explore how Pydantic enhances LLM citation verification, improving data - accuracy and reliability in responses. -draft: false -slug: validate-citations -tags: -- Pydantic -- LLM -- Data Accuracy -- Citation Verification -- Python ---- - -# Verifying LLM Citations with Pydantic - -Ensuring the accuracy of information is crucial. This blog post explores how Pydantic's powerful and flexible validators can enhance data accuracy through citation verification. - -We'll start with using a simple substring check to verify citations. Then we'll use `instructor` itself to power an LLM to verify citations and align answers with the given citations. Finally, we'll explore how we can use these techniques to generate a dataset of accurate responses. - - - -## Example 1: Simple Substring Check - -In this example, we use the `Statements` class to verify if a given substring quote exists within a text chunk. If the substring is not found, an error is raised. - -### Code Example: - -```python -from typing import List -from pydantic import BaseModel, ValidationInfo, field_validator -import instructor - -client = instructor.from_provider("openai/gpt-5-nano") - - -class Statements(BaseModel): - body: str - substring_quote: str - - @field_validator("substring_quote") - @classmethod - def substring_quote_exists(cls, v: str, info: ValidationInfo): - context = info.context.get("text_chunks", None) - - for text_chunk in context.values(): - if v in text_chunk: # (1) - return v - raise ValueError("Could not find substring_quote `{v}` in contexts") - - -class AnswerWithCitaton(BaseModel): - question: str - answer: List[Statements] -``` - -1. While we use a simple substring check in this example, we can use more complex techniques like regex or Levenshtein distance. - -Once the class is defined, we can use it to validate the context and raise an error if the substring is not found. - -```python -try: - AnswerWithCitaton.model_validate( - { - "question": "What is the capital of France?", - "answer": [ - {"body": "Paris", "substring_quote": "Paris is the capital of France"}, - ], - }, - context={ - "text_chunks": { - 1: "Jason is a pirate", - 2: "Paris is not the capital of France", - 3: "Irrelevant data", - } - }, - ) -except ValidationError as e: - print(e) -``` - -### Error Message Example: - -``` -answer.0.substring_quote - Value error, Could not find substring_quote `Paris is the capital of France` in contexts [type=value_error, input_value='Paris is the capital of France', input_type=str] - For further information visit [https://errors.pydantic.dev/2.4/v/value_error](https://errors.pydantic.dev/2.4/v/value_error) -``` - -Pydantic raises a validation error when the `substring_quote` attribute does not exist in the context. This approach can be used to validate more complex data using techniques like regex or Levenshtein distance. - -## Example 2: Using LLM for Verification - -This approach leverages OpenAI's LLM to validate citations. If the citation does not exist in the context, the LLM returns an error message. - -### Code Example: - -```python -class Validation(BaseModel): - is_valid: bool - error_messages: Optional[str] = Field(None, description="Error messages if any") - - -class Statements(BaseModel): - body: str - substring_quote: str - - @model_validator(mode="after") - def substring_quote_exists(self, info: ValidationInfo): - context = info.context.get("text_chunks", None) - - resp: Validation = client.create( - response_model=Validation, - messages=[ - { - "role": "user", - "content": f"Does the following citation exist in the following context?\n\nCitation: {self.substring_quote}\n\nContext: {context}", - } - ], - model="gpt-3.5-turbo", - ) - - if resp.is_valid: - return self - - raise ValueError(resp.error_messages) - - -class AnswerWithCitaton(BaseModel): - question: str - answer: List[Statements] -``` - -Now when we use a correct citation, the LLM returns a valid response. - -```python -resp = AnswerWithCitaton.model_validate( - { - "question": "What is the capital of France?", - "answer": [ - {"body": "Paris", "substring_quote": "Paris is the capital of France"}, - ], - }, - context={ - "text_chunks": { - 1: "Jason is a pirate", - 2: "Paris is the capital of France", - 3: "Irrelevant data", - } - }, -) -print(resp.model_dump_json(indent=2)) -``` - -### Result: - -```json -{ - "question": "What is the capital of France?", - "answer": [ - { - "body": "Paris", - "substring_quote": "Paris is the capital of France" - } - ] -} -``` - -When we have citations that don't exist in the context, the LLM returns an error message. - -```python -try: - AnswerWithCitaton.model_validate( - { - "question": "What is the capital of France?", - "answer": [ - {"body": "Paris", "substring_quote": "Paris is the capital of France"}, - ], - }, - context={ - "text_chunks": { - 1: "Jason is a pirate", - 2: "Paris is not the capital of France", - 3: "Irrelevant data", - } - }, - ) -except ValidationError as e: - print(e) -``` - -### Error Message Example: - -``` -1 validation error for AnswerWithCitaton -answer.0 - Value error, Citation not found in context [type=value_error, input_value={'body': 'Paris', 'substr... the capital of France'}, input_type=dict] - For further information visit [https://errors.pydantic.dev/2.4/v/value_error](https://errors.pydantic.dev/2.4/v/value_error) -``` - -## Example 3: Aligning Citations and Answers - -In this example, we ensure that the provided answers are aligned with the given citations and context. The LLM is used to verify the alignment. - -We use the same `Statements` model as above, but we add a new model for the answer that also verifies the alignment of citations. - -### Code Example: - -```python -class AnswerWithCitaton(BaseModel): - question: str - answer: List[Statements] - - @model_validator(mode="after") - def validate_answer(self, info: ValidationInfo): - context = info.context.get("text_chunks", None) - - resp: Validation = client.create( - response_model=Validation, - messages=[ - { - "role": "user", - "content": f"Does the following answers match the question and the context?\n\nQuestion: {self.question}\n\nAnswer: {self.answer}\n\nContext: {context}", - } - ], - model="gpt-3.5-turbo", - ) - - if resp.is_valid: - return self - - raise ValueError(resp.error_messages) -``` - -When we have a mismatch between the answer and the citation, the LLM returns an error message. - -```python -try: - AnswerWithCitaton.model_validate( - { - "question": "What is the capital of France?", - "answer": [ - {"body": "Texas", "substring_quote": "Paris is the capital of France"}, - ], - }, - context={ - "text_chunks": { - 1: "Jason is a pirate", - 2: "Paris is the capital of France", - 3: "Irrelevant data", - } - }, - ) -except ValidationError as e: - print(e) -``` - -### Error Message Example: - -``` -1 validation error for AnswerWithCitaton - Value error, The answer does not match the question and context [type=value_error, input_value={'question': 'What is the...he capital of France'}]}, input_type=dict] - For further information visit [https://errors.pydantic.dev/2.4/v/value_error](https://errors.pydantic.dev/2.4/v/value_error) -``` - -## Related Documentation -- [Validation Guide](../../concepts/validation.md) - Validate citations - -## See Also -- [RAG Techniques](rag-and-beyond.md) - Use citations in RAG -- [PDF Citations](generating-pdf-citations.md) - Extract from PDFs -- [Validation Basics](validation-part1.md) - Ensure citation quality - -## Conclusion - -These examples demonstrate the potential of using Pydantic and OpenAI to enhance data accuracy through citation verification. While the LLM-based approach may not be efficient for runtime operations, it has exciting implications for generating a dataset of accurate responses. By leveraging this method during data generation, we can fine-tune a model that excels in citation accuracy. Similar to our last post on [finetuning a better summarizer](https://jxnl.github.io/instructor/blog/2023/11/05/chain-of-density/). - -If you like the content check out our [GitHub](https://github.com/jxnl/instructor) as give us a star and checkout the library. \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/consistent-stories.md b/참고/instructor-main/docs/blog/posts/consistent-stories.md deleted file mode 100644 index bcd3a07..0000000 --- a/참고/instructor-main/docs/blog/posts/consistent-stories.md +++ /dev/null @@ -1,275 +0,0 @@ ---- -authors: - - ivanleomk -categories: - - OpenAI -comments: true -date: 2024-12-10 -description: Generating complex DAGS with gpt-4o -draft: false -tags: - - OpenAI - - DAGs ---- - -# Consistent Stories with GPT-4o - -Language Models struggle to generate consistent graphs that have a large number of nodes. Often times, this is because the graph itself is too large for the model to handle. This causes the model to generate inconsistent graphs that have invalid and disconnected nodes among other issues. - -In this article, we'll look at how we can get around this limitation by using a two-phase approach to generate complex DAGs with gpt-4o by looking at a simple example of generating a Choose Your Own Adventure story. - - - -## Why do DAGs matter? - -DAGs are directed acyclic graphs. A graph is considered a DAG when every connection between nodes is directed ( it goes in a single direction ) and there are no cycles ( it doesn't loop back to a previous node ). - -```mermaid -graph TD - A --> B - A --> C - B --> D - C --> D -``` - -This isn't too far away from a Choose Your Own Adventure story where users have a fixed set of choices at each step and can only move forward in the story. We can see this in action below: - -```mermaid -graph TD - A[Story Root] --> B[Choice 1] - A --> C[Choice 2] - A --> D[Choice 3] - B --> E[Choice 1.1] - B --> F[Choice 1.2] - C --> G[Choice 2.1] - C --> H[Choice 2.2] - D --> I[Choice 3.1] - D --> J[Choice 3.2] -``` - -## The Challenge: Scaling Story Generation - -When we try to use a language model to generate a story in a single run, this hits several limitations quickly because just with 4 choices at each step, we're already at 20 nodes by the second level. If users can only make 2 choices before our story ends, that doesn't result in a very interesting story to play with. - -In other words, we'll overflow the context window of the model quickly. To get around this, we can use a two-phase approach to generate the story where we generate an initial story setting and then generate the choices/other options in parallel. - -## Parallel Story Generation - -### Generating an Outline - -First, we generate an outline of the story using gpt-4o. This is important because it gives us a starting setting, visual style and image description ( for the banner image ). We can then use this down the line to ensure the images we generate are consistent as much as possible. - -```python -from pydantic import BaseModel -from typing import List - - -class GeneratedStory(BaseModel): - setting: str - plot_summary: str - choices: List[str] - visual_style: str - image_description: str - - -async def generate_story( - client: instructor.AsyncInstructor, story_input: RestateStoryInput -): - resp = await client.create( - messages=[ - { - "role": "user", - "content": """ - Generate a story with: - - Setting: {{ story_input.setting}} - - Title: {{ story_input.title }} - - Rules: - - Generate 2-4 initial choices that represent actions - - Choices must move story forward - - Include brief setting description - - Generate a visual description for the story - - Required Elements: - 1. Plot Summary: A vivid description of the setting and plot - 2. Initial Choices: 2-4 distinct actions the user can take - 3. Visual Style: Description of art style, color palette - 4. Image Description: One-sentence scene description - """, - } - ], - model="gpt-4o", - response_model=GeneratedStory, - context={"story_input": story_input}, - ) - return resp -``` - -This outputs a story with a setting, plot summary, choices, visual style and image description. - -```bash -# Example generated output -{ - "setting": "A neon-lit cyberpunk metropolis in 2150", - "plot_summary": "In the sprawling city of Neo-Tokyo...", - "choices": [ - "Investigate the mysterious signal in the abandoned district", - "Meet your contact at the underground hacker hub", - "Follow the corporate executive who seems suspicious" - ], - "visual_style": "Vibrant neon colors, detailed cyberpunk architecture", - "image_description": "A towering cyberpunk cityscape at night with neon signs" -} -``` - -### Parallel Choice Expansion - -One of the biggest challenges in generating deep story trees is maintaining consistency as the story branches grow. - -Here's how we solve this with parallel generation and state tracking: - -```mermaid -graph TD - %% Main nodes - A[Find Door] --> B[Open Door] - A --> C[Walk Away] - - B --> D[Read Book] - B --> E[Leave Room] - - C --> F[Go Home] - C --> G[Wait Outside] - - %% Styling for visual hierarchy - classDef start fill:#ff9999,stroke:#333,stroke-width:2px - classDef decision fill:#99ccff,stroke:#333,stroke-width:2px - classDef outcome fill:#99ffff,stroke:#333,stroke-width:1px - - %% Apply styles - class A start - class B,C decision - class D,E,F,G outcome - - %% Add tooltips for context - click B "Door context" "Open Door Context" - click C "Away context" "Walk Away Context" - click D "Door and Book context" "Read Book Context" -``` - -The key insight is that each path through the story tree has its own unique state. We do so by having a simple accumulator that allows us to keep track of the previous choices and the story context. - -It's also important to note here that the model also has the full flexibility to end the story at any point in time. - -Here's how we implement this: - -```python -async def rewrite_choice( - client: instructor.AsyncInstructor, - choice: str, - story: GeneratedStory, - prev_choices: list[dict], # Accumulator for path state - max_depth: int, - sem: asyncio.Semaphore, -) -> FinalStoryChoice: - # Each choice knows its entire path history - async with sem: - rewritten_choice = await client.create( - model="gpt-4o", - response_model=RewrittenChoice, - messages=[ - { - "role": "user", - "content": """ - Given this choice: {{ choice }} - - Story context: - Setting: {{ story.setting }} - Plot: {{ story.plot_summary }} - - Previous choices made in this path: - {% for prev in prev_choices %} - - {{ prev.choice_description }} - Result: {{ prev.choice_consequences }} - {% endfor %} - - Generate the next story beat and 2-4 new choices. - The story should end in {{ max_depth - len(prev_choices) }} more turns. - """, - } - ], - context={ - "choice": choice, - "story": story, - "prev_choices": prev_choices, - }, - ) - - # For terminal nodes (at max depth) - if len(prev_choices) == max_depth - 1: - return FinalStoryChoice( - choice_description=rewritten_choice.choice_description, - choice_consequences=rewritten_choice.choice_consequences, - choices=[], # Terminal node - ) - - # Recursively expand child choices - child_choices = await asyncio.gather( - *[ - rewrite_choice( - client=client, - choice=new_choice, - story=story, - prev_choices=prev_choices - + [ - { - "choice_description": rewritten_choice.choice_description, - "choice_consequences": rewritten_choice.choice_consequences, - } - ], - max_depth=max_depth, - sem=sem, - ) - for new_choice in rewritten_choice.choices - ] - ) - - return FinalStoryChoice( - choice_description=rewritten_choice.choice_description, - choice_consequences=rewritten_choice.choice_consequences, - choices=child_choices, - ) -``` - -This approach gives us several key benefits: - -1. **Path-Specific Context**: Each node maintains the complete history of choices that led to it, ensuring consistency within each branch -2. **Parallel Generation**: Different branches can be generated simultaneously since they each maintain their own state -3. **Controlled Growth**: The `max_depth` parameter prevents exponential expansion -4. **Rate Limiting**: The semaphore controls concurrent API calls while allowing maximum parallelization - -The semaphore isn't just for rate limiting - it ensures we process choices at a manageable pace while maintaining state consistency. - -Each path through the story tree becomes a self-contained narrative with access to its complete history, allowing us to generate coherent stories at a much faster speed and verbosity than a single call would be able to generate. - -Additionally, we can generate stories that are much broader and deeper than a single call would be able to generate. - -## Beyond Story Generation - -The success of this approach comes down to three key principles: - -1. **State Isolation**: Each node maintains only the context it needs, preventing context window overflow -2. **Parallel Processing**: Generation can happen simultaneously across branches, dramatically reducing total generation time -3. **Structured Validation**: Using Pydantic models ensures each generated component meets your requirements - -For example, generating a 20-node story tree sequentially might take 60 seconds (3s per node), but with parallel generation and 10 concurrent requests, it could complete in just 45-50 seconds. - -This pattern is particularly valuable when: - -- Your generation tasks naturally form a tree or graph structure -- Individual nodes need some but not all context from their ancestors -- You need to generate content that exceeds a single context window -- Speed of generation is important - -By combining structured outputs with parallel generation, you can reliably generate complex, interconnected content at scale while maintaining consistency and control. - -`instructor` makes it easy to generate complex Data Structures with language models - whether they're open source models with ollama or proprietary models with providers such as OpenAI. Give us a try today! diff --git a/참고/instructor-main/docs/blog/posts/course.md b/참고/instructor-main/docs/blog/posts/course.md deleted file mode 100644 index 98c78dd..0000000 --- a/참고/instructor-main/docs/blog/posts/course.md +++ /dev/null @@ -1,26 +0,0 @@ ---- -authors: -- jxnl -categories: -- OpenAI -comments: true -date: 2024-02-14 -description: Discover a free one-hour course on Weights and Biases covering essential - techniques for language models. -draft: false -slug: weights-and-biases-course -tags: -- Weights and Biases -- AI course -- machine learning -- language models -- free resources ---- - -# Free course on Weights and Biases - -I just released a free course on wits and biases. It goes over the material from [tutorial](../../tutorials/1-introduction.ipynb). Check it out at [wandb.courses](https://www.wandb.courses/courses/steering-language-models) its free and open to everyone and just under an hour long! - -[![](img/course.png)](https://www.wandb.courses/courses/steering-language-models) - -> Click the image to access the course \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/cursor-rules.md b/참고/instructor-main/docs/blog/posts/cursor-rules.md deleted file mode 100644 index 4ca217e..0000000 --- a/참고/instructor-main/docs/blog/posts/cursor-rules.md +++ /dev/null @@ -1,115 +0,0 @@ ---- -authors: - - jxnl -categories: - - Contributing -comments: true -date: 2025-03-18 -description: - Learn how Instructor's Cursor rules improve Git workflows for contributors, making AI-assisted coding more organized. -draft: false -slug: cursor-rules-for-better-git-practices -tags: - - Git - - Cursor - - Contributing - - Best Practices ---- - -# Instructor Adopting Cursor Rules - -AI-assisted coding is changing how we use version control. Many developers now use what I call "vibe coding" - coding with AI help. This creates new challenges with Git. Today I'll share how we're using Cursor rules in Instructor to solve these problems. - - - -## The Git Problem When Coding with AI - -In my blog post [Version Control for the Vibe Coder (Part 1)](https://jxnl.co/writing/2025/03/18/version-control-for-the-vibe-coder-part-1/), I wrote about the problem: - -> "Imagine this: you open Cursor, ask it to build a feature in YOLO-mode, and let it rip. You feel great as you watch code materialize... until you realize you haven't made a single commit, your branch is a mess, and you have no idea how to organize these changes for review." - -This happens often. When using AI tools like Cursor, we focus on creating code quickly but forget about version control. This leads to big, messy commits that are hard to review. - -## How Cursor Rules Help - -We've added Cursor rules to Instructor. These rules help standardize Git workflows inside Cursor. The rules are simple markdown files in the `.cursor/rules` directory that guide Cursor when working with your code. - -As I wrote in [Version Control for the Vibe Coder (Part 2)](https://jxnl.co/writing/2025/03/18/version-control-for-the-vibe-coder-part-2/): - -> "Add rules to `.cursor/rules` to instruct Cursor clearly and repeatedly... The real key to success with Git is much simpler: Make Small, Frequent Commits... Let Cursor Handle the Rest." - -This balances fast AI coding with good teamwork practices. - -## How Our Cursor Rules Help Contributors - -If you want to contribute to Instructor, our Cursor rules will make it easier. Here's how: - -### 1. Better Branching and Commits - -The rules help Cursor suggest good Git practices. When building a new feature, Cursor will help you: - -- Create well-named branches -- Make small commits with clear messages -- Format PR descriptions correctly - -### 2. Simpler PR Process - -Our rules define how to create and manage pull requests: - -- Format PR descriptions -- Add the right reviewers -- Use stacked PRs for big features (as I explain in my Part 2 blog post) - -### 3. Keeping Docs Updated - -The rules remind you to update docs when code changes, which keeps our project docs accurate. - -## Getting Started - -If you're new to Instructor or Cursor, here's how to use these rules: - -1. **Install Cursor**: Download it from [cursor.sh](https://cursor.sh/) -2. **Clone Instructor**: `git clone https://github.com/instructor-ai/instructor.git` -3. **Open in Cursor**: The `.cursor/rules` will load automatically -4. **Make changes**: Let Cursor guide your Git workflow -5. **Create a PR**: Follow Cursor's suggestions - -You don't need to remember all the Git commands. The rules will help Cursor suggest the right steps. - -## Stacked PRs for Bigger Features - -One key practice in our rules is stacked PRs. As I explain: - -> "Stacked pull requests are a powerful workflow for building complex features incrementally. Instead of one massive PR, you create a series of smaller, dependent PRs that build upon each other." - -This helps Instructor because it allows: - -- Focused code reviews -- Easier merging of changes -- Better organization of big features -- Clear documentation of decisions - -The rules show you how to make and manage stacked PRs without confusion. - -## Keeping the Human Touch - -A big benefit of Cursor rules is keeping people central to the process. While AI helps write code, the rules ensure: - -- Code changes stay clear and reviewable -- Docs stay current -- Commit history tells a clear story -- Contributors get credit for their work - -## Try It Out - -I invite you to make a PR to Instructor with small changes. Using AI-assisted coding with Git through Cursor rules makes contributing easier and more fun. - -Start small - fix a typo or add an example to the cookbook. Open the repo in Cursor and let the rules guide you through making a clean PR. This lets you focus on writing good code instead of figuring out Git commands. - -Remember: "The most important Git skill is making regular, small commits. Everything else - bisecting, stacked PRs, complex rebases - these are just tools that Cursor can handle for you." - -With Cursor rules, you get fast AI coding plus good team practices. - -If you want to add Cursor rules to your own open source projects, I can help! Reach out to me on Twitter at [@jxnlco](https://twitter.com/jxnlco) and I'll share what we've learned. - -Happy coding! \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/distilation-part1.md b/참고/instructor-main/docs/blog/posts/distilation-part1.md deleted file mode 100644 index 6dedb5f..0000000 --- a/참고/instructor-main/docs/blog/posts/distilation-part1.md +++ /dev/null @@ -1,175 +0,0 @@ ---- -authors: -- jxnl -categories: -- LLM Techniques -comments: true -date: 2023-10-17 -description: Explore Instructor for fine-tuning language models with Python, simplifying - function calls, and enhancing performance. -draft: false -tags: -- Instructor -- Fine-tuning -- Python -- Language Models -- Distillation ---- - -# Enhancing Python Functions with Instructor: A Guide to Fine-Tuning and Distillation - -## Introduction - -Get ready to dive deep into the world of fine-tuning task specific language models with Python functions. We'll explore how the `instructor.instructions` streamlines this process, making the task you want to distil more efficient and powerful while preserving its original functionality and backwards compatibility. - -If you want to see the full example checkout [examples/distillation](https://github.com/jxnl/instructor/tree/main/examples/distilations) - - - -## Why use Instructor? - -Imagine you're developing a backend service that uses a mix old and new school ML practises, it may involve pipelines with multiple function calls, validations, and data processing. Sounds cumbersome, right? That's where `Instructor` comes in. It simplifies complex procedures, making them more efficient and easier to manage by adding a decorator to your function that will automatically generate a dataset for fine-tuning and help you swap out the function implementation. - -## Quick Start: How to Use Instructor's Distillation Feature - -Before we dig into the nitty-gritty, let's look at how easy it is to use Instructor's distillation feature to use function calling finetuning to export the data to a JSONL file. - -```python -import logging -import random -from pydantic import BaseModel -from instructor import Instructions # pip install instructor - -# Logging setup -logging.basicConfig(level=logging.INFO) - -instructions = Instructions( - name="three_digit_multiply", - finetune_format="messages", - # log handler is used to save the data to a file - # you can imagine saving it to a database or other storage - # based on your needs! - log_handlers=[logging.FileHandler("math_finetunes.jsonl")], -) - - -class Multiply(BaseModel): - a: int - b: int - result: int - - -# Define a function with distillation -# The decorator will automatically generate a dataset for fine-tuning -# They must return a pydantic model to leverage function calling -@instructions.distil -def fn(a: int, b: int) -> Multiply: - resp = a * b - return Multiply(a=a, b=b, result=resp) - - -# Generate some data -for _ in range(10): - a = random.randint(100, 999) - b = random.randint(100, 999) - print(fn(a, b)) - #> a=268 b=548 result=146864 - #> a=774 b=447 result=345978 - #> a=154 b=902 result=138908 - #> a=304 b=808 result=245632 - #> a=980 b=104 result=101920 - #> a=725 b=455 result=329875 - #> a=206 b=386 result=79516 - #> a=488 b=920 result=448960 - #> a=989 b=889 result=879221 - #> a=815 b=343 result=279545 -``` - -## The Intricacies of Fine-tuning Language Models - -Fine-tuning isn't just about writing a function like `def f(a, b): return a * b`. It requires detailed data preparation and logging. However, Instructor provides a built-in logging feature and structured outputs to simplify this. - -## Why Instructor and Distillation are Game Changers - -The library offers two main benefits: - -1. **Efficiency**: Streamlines functions, distilling requirements into model weights and a few lines of code. -2. **Integration**: Eases combining classical machine learning and language models by providing a simple interface that wraps existing functions. - -## Role of Instructor in Simplifying Fine-Tuning - -The `from instructor import Instructions` feature is a time saver. It auto-generates a fine-tuning dataset, making it a breeze to imitate a function's behavior. - -## Logging Output and Running a Finetune - -Here's how the logging output would look: - -```python -{ - "messages": [ - {"role": "system", "content": 'Predict the results of this function: ...'}, - {"role": "user", "content": 'Return fn(133, b=539)'}, - { - "role": "assistant", - "function_call": { - "name": "Multiply", - "arguments": '{"a":133,"b":539,"result":89509}', - }, - }, - ], - "functions": [ - {"name": "Multiply", "description": "Correctly extracted `Multiply`..."} - ], -} -``` - -Run a finetune like this: - -!!! note annotate "Don't forget to set your OpenAI Key as an environment variable" - - All of the `instructor jobs` commands assume you've set an environment variable of `OPENAI_API_KEY` in your shell. You can set this by running the command `export OPENAI_API_KEY=` in your shell - -```bash -instructor jobs create-from-file math_finetunes.jsonl -``` - -## Next Steps and Future Plans - -Here's a sneak peek of what I'm planning: - -```python -from instructor import Instructions, patch - -patch() # (1)! - - -class Multiply(BaseModel): - a: int - b: int - result: int - - -instructions = Instructions( - name="three_digit_multiply", -) - - -@instructions.distil(model='gpt-3.5-turbo:finetuned-123', mode="dispatch") # (2)! -def fn(a: int, b: int) -> Multiply: - resp = a + b - return Multiply(a=a, b=b, result=resp) -``` - -1. Don't forget to run the `patch()` command that we provide with the `Instructor` package. This helps - automatically serialize the content back into the `Pydantic`` model that we're looking for. - -2. Don't forget to replace this with your new model id. OpenAI identifies fine tuned models with an id - of `ft:gpt-3.5-turbo-0613:personal::` under their **Fine-tuning** tab on their dashboard - -With this, you can swap the function implementation, making it backward compatible. You can even imagine using the different models for different tasks or validating and running evals by using the original function and comparing it to the distillation. - -## Conclusion - -We've seen how `Instructor` can make your life easier, from fine-tuning to distillation. Now if you're thinking wow, I'd love a backend service to do this for continuously, you're in luck! Please check out the survey at [useinstructor.com](https://useinstructor.com) and let us know who you are. - -If you enjoy the content or want to try out `instructor` please check out the [github](https://github.com/jxnl/instructor) and give us a star! \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/extract-model-looks.md b/참고/instructor-main/docs/blog/posts/extract-model-looks.md deleted file mode 100644 index 2194ebd..0000000 --- a/참고/instructor-main/docs/blog/posts/extract-model-looks.md +++ /dev/null @@ -1,275 +0,0 @@ ---- -authors: - - ivanleomk -categories: - - OpenAI -comments: true -date: 2024-12-10 -description: Generating complex DAGS with gpt-4o -draft: false -tags: - - OpenAI - - Multimodal ---- - -# Consistent Stories with GPT-4o - -Language Models struggle to generate consistent graphs that have a large number of nodes. Often times, this is because the graph itself is too large for the model to handle. This causes the model to generate inconsistent graphs that have invalid and disconnected nodes among other issues. - -In this article, we'll look at how we can get around this limitation by using a two-phase approach to generate complex DAGs with gpt-4o by looking at a simple example of generating a Choose Your Own Adventure story. - - - -## Why do DAGs matter? - -DAGs are directed acyclic graphs. A graph is considered a DAG when every connection between nodes is directed ( it goes in a single direction ) and there are no cycles ( it doesn't loop back to a previous node ). - -```mermaid -graph TD - A --> B - A --> C - B --> D - C --> D -``` - -This isn't too far away from a Choose Your Own Adventure story where users have a fixed set of choices at each step and can only move forward in the story. We can see this in action below: - -```mermaid -graph TD - A[Story Root] --> B[Choice 1] - A --> C[Choice 2] - A --> D[Choice 3] - B --> E[Choice 1.1] - B --> F[Choice 1.2] - C --> G[Choice 2.1] - C --> H[Choice 2.2] - D --> I[Choice 3.1] - D --> J[Choice 3.2] -``` - -## The Challenge: Scaling Story Generation - -When we try to use a language model to generate a story in a single run, this hits several limitations quickly because just with 4 choices at each step, we're already at 20 nodes by the second level. If users can only make 2 choices before our story ends, that doesn't result in a very interesting story to play with. - -In other words, we'll overflow the context window of the model quickly. To get around this, we can use a two-phase approach to generate the story where we generate an initial story setting and then generate the choices/other options in parallel. - -## Parallel Story Generation - -### Generating an Outline - -First, we generate an outline of the story using gpt-4o. This is important because it gives us a starting setting, visual style and image description ( for the banner image ). We can then use this down the line to ensure the images we generate are consistent as much as possible. - -```python -from pydantic import BaseModel -from typing import List - - -class GeneratedStory(BaseModel): - setting: str - plot_summary: str - choices: List[str] - visual_style: str - image_description: str - - -async def generate_story( - client: instructor.AsyncInstructor, story_input: RestateStoryInput -): - resp = await client.create( - messages=[ - { - "role": "user", - "content": """ - Generate a story with: - - Setting: {{ story_input.setting}} - - Title: {{ story_input.title }} - - Rules: - - Generate 2-4 initial choices that represent actions - - Choices must move story forward - - Include brief setting description - - Generate a visual description for the story - - Required Elements: - 1. Plot Summary: A vivid description of the setting and plot - 2. Initial Choices: 2-4 distinct actions the user can take - 3. Visual Style: Description of art style, color palette - 4. Image Description: One-sentence scene description - """, - } - ], - model="gpt-4o", - response_model=GeneratedStory, - context={"story_input": story_input}, - ) - return resp -``` - -This outputs a story with a setting, plot summary, choices, visual style and image description. - -```bash -# Example generated output -{ - "setting": "A neon-lit cyberpunk metropolis in 2150", - "plot_summary": "In the sprawling city of Neo-Tokyo...", - "choices": [ - "Investigate the mysterious signal in the abandoned district", - "Meet your contact at the underground hacker hub", - "Follow the corporate executive who seems suspicious" - ], - "visual_style": "Vibrant neon colors, detailed cyberpunk architecture", - "image_description": "A towering cyberpunk cityscape at night with neon signs" -} -``` - -### Parallel Choice Expansion - -One of the biggest challenges in generating deep story trees is maintaining consistency as the story branches grow. - -Here's how we solve this with parallel generation and state tracking: - -```mermaid -graph TD - %% Main nodes - A[Find Door] --> B[Open Door] - A --> C[Walk Away] - - B --> D[Read Book] - B --> E[Leave Room] - - C --> F[Go Home] - C --> G[Wait Outside] - - %% Styling for visual hierarchy - classDef start fill:#ff9999,stroke:#333,stroke-width:2px - classDef decision fill:#99ccff,stroke:#333,stroke-width:2px - classDef outcome fill:#99ffff,stroke:#333,stroke-width:1px - - %% Apply styles - class A start - class B,C decision - class D,E,F,G outcome - - %% Add tooltips for context - click B "Door context" "Open Door Context" - click C "Away context" "Walk Away Context" - click D "Door and Book context" "Read Book Context" -``` - -The key insight is that each path through the story tree has its own unique state. We do so by having a simple accumulator that allows us to keep track of the previous choices and the story context. - -It's also important to note here that the model also has the full flexibility to end the story at any point in time. - -Here's how we implement this: - -```python -async def rewrite_choice( - client: instructor.AsyncInstructor, - choice: str, - story: GeneratedStory, - prev_choices: list[dict], # Accumulator for path state - max_depth: int, - sem: asyncio.Semaphore, -) -> FinalStoryChoice: - # Each choice knows its entire path history - async with sem: - rewritten_choice = await client.create( - model="gpt-4o", - response_model=RewrittenChoice, - messages=[ - { - "role": "user", - "content": """ - Given this choice: {{ choice }} - - Story context: - Setting: {{ story.setting }} - Plot: {{ story.plot_summary }} - - Previous choices made in this path: - {% for prev in prev_choices %} - - {{ prev.choice_description }} - Result: {{ prev.choice_consequences }} - {% endfor %} - - Generate the next story beat and 2-4 new choices. - The story should end in {{ max_depth - len(prev_choices) }} more turns. - """, - } - ], - context={ - "choice": choice, - "story": story, - "prev_choices": prev_choices, - }, - ) - - # For terminal nodes (at max depth) - if len(prev_choices) == max_depth - 1: - return FinalStoryChoice( - choice_description=rewritten_choice.choice_description, - choice_consequences=rewritten_choice.choice_consequences, - choices=[], # Terminal node - ) - - # Recursively expand child choices - child_choices = await asyncio.gather( - *[ - rewrite_choice( - client=client, - choice=new_choice, - story=story, - prev_choices=prev_choices - + [ - { - "choice_description": rewritten_choice.choice_description, - "choice_consequences": rewritten_choice.choice_consequences, - } - ], - max_depth=max_depth, - sem=sem, - ) - for new_choice in rewritten_choice.choices - ] - ) - - return FinalStoryChoice( - choice_description=rewritten_choice.choice_description, - choice_consequences=rewritten_choice.choice_consequences, - choices=child_choices, - ) -``` - -This approach gives us several key benefits: - -1. **Path-Specific Context**: Each node maintains the complete history of choices that led to it, ensuring consistency within each branch -2. **Parallel Generation**: Different branches can be generated simultaneously since they each maintain their own state -3. **Controlled Growth**: The `max_depth` parameter prevents exponential expansion -4. **Rate Limiting**: The semaphore controls concurrent API calls while allowing maximum parallelization - -The semaphore isn't just for rate limiting - it ensures we process choices at a manageable pace while maintaining state consistency. - -Each path through the story tree becomes a self-contained narrative with access to its complete history, allowing us to generate coherent stories at a much faster speed and verbosity than a single call would be able to generate. - -Additionally, we can generate stories that are much broader and deeper than a single call would be able to generate. - -## Beyond Story Generation - -The success of this approach comes down to three key principles: - -1. **State Isolation**: Each node maintains only the context it needs, preventing context window overflow -2. **Parallel Processing**: Generation can happen simultaneously across branches, dramatically reducing total generation time -3. **Structured Validation**: Using Pydantic models ensures each generated component meets your requirements - -For example, generating a 20-node story tree sequentially might take 60 seconds (3s per node), but with parallel generation and 10 concurrent requests, it could complete in just 45-50 seconds. - -This pattern is particularly valuable when: - -- Your generation tasks naturally form a tree or graph structure -- Individual nodes need some but not all context from their ancestors -- You need to generate content that exceeds a single context window -- Speed of generation is important - -By combining structured outputs with parallel generation, you can reliably generate complex, interconnected content at scale while maintaining consistency and control. - -`instructor` makes it easy to generate complex Data Structures with language models - whether they're open source models with ollama or proprietary models with providers such as OpenAI. Give us a try today! diff --git a/참고/instructor-main/docs/blog/posts/extracting-model-metadata.md b/참고/instructor-main/docs/blog/posts/extracting-model-metadata.md deleted file mode 100644 index 829f739..0000000 --- a/참고/instructor-main/docs/blog/posts/extracting-model-metadata.md +++ /dev/null @@ -1,238 +0,0 @@ ---- -title: "Extracting Metadata from Images using Structured Extraction" -date: 2024-12-11 -description: Structured Extraction makes working with images easy, in this post we'll see how to use it to extract metadata from images -categories: - - OpenAI - - Multimodal -authors: - - ivanleomk ---- - -Multimodal Language Models like gpt-4o excel at processing multimodal, enabling us to extract rich, structured metadata from images. - -This is particularly valuable in areas like fashion where we can use these capabilities to understand user style preferences from images and even videos. In this post, we'll see how to use instructor to map images to a given product taxonomy so we can recommend similar products for users. - - - -## Why Image Metadata is useful - -Most online e-commerce stores have a taxonomy of products that they sell. This is a way of categorizing products so that users can easily find what they're looking for. - -A small example of a taxonomy is shown below. You can think of this as a way of mapping a product to a set of attributes, with some common attributes that are shared across all products. - -```yaml -tops: - t-shirts: - - crew_neck - - v_neck - - graphic_tees - sweaters: - - crewneck - - cardigan - - pullover - jackets: - - bomber_jackets - - denim_jackets - - leather_jackets - -bottoms: - pants: - - chinos - - dress_pants - - cargo_pants - shorts: - - athletic_shorts - - cargo_shorts - -colors: - - black - - navy - - white - - beige - - brown -``` - -By using this taxonomy, we can ensure that our model is able to extract metadata that is consistent with the products we sell. In this example, we'll analyze style photos from a fitness influencer to understand their fashion preferences and possibily see what products we can recommend from our own catalog to him. - -We're using some photos from a fitness influencer called [Jpgeez](https://www.instagram.com/jpgeez/) which you can see below. - -
-![](./img/style_1.png){: style="height:200px"} -![](./img/style_2.png){: style="height:200px"} -![](./img/style_3.png){: style="height:200px"} -![](./img/style_4.png){: style="height:200px"} -![](./img/style_5.png){: style="height:200px"} -![](./img/style_6.png){: style="height:200px"} -
- -While we're mapping these visual elements over to a taxonomy, this is really applicable to any other use case where you want to extract metadata from images. - -## Extracting metadata from images - -### Instructor's `Image` class - -With instructor, working with `multimodal` data is easy. We can use the `Image` class to load images from a URL or local file. We can see this below in action. - -```python -import instructor - -# Load images using instructor.Image.from_path -images = [] -for image_file in image_files: - image_path = os.path.join("./images", image_file) - image = instructor.Image.from_path(image_path) - images.append(image) -``` - -We provide a variety of different methods for loading images, including from a URL, local file, and even from a base64 encoded string which you [can read about here](../../concepts/multimodal.md) - -### Defining a response model - -Since our taxonomy is defined as a yaml file, we can't use literals to define the response model. Instead, we can read in the configuration from a yaml file and then use that in a `model_validator` step to make sure that the metadata we extract is consistent with the taxonomy. - -First, we read in the taxonomy from a yaml file and create a set of categories, subcategories, and product types. - -```python -import yaml - -with open("taxonomy.yml") as file: - taxonomy = yaml.safe_load(file) - -colors = taxonomy["colors"] -categories = set(taxonomy.keys()) -categories.remove("colors") - -subcategories = set() -product_types = set() -for category in categories: - for subcategory in taxonomy[category].keys(): - subcategories.add(subcategory) - for product_type in taxonomy[category][subcategory]: - product_types.add(product_type) -``` - -Then we can use these in our `response_model` to make sure that the metadata we extract is consistent with the taxonomy. - -```python -class PersonalStyle(BaseModel): - """ - Ideally you map this to a specific taxonomy - """ - - categories: list[str] - subcategories: list[str] - product_types: list[str] - colors: list[str] - - @model_validator(mode="after") - def validate_options(self, info: ValidationInfo): - context = info.context - colors = context["colors"] - categories = context["categories"] - subcategories = context["subcategories"] - product_types = context["product_types"] - - # Validate colors - for color in self.colors: - if color not in colors: - raise ValueError( - f"Color {color} is not in the taxonomy. Valid colors are {colors}" - ) - for category in self.categories: - if category not in categories: - raise ValueError( - f"Category {category} is not in the taxonomy. Valid categories are {categories}" - ) - - for subcategory in self.subcategories: - if subcategory not in subcategories: - raise ValueError( - f"Subcategory {subcategory} is not in the taxonomy. Valid subcategories are {subcategories}" - ) - - for product_type in self.product_types: - if product_type not in product_types: - raise ValueError( - f"Product type {product_type} is not in the taxonomy. Valid product types are {product_types}" - ) - - return self -``` - -### Making the API call - -Lastly, we can combine these all into a single api call to `gpt-4o` where we pass in all of the images and the response model into the `response_model` parameter. - -With our inbuilt support for `jinja` formatting using the `context` keyword that exposes data we can also re-use in our validation, this becomes an incredibly easy step to execute. - -```python -import instructor - -client = instructor.from_provider("openai/gpt-5-nano") - -resp = client.create( - model="gpt-4o", - messages=[ - { - "role": "system", - "content": """ -You are a helpful assistant. You are given a list of images and you need to map the person style of the person in the image to a given taxonomy. - -Here is the taxonomy that you should use - -Colors: -{% for color in colors %} -* {{ color }} -{% endfor %} - -Categories: -{% for category in categories %} -* {{ category }} -{% endfor %} - -Subcategories: -{% for subcategory in subcategories %} -* {{ subcategory }} -{% endfor %} - -Product types: -{% for product_type in product_types %} -* {{ product_type }} -{% endfor %} -""", - }, - { - "role": "user", - "content": [ - "Here are the images of the person, describe the personal style of the person in the image from a first-person perspective( Eg. You are ... )", - *images, - ], - }, - ], - response_model=PersonalStyle, - context={ - "colors": colors, - "categories": list(categories), - "subcategories": list(subcategories), - "product_types": list(product_types), - }, -) -``` - -This then returns the following response. - -```python -PersonalStyle( - categories=['tops', 'bottoms'], - subcategories=['sweaters', 'jackets', 'pants'], - product_types=['cardigan', 'crewneck', 'denim_jackets', 'chinos'], - colors=['brown', 'beige', 'black', 'white', 'navy'], -) -``` - -## Looking Ahead - -The ability to extract structured metadata from images opens up exciting possibilities for personalization in e-commerce. The key is maintaining the bridge between unstructured visual inspiration and structured product data through well-defined taxonomies and robust validation. - -`instructor` makes working with multimodal data easy, and we're excited to see what you build with it. Give us a try today with `pip install instructor` and see how easy it is to work with language models using structured extraction. diff --git a/참고/instructor-main/docs/blog/posts/fake-data.md b/참고/instructor-main/docs/blog/posts/fake-data.md deleted file mode 100644 index 70e016b..0000000 --- a/참고/instructor-main/docs/blog/posts/fake-data.md +++ /dev/null @@ -1,193 +0,0 @@ ---- -authors: -- jxnl -categories: -- Pydantic -comments: true -date: 2024-03-08 -description: Learn to generate synthetic data using Pydantic and OpenAI's models with - practical examples and configurations. -draft: false -tags: -- Synthetic Data -- Pydantic -- OpenAI -- Data Generation -- Python ---- - -# Simple Synthetic Data Generation - -What that people have been using instructor for is to generate synthetic data rather than extracting data itself. We can even use the J-Schemo extra fields to give specific examples to control how we generate data. - -Consider the example below. We'll likely generate very simple names. - -```python -from typing import Iterable -from pydantic import BaseModel -import instructor - - -# Define the UserDetail model -class UserDetail(BaseModel): - name: str - age: int - - -# Patch the OpenAI client to enable the response_model functionality -client = instructor.from_provider("openai/gpt-5-nano") - - -def generate_fake_users(count: int) -> Iterable[UserDetail]: - return client.create( - model="gpt-3.5-turbo", - response_model=Iterable[UserDetail], - messages=[ - {"role": "user", "content": f"Generate a {count} synthetic users"}, - ], - ) - - -for user in generate_fake_users(5): - print(user) - #> name='Alice' age=25 - #> name='Bob' age=30 - #> name='Charlie' age=22 - #> name='David' age=28 - #> name='Eve' age=35 -``` - -## Leveraging Simple Examples - -We might want to set examples as part of the prompt by leveraging Pydantics configuration. We can set examples directly in the JSON scheme itself. - -```python -from typing import Iterable -from pydantic import BaseModel, Field -import instructor - - -# Define the UserDetail model -class UserDetail(BaseModel): - name: str = Field(examples=["Timothee Chalamet", "Zendaya"]) - age: int - - -# Patch the OpenAI client to enable the response_model functionality -client = instructor.from_provider("openai/gpt-5-nano") - - -def generate_fake_users(count: int) -> Iterable[UserDetail]: - return client.create( - model="gpt-3.5-turbo", - response_model=Iterable[UserDetail], - messages=[ - {"role": "user", "content": f"Generate a {count} synthetic users"}, - ], - ) - - -for user in generate_fake_users(5): - print(user) - #> name='John Doe' age=25 - #> name='Alice Smith' age=30 - #> name='Bob Johnson' age=28 - #> name='Emily Brown' age=35 - #> name='Michael Williams' age=27 -``` - -By incorporating names of celebrities as examples, we have shifted towards generating synthetic data featuring well-known personalities, moving away from the simplistic, single-word names previously used. - -## Leveraging Complex Example - -To effectively generate synthetic examples with more nuance, lets upgrade to the "gpt-4-turbo-preview" model, use model level examples rather than attribute level examples: - -```Python -import instructor - -from typing import Iterable -from pydantic import BaseModel, ConfigDict - - -# Define the UserDetail model -class UserDetail(BaseModel): - """Old Wizards""" - - name: str - age: int - - model_config = ConfigDict( - json_schema_extra={ - "examples": [ - {"name": "Gandalf the Grey", "age": 1000}, - {"name": "Albus Dumbledore", "age": 150}, - ] - } - ) - - -# Patch the OpenAI client to enable the response_model functionality -client = instructor.from_provider("openai/gpt-5-nano") - - -def generate_fake_users(count: int) -> Iterable[UserDetail]: - return client.create( - model="gpt-4-turbo-preview", - response_model=Iterable[UserDetail], - messages=[ - {"role": "user", "content": f"Generate `{count}` synthetic examples"}, - ], - ) - - -for user in generate_fake_users(5): - print(user) - #> name='Merlin' age=600 - #> name='Radagast the Brown' age=950 - #> name='Rincewind' age=70 - #> name='Harry Potter' age=17 - #> name='Elminster Aumar' age=1200 -``` - -## Leveraging Descriptions - -By adjusting the descriptions within our Pydantic models, we can subtly influence the nature of the synthetic data generated. This method allows for a more nuanced control over the output, ensuring that the generated data aligns more closely with our expectations or requirements. - -For instance, specifying "Fancy French sounding names" as a description for the `name` field in our `UserDetail` model directs the generation process to produce names that fit this particular criterion, resulting in a dataset that is both diverse and tailored to specific linguistic characteristics. - - -```python -import instructor - -from typing import Iterable -from pydantic import BaseModel, Field - - -# Define the UserDetail model -class UserDetail(BaseModel): - name: str = Field(description="Fancy French sounding names") - age: int - - -# Patch the OpenAI client to enable the response_model functionality -client = instructor.from_provider("openai/gpt-5-nano") - - -def generate_fake_users(count: int) -> Iterable[UserDetail]: - return client.create( - model="gpt-3.5-turbo", - response_model=Iterable[UserDetail], - messages=[ - {"role": "user", "content": f"Generate `{count}` synthetic users"}, - ], - ) - - -for user in generate_fake_users(5): - print(user) - #> name='Jean Luc' age=25 - #> name='Marcelle' age=30 - #> name='Antoinette' age=22 - #> name='Gaspard' age=28 - #> name='Eloise' age=35 -``` \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/full-fastapi-visibility.md b/참고/instructor-main/docs/blog/posts/full-fastapi-visibility.md deleted file mode 100644 index 26ca268..0000000 --- a/참고/instructor-main/docs/blog/posts/full-fastapi-visibility.md +++ /dev/null @@ -1,416 +0,0 @@ ---- -authors: -- ivanleomk -- jxnl -categories: -- LLM Observability -comments: true -date: 2024-05-03 -description: Discover how Logfire enhances FastAPI applications with OpenTelemetry - for better visibility and performance tracking. -draft: false -slug: fastapi-open-telemetry-and-instructor -tags: -- FastAPI -- Logfire -- OpenTelemetry -- Pydantic -- AsyncIO ---- - -# Why Logfire is a perfect fit for FastAPI + Instructor - -Logfire is a new tool that provides key insight into your application with Open Telemetry. Instead of using ad-hoc print statements, Logfire helps to profile every part of your application and is integrated directly into Pydantic and FastAPI, two popular libraries amongst Instructor users. - -In short, this is the secret sauce to help you get your application to the finish line and beyond. We'll show you how to easily integrate Logfire into FastAPI, one of the most popular choices amongst users of Instructor using two examples - -1. Data Extraction from a single User Query -2. Using `asyncio` to process multiple users in parallel -3. Streaming multiple objects using an `Iterable` so that they're available on demand - - - -As usual, all of the code that we refer to here is provided in [examples/logfire-fastapi](https://www.github.com/jxnl/instructor/tree/main/examples/logfire-fastapi) for you to use in your projects. - -??? info "Configure Logfire" - - Before starting this tutorial, make sure that you've registered for a [Logfire](https://logfire.pydantic.dev/) account. You'll also need to create a project to track these logs. Lastly, in order to see the request body, you'll also need to configure the default log level to `debug` instead of the default `info` on the dashboard console. - -Make sure to create a virtual environment and install all of the packages inside the `requirements.txt` file at [examples/logfire-fastapi](https://www.github.com/jxnl/instructor/tree/main/examples/logfire-fastapi). - -## Data Extraction - -Let's start by trying to extract some user information given a user query. We can do so with a simple Pydantic model as seen below. - -```python -from pydantic import BaseModel -from fastapi import FastAPI -import instructor - - -class UserData(BaseModel): - query: str - - -class UserDetail(BaseModel): - name: str - age: int - - -app = FastAPI() -client = instructor.from_provider("openai/gpt-5-nano", async_client=True) - - -@app.post("/user", response_model=UserDetail) -async def endpoint_function(data: UserData) -> UserDetail: - user_detail = await client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": f"Extract: `{data.query}`"}, - ], - ) - - return user_detail -``` - -This simple endpoint takes in a user query and extracts out a user from the statement. Let's see how we can add in Logfire into this endpoint with just a few lines of code - -```python hl_lines="5 18-21" -from pydantic import BaseModel -from fastapi import FastAPI -import instructor -import logfire # (1)! - - -class UserData(BaseModel): - query: str - - -class UserDetail(BaseModel): - name: str - age: int - - -app = FastAPI() -openai_client = AsyncOpenAI() # (2)! -logfire.configure(pydantic_plugin=logfire.PydanticPlugin(record="all")) -logfire.instrument_openai(openai_client) -logfire.instrument_fastapi(app) -client = instructor.from_provider("openai/gpt-4o") - - -@app.post("/user", response_model=UserDetail) -async def endpoint_function(data: UserData) -> UserDetail: - user_detail = await client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": f"Extract: `{data.query}`"}, - ], - ) - - return user_detail -``` - -1. Import in the logfire package -2. Setup logging using their native integrations with FastAPI and OpenAI - -With just those few lines of code, we've got ourselves a working integration with Logfire. When we call our endpoint at `/user` with the following payload, everything is immediately logged in the console. - -```bash -curl -X 'POST' \ - 'http://localhost:8000/user' \ - -H 'accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ - "query": "Daniel is a 24 year man living in New York City" -}' -``` - -We can see that Pydantic has nicely logged for us the validation result of our openai call here. Just right above, we also have the result of the OpenAI call. - -![Pydantic Validation](img/logfire-sync-pydantic-validation.png) - -We've also got full visibility into the arguments that were passed into the endpoint when we called it. This is extremely useful for users when they eventually want to reproduce errors in production locally. - -![FastAPI arguments](img/logfire-sync-fastapi-arguments.png) - -## Using Asyncio - -Sometimes, we might need to run multiple jobs in parallel. Let's see how we can take advantage of `asyncio` so that we can speed up our operations. We can do so by adding the following bits of code to our previous file. - -??? info "What is Asyncio?" - - For a deeper guide into how to work with Asycnio, see our previous guide [here](./learn-async.md). - -=== "New Code" - - ```python - import asyncio - - - class MultipleUserData(BaseModel): - queries: list[str] - - - @app.post("/many-users", response_model=list[UserDetail]) - async def extract_many_users(data: MultipleUserData): - async def extract_user(query: str): - user_detail = await client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": f"Extract: `{query}`"}, - ], - ) - logfire.info("/User returning", value=user_detail) - return user_detail - - coros = [extract_user(query) for query in data.queries] - return await asyncio.gather(*coros) - ``` - -=== "Full File" - - ```python - from pydantic import BaseModel - from fastapi import FastAPI - import instructor - import logfire - import asyncio - - - class UserData(BaseModel): - query: str - - - class MultipleUserData(BaseModel): - queries: list[str] - - - class UserDetail(BaseModel): - name: str - age: int - - - app = FastAPI() - openai_client = AsyncOpenAI() - logfire.configure(pydantic_plugin=logfire.PydanticPlugin(record="all")) - logfire.instrument_openai(openai_client) - logfire.instrument_fastapi(app) - client = instructor.from_provider("openai/gpt-4o") - - - @app.post("/user", response_model=UserDetail) - async def endpoint_function(data: UserData) -> UserDetail: - user_detail = await client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": f"Extract: `{data.query}`"}, - ], - ) - logfire.info("/User returning", value=user_detail) - return user_detail - - - @app.post("/many-users", response_model=list[UserDetail]) - async def extract_many_users(data: MultipleUserData): - async def extract_user(query: str): - user_detail = await client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": f"Extract: `{query}`"}, - ], - ) - logfire.info("/User returning", value=user_detail) - return user_detail - - coros = [extract_user(query) for query in data.queries] - return await asyncio.gather(*coros) - ``` - -We can call this endpoint with a simple `curl` call - -```bash -curl -X 'POST' \ - 'http://localhost:8000/many-users' \ - -H 'accept: application/json' \ - -H 'Content-Type: application/json' \ - -d '{ - "queries": [ - "Daniel is a 34 year man in New York City","Sarah is a 20 year old living in Tokyo", "Jeffrey is 55 and lives down in Leeds" - ] -}' -``` - -This is all logged in Logfire as seen below. We have complete visibility into the performance of our entire application and it's pretty clear that a large chunk of the latency is taken up by the OpenAI Call. - -We could also potentially separate the logs into more graunular levels by creating a new span for each instance of `extract_user` created. - -![Logfire Asyncio](img/logfire-asyncio.png) - -## Streaming - -Now let's see how we can take advantage of Instructor's `Iterable` support to stream multiple instances of an extracted object. This is extremely useful for application where speed is crucial and users want to get the results quickly. - -Let's add a new endpoint to our server to see how this might work - -=== "New Code" - - ```python - from collections.abc import Iterable - from fastapi.responses import StreamingResponse - - - class MultipleUserData(BaseModel): - queries: list[str] - - - @app.post("/extract", response_class=StreamingResponse) - async def extract(data: UserData): - suppressed_client = AsyncOpenAI() - logfire.instrument_openai( - suppressed_client, suppress_other_instrumentation=False - ) # (1)! - client = instructor.from_provider("openai/gpt-4o") - users = await client.create( - model="gpt-3.5-turbo", - response_model=Iterable[UserDetail], - stream=True, - messages=[ - {"role": "user", "content": data.query}, - ], - ) - - async def generate(): - with logfire.span("Generating User Response Objects"): - async for user in users: - resp_json = user.model_dump_json() - logfire.info("Returning user object", value=resp_json) - - yield resp_json - - return StreamingResponse(generate(), media_type="text/event-stream") - ``` - - 1. Note that we suppress instrumentation to print out the stream objects. This has to do with the parsing of partials in Instructor. - -=== "Full File" - - ```python - from pydantic import BaseModel - from fastapi import FastAPI - import instructor - import logfire - import asyncio - from collections.abc import Iterable - from fastapi.responses import StreamingResponse - - - class UserData(BaseModel): - query: str - - - class MultipleUserData(BaseModel): - queries: list[str] - - - class UserDetail(BaseModel): - name: str - age: int - - - app = FastAPI() - openai_client = AsyncOpenAI() - logfire.configure(pydantic_plugin=logfire.PydanticPlugin(record="all")) - logfire.instrument_fastapi(app) - logfire.instrument_openai(openai_client) - client = instructor.from_provider("openai/gpt-4o") - - - @app.post("/user", response_model=UserDetail) - async def endpoint_function(data: UserData) -> UserDetail: - user_detail = await client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": f"Extract: `{data.query}`"}, - ], - ) - logfire.info("/User returning", value=user_detail) - return user_detail - - - @app.post("/many-users", response_model=list[UserDetail]) - async def extract_many_users(data: MultipleUserData): - async def extract_user(query: str): - user_detail = await client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": f"Extract: `{query}`"}, - ], - ) - logfire.info("/User returning", value=user_detail) - return user_detail - - coros = [extract_user(query) for query in data.queries] - return await asyncio.gather(*coros) - - - @app.post("/extract", response_class=StreamingResponse) - async def extract(data: UserData): - suppressed_client = AsyncOpenAI() - logfire.instrument_openai(suppressed_client, suppress_other_instrumentation=False) - client = instructor.from_provider("openai/gpt-4o") - users = await client.create( - model="gpt-3.5-turbo", - response_model=Iterable[UserDetail], - stream=True, - messages=[ - {"role": "user", "content": data.query}, - ], - ) - - async def generate(): - with logfire.span("Generating User Response Objects"): - async for user in users: - resp_json = user.model_dump_json() - logfire.info("Returning user object", value=resp_json) - - yield resp_json - - return StreamingResponse(generate(), media_type="text/event-stream") - ``` - -We can call and log out the stream returned using the `requests` library and using the `iter_content` method - -```python -import requests - -response = requests.post( - "http://127.0.0.1:3000/extract", - json={ - "query": "Alice and Bob are best friends. They are currently 32 and 43 respectively. " - }, - stream=True, -) - -for chunk in response.iter_content(chunk_size=1024): - if chunk: - print(str(chunk, encoding="utf-8"), end="\n") -``` - -This gives us the output of - -```bash -{"name":"Alice","age":32} -{"name":"Bob","age":43} -``` - -We can also see the individual stream objects inside the Logfire dashboard as seen below. Note that we've grouped the generated logs inside a span of its own for easy logging. - -![Logfire Stream](img/logfire-stream.png) \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/generating-pdf-citations.md b/참고/instructor-main/docs/blog/posts/generating-pdf-citations.md deleted file mode 100644 index 8233d67..0000000 --- a/참고/instructor-main/docs/blog/posts/generating-pdf-citations.md +++ /dev/null @@ -1,161 +0,0 @@ ---- -authors: - - ivanleomk -categories: - - Gemini - - Document Processing -comments: true -date: 2024-11-15 -description: Generate accurate citations and eliminate hallucinations with structured outputs using Gemini. -draft: false -tags: - - Gemini - - Document Processing - - PDF Analysis - - Pydantic - - Python ---- - -# Eliminating Hallucinations with Structured Outputs using Gemini - -In this post, we'll explore how to use Google's Gemini model with Instructor to generate accurate citations from PDFs. This approach ensures that answers are grounded in the actual content of the PDF, reducing the risk of hallucinations. - -We'll be using the Nvidia 10k report for this example which you can download at this [link](https://d18rn0p25nwr6d.cloudfront.net/CIK-0001045810/78501ce3-7816-4c4d-8688-53dd140df456.pdf). - - - -## Introduction - -When processing PDFs, it's crucial to ensure that any answers or insights derived are directly linked to the source material. This is especially important in applications where users need to verify the origin of information, such as legal or academic contexts. - -We're using PyMuPDF here to handle PDF parsing but you can use any other library that you want. Ultimately when your citations get more complex, you'll want to invest more time into validating the PDF citations against a document. - -## Setting Up the Environment - -First, let's set up our environment with the necessary libraries: - -```bash -pip install "instructor[google-generativeai]" pymupdf -``` - -Then let's import the necessary libraries: - -```python -``` - -## Defining Our Data Models - -We'll use Pydantic to define our data models for citations and answers: - -```python -class Citation(BaseModel): - reason_for_relevance: str - text: list[str] - page_number: int - - -class Answer(BaseModel): - chain_of_thought: str - citations: list[Citation] - answer: str -``` - -## Initializing the Gemini Client - -Next, we'll set up our Gemini client using Instructor: - -```python -client = instructor.from_provider("google/gemini-2.5-flash") -) -``` - -## Processing the PDF - -To analyze a PDF and generate citations, follow these steps: - -```python -pdf_path = "./10k.pdf" -doc = pymupdf.open(pdf_path) - -# Upload the PDF -file = genai.upload_file(pdf_path) - -# Wait for file to finish processing -while file.state != File.State.ACTIVE: - time.sleep(1) - file = genai.get_file(file.name) - print(f"File is still uploading, state: {file.state}") - -resp: Answer = client.create( - messages=[ - { - "role": "system", - "content": "You are a helpful assistant that can answer questions about the provided pdf file. You will be given a question and a pdf file. Your job is to answer the question using the information in the pdf file. Provide all citations that are relevant to the question and make sure that the coordinates are accurate.", - }, - { - "role": "user", - "content": [ - "What were all of the export restrictions announced by the USG in 2023? What chips did they affect?", - file, - ], - }, - ], - response_model=Answer, -) - -print(resp) -# Answer( -# chain_of_thought="The question asks about export restrictions in 2023. Page 25 mentions the USG announcing licensing requirements for A100 and H100 chips in August 2022, and additional licensing requirements for a subset of these products in July 2023.", -# citations=[ -# Citation( -# reason_for_relevance="Describes the export licensing requirements and which chips they affect.", -# text=[ -# "In August 2022, the U.S. government, or the USG, announced licensing requirements that, with certain exceptions, impact exports to China (including Hong", -# "Kong and Macau) and Russia of our A100 and H100 integrated circuits, DGX or any other systems or boards which incorporate A100 or H100 integrated circuits.", -# "In July 2023, the USG informed us of an additional licensing requirement for a subset of A100 and H100 products destined to certain customers and other", -# "regions, including some countries in the Middle East.", -# ], -# page_number=25, -# ) -# ], -# answer="In 2023, the U.S. government (USG) announced new licensing requirements for the export of certain chips to China, Russia, and other countries. These chips included the A100 and H100 integrated circuits, the DGX system, and any other systems or boards incorporating the A100 or H100 chips.", -# ) -``` - -## Highlighting Citations in the PDF - -Once you have the citations, you can highlight them in the PDF: - -```python -for citation in resp.citations: - page = doc.load_page(citation.page_number - 1) - for text in citation.text: - text_instances = page.search_for(text) - for instance in text_instances: - page.add_highlight_annot(instance) - -doc.save("./highlighted.pdf") -doc.close() -``` - -In our case, we can see that the citations are accurate and the answer is correct. - -![Gemini Citations](./img/gemini_citations.png) - -## Why Structured Outputs? - -One of the significant advantages of using structured outputs is the ability to handle complex data extraction tasks with ease and reliability. When dealing with raw completion strings or JSON data, developers often face challenges related to parsing complexity and code maintainability. - -Over time, this just becomes error-prone, difficult to iterate upon and impossible to maintain. Instead, by leveraging pydantic, you get access to one of the best tools available for validating and parsing data. - -1. Ease of Definition: Pydantic allows you to define data models with specific fields effortlessly. This makes it easy to understand and maintain the structure of your data. -2. Robust Validation: With Pydantic, you can build validators to test against various edge cases, ensuring that your data is accurate and reliable. This is particularly useful when working with PDFs and citations, as you can validate the extracted data without worrying about the underlying language model. -3. Separation of Concerns: By using structured outputs, the language model's role is reduced to a single function call. This separation allows you to focus on building reliable and efficient data processing pipelines without being bogged down by the intricacies of the language model. - -In summary, structured outputs with Pydantic provide a powerful and ergonomic way to manage complex data extraction tasks. They enhance reliability, simplify code maintenance, and enable developers to build better applications with less effort. - -## Conclusion - -By using Gemini and Instructor, you can generate accurate citations from PDFs, ensuring that your answers are grounded in the source material. This approach is invaluable for applications requiring high levels of accuracy and traceability. - -Give instructor a try today and see how you can build reliable applications. Just run `pip install instructor` or check out our [Getting Started Guide](../../index.md) diff --git a/참고/instructor-main/docs/blog/posts/generator.md b/참고/instructor-main/docs/blog/posts/generator.md deleted file mode 100644 index 11df2c4..0000000 --- a/참고/instructor-main/docs/blog/posts/generator.md +++ /dev/null @@ -1,340 +0,0 @@ ---- -authors: -- jxnl -- anmol -categories: -- LLM Techniques -comments: true -date: 2023-11-26 -description: Explore Python generators and their role in enhancing LLM streaming for - improved latency and user experience in applications. -draft: false -slug: python-generators-and-llm-streaming -tags: -- Python -- Generators -- LLM Streaming -- Data Processing -- Performance Optimization ---- - -# Generators and LLM Streaming - -Latency is crucial, especially in eCommerce and newer chat applications like ChatGPT. Streaming is the solution that enables us to enhance the user experience without the need for faster response times. - -And what makes streaming possible? Generators! - - - -In this post, we're going to dive into the cool world of Python generators - these tools are more than just a coding syntax trick. We'll explore Python generators from the ground up and then delve into LLM streaming using the Instructor library. - -## Python Generators: An Efficient Approach to Iterables - -Generators in Python are a game-changer for handling large data sets and stream processing. They allow functions to yield values one at a time, pausing and resuming their state, which is a faster and more memory-efficient approach compared to traditional collections that store all elements in memory. - -### The Basics: Yielding Values - -A generator function in Python uses the `yield` keyword. It yields values one at a time, allowing the function to pause and resume its state. - -```python -def count_to_3(): - yield 1 - yield 2 - yield 3 - - -for num in count_to_3(): - print(num) - #> 1 - #> 2 - #> 3 -``` - -``` -1 -2 -3 -``` - -### Advantages Over Traditional Collections - -- **Lazy Evaluation & reduced latency**: The time to get the first element (or time-to-first-token in LLM land) from a generator is significantly lower. Generators only produce one value at a time, whereas accessing the first element of a collection will require that the whole collection be created first. -- **Memory Efficiency**: Only one item is in memory at a time. -- **Maintain State**: Automatically maintains state between executions. - -Let's see how much faster generators are and where they really shine: - -```python -import time - - -def expensive_func(x): - """Simulate an expensive operation.""" - time.sleep(1) - return x**2 - - -def calculate_time_for_first_result_with_list(func_input, func): - """Calculate using a list comprehension and return the first result with its computation time.""" - start_perf = time.perf_counter() - result = [func(x) for x in func_input][0] - end_perf = time.perf_counter() - print(f"Time for first result (list): {end_perf - start_perf:.2f} seconds") - #> Time for first result (list): 5.02 seconds - return result - - -def calculate_time_for_first_result_with_generator(func_input, func): - """Calculate using a generator and return the first result with its computation time.""" - start_perf = time.perf_counter() - result = next(func(x) for x in func_input) - end_perf = time.perf_counter() - print(f"Time for first result (generator): {end_perf - start_perf:.2f} seconds") - #> Time for first result (generator): 1.01 seconds - return result - - -# Prepare inputs for the function -numbers = [1, 2, 3, 4, 5] - -# Benchmarking -first_result_list = calculate_time_for_first_result_with_list(numbers, expensive_func) -first_result_gen = calculate_time_for_first_result_with_generator( - numbers, expensive_func -) -``` - -``` -Time for first result (list): 5.02 seconds -Time for first result (generator): 1.01 seconds -``` - -The generator computes one expensive operation and returns the first result immediately, while the list comprehension computes the expensive operation for all elements in the list before returning the first result. - -### Generator Expressions: A Shortcut - -Python also allows creating generators in a single line of code, known as generator expressions. They are syntactically similar to list comprehensions but use parentheses. - -```python -squares = (x * x for x in range(10)) -``` - -### Use Cases in Real-World Applications - -Generators shine in scenarios like reading large files, data streaming (eg. llm token streaming), and pipeline creation for data processing. - -## LLM Streaming - -If you've used ChatGPT, you'll see that the tokens are streamed out one by one, instead of the full response being shown at the end (can you imagine waiting for the full response??). This is made possible by generators. - -Here's how a vanilla openai generator looks: - -```python -from openai import OpenAI - -# Set your OpenAI API key -client = OpenAI( - api_key="My API Key", -) - -response_generator = client.create( - model='gpt-3.5-turbo', - messages=[{'role': 'user', 'content': "What are some good reasons to smile?"}], - temperature=0, - stream=True, -) - -for chunk in response_generator: - print(chunk.choices[0].delta.content, end="") -``` - -This is great, but what if we want to do some structured extraction on this stream? For instance, we might want to render frontend components based on product rankings that are streamed out by an LLM. - -Should we wait for the entire stream to finish before extracting & validating the list of components or can we extract & validate the components in real time as they are streamed? - -In e-commerce, every millisecond matters so the time-to-first-render can differentiate a successful and not-so-successful e commerce store (and i know how a failing e commerce store feels :/ ). - -Let's see how we can use Instructor to handle extraction from this real time stream! - -### E-commerce Product Ranking - -#### Scenario - -Imagine an e-commerce platform where we have: - -• **a customer profile**: this includes a detailed history of purchases, browsing behavior, product ratings, preferences in various categories, search history, and even responses to previous recommendations. This extensive data is crucial for generating highly personalized and relevant product suggestions. - -• **a list of candidate products**: these could be some shortlisted products we think the customer would like. - -Our goal is to re-rerank these candidate products for the best conversion and we'll use an LLM! - -#### Stream Processing - -**User Data**: - -Let's assume we have the following user profile: - -```python -profile_data = """ -Customer ID: 12345 -Recent Purchases: [Laptop, Wireless Headphones, Smart Watch] -Frequently Browsed Categories: [Electronics, Books, Fitness Equipment] -Product Ratings: {Laptop: 5 stars, Wireless Headphones: 4 stars} -Recent Search History: [best budget laptops 2023, latest sci-fi books, yoga mats] -Preferred Brands: [Apple, AllBirds, Bench] -Responses to Previous Recommendations: {Philips: Not Interested, Adidas: Not Interested} -Loyalty Program Status: Gold Member -Average Monthly Spend: $500 -Preferred Shopping Times: Weekend Evenings -... -""" -``` - -We want to rank the following products for this user: - -```python -products = [ - { - "product_id": 1, - "product_name": "Apple MacBook Air (2023) - Latest model, high performance, portable", - }, - { - "product_id": 2, - "product_name": "Sony WH-1000XM4 Wireless Headphones - Noise-canceling, long battery life", - }, - { - "product_id": 3, - "product_name": "Apple Watch Series 7 - Advanced fitness tracking, seamless integration with Apple ecosystem", - }, - { - "product_id": 4, - "product_name": "Kindle Oasis - Premium e-reader with adjustable warm light", - }, - { - "product_id": 5, - "product_name": "AllBirds Wool Runners - Comfortable, eco-friendly sneakers", - }, - { - "product_id": 6, - "product_name": "Manduka PRO Yoga Mat - High-quality, durable, eco-friendly", - }, - { - "product_id": 7, - "product_name": "Bench Hooded Jacket - Stylish, durable, suitable for outdoor activities", - }, - { - "product_id": 8, - "product_name": "GoPro HERO9 Black - 5K video, waterproof, for action photography", - }, - { - "product_id": 9, - "product_name": "Nespresso Vertuo Next Coffee Machine - Quality coffee, easy to use, compact design", - }, - { - "product_id": 10, - "product_name": "Project Hail Mary by Andy Weir - Latest sci-fi book from a renowned author", - }, -] -``` - -Let's now define our models for structured extraction. Note: instructor will conveniently let us use `Iterable` to model an iterable of our class. In this case, once we define our product recommendation model, we can slap on `Iterable` to define what we ultimately want - a (ranked) list of product recommendations. - -```python -import instructor -from openai import OpenAI -from typing import Iterable -from pydantic import BaseModel - -client = instructor.from_openai(OpenAI(), mode=instructor.function_calls.Mode.JSON) - - -class ProductRecommendation(BaseModel): - product_id: str - product_name: str - - -Recommendations = Iterable[ProductRecommendation] -``` - -Now let's use our instructor patch. Since we don't want to wait for all the tokens to finish, will set stream to `True` and process each product recommendation as it comes in: - -```python -prompt = ( - f"Based on the following user profile:\n{profile_data}\nRank the following products from most relevant to least relevant:\n" - + '\n'.join( - f"{product['product_id']} {product['product_name']}" for product in products - ) -) - -start_perf = time.perf_counter() -recommendations_stream = client.create( - model="gpt-3.5-turbo-1106", - temperature=0.1, - response_model=Iterable[ProductRecommendation], - stream=True, - messages=[ - { - "role": "system", - "content": "Generate product recommendations based on the customer profile. Return in order of highest recommended first.", - }, - {"role": "user", "content": prompt}, - ], -) -for product in recommendations_stream: - print(product) - end_perf = time.perf_counter() - print(f"Time for first result (generator): {end_perf - start_perf:.2f} seconds") - break -``` - -``` -product_id='1' product_name='Apple MacBook Air (2023)' -Time for first result (generator): 4.33 seconds -``` - -`recommendations_stream` is a generator! It yields the extracted products as it's processing the stream in real-time. Now let's get the same response without streaming and see how they compare. - -```python -start_perf = time.perf_counter() -recommendations_list = client.create( - model="gpt-3.5-turbo-1106", - temperature=0.1, - response_model=Iterable[ProductRecommendation], - stream=False, - messages=[ - { - "role": "system", - "content": "Generate product recommendations based on the customer profile. Return in order of highest recommended first.", - }, - {"role": "user", "content": prompt}, - ], -) -print(recommendations_list[0]) -end_perf = time.perf_counter() -print(f"Time for first result (list): {end_perf - start_perf:.2f} seconds") -``` - -``` -product_id='1' product_name='Apple MacBook Air (2023)' -Time for first result (list): 8.63 seconds -``` - -Our web application now displays results faster. Even a 100ms improvement can lead to a 1% increase in revenue. - -### FastAPI - -We can also take this and set up a streaming LLM API endpoint using FastAPI. Check out our docs on using FastAPI [here](../../concepts/fastapi.md)! - -## Key Takeaways - -To summarize, we looked at: - -• Generators in Python: A powerful feature that allows for efficient data handling with reduced latency - -• LLM Streaming: LLMs provide us generators to stream tokens and Instructor can let us validate and extract data from this stream. Real-time data validation ftw! - -Don't forget to check our [GitHub](https://github.com/jxnl/instructor) for more resources and give us a star if you find the library helpful! - ---- - -If you have any questions or need further clarifications, feel free to reach out or dive into the Instructor library's documentation for more detailed information. Happy coding! \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/google-openai-client.md b/참고/instructor-main/docs/blog/posts/google-openai-client.md deleted file mode 100644 index 0af1ebb..0000000 --- a/참고/instructor-main/docs/blog/posts/google-openai-client.md +++ /dev/null @@ -1,270 +0,0 @@ ---- -authors: - - ivanleomk -categories: - - Google - - OpenAI -comments: true -date: 2024-11-10 -description: Learn why Instructor remains essential even with Google's new OpenAI-compatible client for Gemini -draft: false -tags: - - Gemini ---- - -# Do I Still Need Instructor with Google's New OpenAI Integration? - -Google recently launched OpenAI client compatibility for Gemini. - -While this is a significant step forward for developers by simplifying Gemini model interactions, **you absolutely still need instructor**. - -If you're unfamiliar with instructor, we provide a simple interface to get structured outputs from LLMs across different providers. - -This makes it easy to switch between providers, get reliable outputs from language models and ultimately build production grade LLM applications. - - - -## The current state - -The new integration provides an easy integration with the Open AI Client, this means that using function calling with Gemini models has become much easier. We don't need to use a gemini specific library like `vertexai` or `google.generativeai` anymore to define response models. - -This looks something like this: - -```python -from openai import OpenAI - -client = OpenAI( - base_url="https://generativelanguage.googleapis.com/v1beta/", api_key="YOUR_API_KEY" -) - -response = client.create( - model="gemini-3-flash", - messages=[{"role": "user", "content": "Extract name and age from: John is 30"}], -) -``` - -While this seems convenient, there are three major limitations that make `instructor` still essential: - -### 1. Limited Schema Support - -The current implementation only supports simple, single-level schemas. This means you can't use complex nested schemas that are common in real-world applications. For example, this won't work: - -```python -class User(BaseModel): - name: str - age: int - - -class Users(BaseModel): - users: list[User] # Nested schema - will throw an error -``` - -### 2. No Streaming Support for Function Calling - -The integration doesn't support streaming for function calling. This is a significant limitation if your application relies on streaming responses, which is increasingly common for: - -- Real-time user interfaces -- Progressive rendering -- Long-running extractions - -### 3. No Multimodal Support - -Perhaps the biggest limitation is the lack of multimodal support. Gemini's strength lies in its ability to process multiple types of inputs (images, video, audio), but the OpenAI compatibility layer doesn't support this. This means you can't: - -- Perform visual question answering -- Extract structured data from images -- Analyze video content -- Process audio inputs - -## Why Instructor Remains Essential - -Let's see how instructor solves these issues. - -### 1. Easy Schema Management - -It's easy to define and experiment with different response models when you're building your application up. In our [own experiments](./bad-schemas-could-break-llms.md), we found that changing a single field name from `final_choice` to `answer` improved model accuracy from 4.5% to 95%. - -The way we structure and name fields in our response models can fundamentally alter how the model interprets and responds to queries. Manually editing schemas constrains your ability to iterate on your response models, introduces room for catastrophic errors and limits what you can squeeze out of your models. - -You can get the full power of Pydantic with `instructor` with gemini using our `from_gemini` and `from_vertexai` integration instead of the limited support in the OpenAI integration. - -### 2. Streaming Support - -`instructor` provides built in support for streaming, allowing you to stream partial results as they're generated. - -A common use case for streaming is to extract multiple items that have the same structure - Eg. extracting multiple users, extracting multiple products, extracting multiple events, etc. - -This is relatively easy to do with `instructor` - -```python -from instructor import from_openai -from openai import OpenAI -from instructor import Mode -from pydantic import BaseModel -import os - -client = from_openai( - OpenAI( - api_key=os.getenv("GOOGLE_API_KEY"), - base_url="https://generativelanguage.googleapis.com/v1beta/", - ), - mode=Mode.MD_JSON, -) - - -class User(BaseModel): - name: str - age: int - - -resp = client.create_iterable( - model="gemini-3-flash", - messages=[ - { - "role": "user", - "content": "Generate 10 random users", - } - ], - response_model=User, -) - -for r in resp: - print(r) -# name='Alice' age=25 -# name='Bob' age=32 -# name='Charlie' age=19 -# name='David' age=48 -# name='Emily' age=28 -# name='Frank' age=36 -# name='Grace' age=22 -# name='Henry' age=41 -# name='Isabella' age=30 -# name='Jack' age=27 -``` - -If you want to instead stream out an item as it's being generated, you can do so by using the `create_partial` method instead - -```python -from instructor import from_openai -from openai import OpenAI -from instructor import Mode -from pydantic import BaseModel -import os - -client = from_openai( - OpenAI( - api_key=os.getenv("GOOGLE_API_KEY"), - base_url="https://generativelanguage.googleapis.com/v1beta/", - ), - mode=Mode.MD_JSON, -) - - -class Story(BaseModel): - title: str - summary: str - - -resp = client.create_partial( - model="gemini-3-flash", - messages=[ - { - "role": "user", - "content": "Generate a random bedtime story + 1 sentence summary", - } - ], - response_model=Story, -) - -for r in resp: - print(r) - - -# title = None summary = None -# title='The Little Firefly Who Lost His Light' summary=None -# title='The Little Firefly Who Lost His Light' summary='A tiny firefly learns the true meaning of friendship when he loses his glow and a wise old owl helps him find it again.' -``` - -### 3. Multimodal Support - -`instructor` supports multimodal inputs for Gemini models, allowing you to perform tasks like visual question answering, image analysis, and more. - -You can see an example of how to use instructor with Gemini to [extract travel recommendations from videos](./multimodal-gemini.md) post. - -## What else does Instructor offer? - -Beyond solving the core limitations of Gemini's new OpenAI integration, instructor provides a list of features that make it indispensable for production grade applications. - -### 1. Provider Agnostic API - -Switching between providers shouldn't require rewriting your entire codebase. With instructor, it's as simple as changing just a few lines of code. - -``` -from openai import OpenAI -from instructor import from_openai - -client = from_openai( - OpenAI() -) - -# rest of code -``` - -If we wanted to switch to Anthropic, all it takes is changing the following lines of code - -```python -from anthropic import Anthropic -from instructor import from_anthropic - -client = from_anthropic(Anthropic()) - -# rest of code -``` - -### 2. Automatic Validation and Retries - -Production applications need reliable outputs. Instructor handles this by validating all outputs against your desired response model and automatically retrying outputs that fail validation. - -With [our tenacity integration](../../concepts/retrying.md), you get full control over the retries if needed, allowing you to mechanisms like exponential backoff and other retry strategies easily. - -```python -import instructor -from pydantic import BaseModel -from tenacity import Retrying, stop_after_attempt, wait_fixed - -client = instructor.from_provider("openai/gpt-5-nano", mode=instructor.Mode.TOOLS) - - -class UserDetail(BaseModel): - name: str - age: int - - -response = client.create( - model="gpt-4o-mini", - response_model=UserDetail, - messages=[ - {"role": "user", "content": "Extract `jason is 12`"}, - ], - # Stop after the second attempt and wait a fixed 1 second between attempts - max_retries=Retrying( - stop=stop_after_attempt(2), - wait=wait_fixed(1), - ), -) -print(response.model_dump_json(indent=2)) -""" -{ - "name": "jason", - "age": 12 -} -""" -``` - -## Conclusion - -While Google's OpenAI compatibility layer is a welcome addition, there are still a few reasons why you might want to stick with instructor for now. - -Within a single package, you get features such as a provider agnostic API, streaming capabilities, multimodal support, automatic re-asking and more. - -Give us a try today by installing with `pip install instructor` and see why Pydantic is all you need for a production grade LLM application.. diff --git a/참고/instructor-main/docs/blog/posts/img/Structured_Output_Extraction.gif b/참고/instructor-main/docs/blog/posts/img/Structured_Output_Extraction.gif deleted file mode 100644 index e14ceb1..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/Structured_Output_Extraction.gif and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/agent_mcp_example.png b/참고/instructor-main/docs/blog/posts/img/agent_mcp_example.png deleted file mode 100644 index 4bf6563..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/agent_mcp_example.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/async_type.png b/참고/instructor-main/docs/blog/posts/img/async_type.png deleted file mode 100644 index 3208025..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/async_type.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/chain-of-density.png b/참고/instructor-main/docs/blog/posts/img/chain-of-density.png deleted file mode 100644 index 75e361a..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/chain-of-density.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/classification-logfire.png b/참고/instructor-main/docs/blog/posts/img/classification-logfire.png deleted file mode 100644 index f649514..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/classification-logfire.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/claude_desktop_mcp.png b/참고/instructor-main/docs/blog/posts/img/claude_desktop_mcp.png deleted file mode 100644 index 63c3711..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/claude_desktop_mcp.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/claude_desktop_screenshot.png b/참고/instructor-main/docs/blog/posts/img/claude_desktop_screenshot.png deleted file mode 100644 index bb25051..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/claude_desktop_screenshot.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/course.png b/참고/instructor-main/docs/blog/posts/img/course.png deleted file mode 100644 index e6427cf..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/course.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/cursor_mcp_agent.png b/참고/instructor-main/docs/blog/posts/img/cursor_mcp_agent.png deleted file mode 100644 index 19cfdce..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/cursor_mcp_agent.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/cursor_mcp_support.png b/참고/instructor-main/docs/blog/posts/img/cursor_mcp_support.png deleted file mode 100644 index 0c66b16..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/cursor_mcp_support.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/distil_openai.png b/참고/instructor-main/docs/blog/posts/img/distil_openai.png deleted file mode 100644 index b7e7ae0..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/distil_openai.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/downloads.png b/참고/instructor-main/docs/blog/posts/img/downloads.png deleted file mode 100644 index 56447f6..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/downloads.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/dumb_rag.png b/참고/instructor-main/docs/blog/posts/img/dumb_rag.png deleted file mode 100644 index 38a5c7b..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/dumb_rag.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/gemini_citations.png b/참고/instructor-main/docs/blog/posts/img/gemini_citations.png deleted file mode 100644 index 4dce289..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/gemini_citations.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/generator.png b/참고/instructor-main/docs/blog/posts/img/generator.png deleted file mode 100644 index c3a6fa0..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/generator.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/image-logfire.png b/참고/instructor-main/docs/blog/posts/img/image-logfire.png deleted file mode 100644 index 89197e4..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/image-logfire.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/instructor-autocomplete.png b/참고/instructor-main/docs/blog/posts/img/instructor-autocomplete.png deleted file mode 100644 index 395658d..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/instructor-autocomplete.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/iterable.png b/참고/instructor-main/docs/blog/posts/img/iterable.png deleted file mode 100644 index 05b52b6..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/iterable.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/langsmith.png b/참고/instructor-main/docs/blog/posts/img/langsmith.png deleted file mode 100644 index e5ad525..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/langsmith.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/logfire-asyncio.png b/참고/instructor-main/docs/blog/posts/img/logfire-asyncio.png deleted file mode 100644 index cc6160a..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/logfire-asyncio.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/logfire-stream.png b/참고/instructor-main/docs/blog/posts/img/logfire-stream.png deleted file mode 100644 index 175abf9..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/logfire-stream.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/logfire-sync-fastapi-arguments.png b/참고/instructor-main/docs/blog/posts/img/logfire-sync-fastapi-arguments.png deleted file mode 100644 index 4d87986..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/logfire-sync-fastapi-arguments.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/logfire-sync-pydantic-validation.png b/참고/instructor-main/docs/blog/posts/img/logfire-sync-pydantic-validation.png deleted file mode 100644 index 494960a..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/logfire-sync-pydantic-validation.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/mcp_architecture.png b/참고/instructor-main/docs/blog/posts/img/mcp_architecture.png deleted file mode 100644 index 61b3ea0..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/mcp_architecture.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/mcp_stars.webp b/참고/instructor-main/docs/blog/posts/img/mcp_stars.webp deleted file mode 100644 index 18cda36..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/mcp_stars.webp and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/meta.png b/참고/instructor-main/docs/blog/posts/img/meta.png deleted file mode 100644 index fad7441..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/meta.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/parea/form-mode.gif b/참고/instructor-main/docs/blog/posts/img/parea/form-mode.gif deleted file mode 100644 index 5440d5c..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/parea/form-mode.gif and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/parea/trace.png b/참고/instructor-main/docs/blog/posts/img/parea/trace.png deleted file mode 100644 index 060c7d5..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/parea/trace.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/parea/validation-error-chart.png b/참고/instructor-main/docs/blog/posts/img/parea/validation-error-chart.png deleted file mode 100644 index 8b63372..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/parea/validation-error-chart.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/query_understanding.png b/참고/instructor-main/docs/blog/posts/img/query_understanding.png deleted file mode 100644 index 146dec3..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/query_understanding.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/statista-image.jpeg b/참고/instructor-main/docs/blog/posts/img/statista-image.jpeg deleted file mode 100644 index c3aa3b7..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/statista-image.jpeg and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/style_1.png b/참고/instructor-main/docs/blog/posts/img/style_1.png deleted file mode 100644 index 6ea80cd..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/style_1.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/style_2.png b/참고/instructor-main/docs/blog/posts/img/style_2.png deleted file mode 100644 index 14c2c84..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/style_2.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/style_3.png b/참고/instructor-main/docs/blog/posts/img/style_3.png deleted file mode 100644 index 7f899de..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/style_3.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/style_4.png b/참고/instructor-main/docs/blog/posts/img/style_4.png deleted file mode 100644 index aec4003..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/style_4.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/style_5.png b/참고/instructor-main/docs/blog/posts/img/style_5.png deleted file mode 100644 index 2475038..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/style_5.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/style_6.png b/참고/instructor-main/docs/blog/posts/img/style_6.png deleted file mode 100644 index 9549ac3..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/style_6.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/type.png b/참고/instructor-main/docs/blog/posts/img/type.png deleted file mode 100644 index b65c510..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/type.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/untidy_table.png b/참고/instructor-main/docs/blog/posts/img/untidy_table.png deleted file mode 100644 index 3e6b947..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/untidy_table.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/validation-logfire.png b/참고/instructor-main/docs/blog/posts/img/validation-logfire.png deleted file mode 100644 index 60e9abb..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/validation-logfire.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/with_completion.png b/참고/instructor-main/docs/blog/posts/img/with_completion.png deleted file mode 100644 index 48121fd..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/with_completion.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/youtube-clips.gif b/참고/instructor-main/docs/blog/posts/img/youtube-clips.gif deleted file mode 100644 index 6cf8802..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/youtube-clips.gif and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/youtube-flashcards/annotations.png b/참고/instructor-main/docs/blog/posts/img/youtube-flashcards/annotations.png deleted file mode 100644 index a3b65b4..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/youtube-flashcards/annotations.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/youtube-flashcards/flashcards.png b/참고/instructor-main/docs/blog/posts/img/youtube-flashcards/flashcards.png deleted file mode 100644 index 40fa410..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/youtube-flashcards/flashcards.png and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/img/youtube-flashcards/telemetry.gif b/참고/instructor-main/docs/blog/posts/img/youtube-flashcards/telemetry.gif deleted file mode 100644 index 2fae2aa..0000000 Binary files a/참고/instructor-main/docs/blog/posts/img/youtube-flashcards/telemetry.gif and /dev/null differ diff --git a/참고/instructor-main/docs/blog/posts/introducing-structured-outputs-with-cerebras-inference.md b/참고/instructor-main/docs/blog/posts/introducing-structured-outputs-with-cerebras-inference.md deleted file mode 100644 index 1d5d54f..0000000 --- a/참고/instructor-main/docs/blog/posts/introducing-structured-outputs-with-cerebras-inference.md +++ /dev/null @@ -1,130 +0,0 @@ ---- -authors: - - ivanleomk - - sarahchieng -categories: - - API Development - - Pydantic - - Performance Optimization -comments: true -date: 2024-10-15 -description: - Learn how to use Cerebras Inference for structured outputs, faster model - inference, and seamless integration with Pydantic models. -draft: false -slug: introducing-structured-outputs-with-cerebras-inference -tags: - - Cerebras Inference - - Pydantic - - API Integration - - Fast Inference - - Structured Outputs ---- - -# Introducing structured outputs with Cerebras Inference - -## What's Cerebras? - -Cerebras offers the fastest inference on the market, 20x faster than on GPUs. - -Sign up for a Cerebras Inference API key here at [cloud.cerebras.ai](http://cloud.cerebras.ai). - -### Basic Usage - -To get guaranteed structured outputs with Cerebras Inference, you - - - -1. Create a new Instructor client with the `from_cerebras` method -2. Define a Pydantic model to pass into the `response_model` parameter -3. Get back a validated response exactly as you would expect - -You'll also need to install the Cerebras SDK to use the client. You can install it with the command below. - - - -```bash -pip install "instructor[cerebras_cloud_sdk]" -``` - -This ensures that you have the necessary dependencies to use the Cerebras SDK with instructor. - -### Getting Started - -Before running the following code, you'll need to make sure that you have your CEREBRAS_API_KEY. Sign up for one [here](https://cloud.cerebras.ai/). - -Make sure to set the `CEREBRAS_API_KEY` as an alias in your shell. - -```bash -export CEREBRAS_API_KEY= -``` - -Once you've done so, you can use the following code to get started. - -```python -import instructor -from pydantic import BaseModel - -client = instructor.from_provider("cerebras/llama3.1-70b") - - -class Person(BaseModel): - name: str - age: int - - -resp = client.create( - model="llama3.1-70b", - messages=[ - { - "role": "user", - "content": "Extract the name and age of the person in this sentence: John Smith is 29 years old.", - } - ], - response_model=Person, -) - -print(resp) -#> Person(name='John Smith', age=29) -``` - -We support both the `AsyncCerebras` and `Cerebras` clients. - -### Streaming - -We also support streaming with the Cerebras client with the `CEREBRAS_JSON` mode so that you can take advantage of Cerebras’s inference speeds and process the response as it comes in. - -```python -import instructor -from cerebras.cloud.sdk import Cerebras -from pydantic import BaseModel -from typing import Iterable - -client = instructor.from_cerebras(Cerebras(), mode=instructor.Mode.MD_JSON) - - -class Person(BaseModel): - name: str - age: int - - -resp = client.create( - model="llama3.1-70b", - messages=[ - { - "role": "user", - "content": "Extract all users from this sentence : Chris is 27 and lives in San Francisco, John is 30 and lives in New York while their college roommate Jessica is 26 and lives in London", - } - ], - response_model=Iterable[Person], - stream=True, -) - -for person in resp: - print(person) - #> Person(name='Chris', age=27) - #> Person(name='John', age=30) - #> Person(name='Jessica', age=26) -``` - -And that’s it! We're excited to see what you build with Instructor and Cerebras! If you have any questions about Cerebras or need to get off the API key waitlist, please reach out to sarah.chieng@cerebras.net. diff --git a/참고/instructor-main/docs/blog/posts/introducing-structured-outputs.md b/참고/instructor-main/docs/blog/posts/introducing-structured-outputs.md deleted file mode 100644 index 8387212..0000000 --- a/참고/instructor-main/docs/blog/posts/introducing-structured-outputs.md +++ /dev/null @@ -1,405 +0,0 @@ ---- -authors: -- ivanleomk -categories: -- OpenAI -comments: true -date: 2024-08-20 -description: Explore the challenges of OpenAI's Structured Outputs and how 'instructor' - offers solutions for LLM workflows. -draft: false -slug: should-i-be-using-structured-outputs -tags: -- OpenAI -- Structured Outputs -- Pydantic -- Data Validation -- LLM Techniques ---- - -# Should I Be Using Structured Outputs? - -OpenAI recently announced Structured Outputs which ensures that generated responses match any arbitrary provided JSON Schema. In their [announcement article](https://openai.com/index/introducing-structured-outputs-in-the-api/), they acknowledged that it had been inspired by libraries such as `instructor`. - -## Main Challenges - -If you're building complex LLM workflows, you've likely considered OpenAI's Structured Outputs as a potential replacement for `instructor`. - -But before you do so, three key challenges remain: - -1. **Limited Validation And Retry Logic**: Structured Outputs ensure adherence to the schema but not useful content. You might get perfectly formatted yet unhelpful responses -2. **Streaming Challenges**: Parsing raw JSON objects from streamed responses with the sdk is error-prone and inefficient -3. **Unpredictable Latency Issues** : Structured Outputs suffers from random latency spikes that might result in an almost 20x increase in response time - -Additionally, adopting Structured Outputs locks you into OpenAI's ecosystem, limiting your ability to experiment with diverse models or providers that might better suit specific use-cases. - -This vendor lock-in increases vulnerability to provider outages, potentially causing application downtime and SLA violations, which can damage user trust and impact your business reputation. - -In this article, we'll show how `instructor` addresses many of these challenges with features such as automatic reasking when validation fails, automatic support for validated streaming data and more. - - - -### Limited Validation and Retry Logic - -Validation is crucial for building reliable and effective applications. We want to catch errors in real time using `Pydantic` [validators](../../concepts/reask_validation.md) in order to allow our LLM to correct its responses on the fly. - -Let's see an example of a simple validator below which ensures user names are always in uppercase. - -```python -import openai -from pydantic import BaseModel, field_validator - - -class User(BaseModel): - name: str - age: int - - @field_validator("name") - def ensure_uppercase(cls, v: str) -> str: - if not v.isupper(): - raise ValueError("All letters must be uppercase. Got: " + v) - return v - - -client = openai.OpenAI() -try: - resp = client.beta.chat.completions.parse( - response_format=User, - messages=[ - { - "role": "user", - "content": "Extract the following user: Jason is 25 years old.", - }, - ], - model="gpt-4o-mini", - ) -except Exception as e: - print(e) - """ - 1 validation error for User - name - Value error, All letters must be uppercase. Got: Jason [type=value_error, input_value='Jason', input_type=str] - For further information visit https://errors.pydantic.dev/2.11/v/value_error - """ -``` - -We can see that we lose the original completion when validation fails. This leaves developers without the means to implement retry logic so that the LLM can provide a targeted correction and regenerate its response. - -Without robust validation, applications risk producing inconsistent outputs and losing valuable context for error correction. This leads to degraded user experience and missed opportunities for targeted improvements in LLM responses. - -### Streaming Challenges - -Streaming with Structured Outputs is complex. It requires manual parsing, lacks partial validation, and needs a context manager to be used with. Effective implementation with the `beta.chat.completions.stream` method demands significant effort. - -Let's see an example below. - -```python -import openai -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -client = openai.OpenAI() -with client.beta.chat.completions.stream( - response_format=User, - messages=[ - { - "role": "user", - "content": "Extract the following user: Jason is 25 years old.", - }, - ], - model="gpt-4o-mini", -) as stream: - for event in stream: - if event.type == "content.delta": - print(event.snapshot, flush=True, end="\n") - #> - #> {" - #> {"name - #> {"name":" - #> {"name":"Jason - #> {"name":"Jason"," - #> {"name":"Jason","age - #> {"name":"Jason","age": - #> {"name":"Jason","age":25 - #> {"name":"Jason","age":25} - # > - #> {" - #> {"name - #> {"name":" - #> {"name":"Jason - #> {"name":"Jason"," - #> {"name":"Jason","age - #> {"name":"Jason","age": - #> {"name":"Jason","age":25 - #> {"name":"Jason","age":25} - # > - #> {" - #> {"name - #> {"name":" - #> {"name":"Jason - #> {"name":"Jason"," - #> {"name":"Jason","age - #> {"name":"Jason","age": - #> {"name":"Jason","age":25 - #> {"name":"Jason","age":25} - # > - #> {" - #> {"name - #> {"name":" - #> {"name":"Jason - #> {"name":"Jason"," - #> {"name":"Jason","age - #> {"name":"Jason","age": - #> {"name":"Jason","age":25 - #> {"name":"Jason","age":25} -``` - -### Unpredictable Latency Spikes - -In order to benchmark the two modes, we made 200 identical requests to OpenAI and noted the time taken for each request to complete. The results are summarized in the following table: - -| mode | mean | min | max | std_dev | variance | -| ------------------ | ----- | ----- | ------ | ------- | -------- | -| Tool Calling | 6.84 | 6.21 | 12.84 | 0.69 | 0.47 | -| Structured Outputs | 28.20 | 14.91 | 136.90 | 9.27 | 86.01 | - -Structured Outputs suffers from unpredictable latency spikes while Tool Calling maintains consistent performance. This could cause users to occasionally experience significant delays in response times, potentially impacting the overall user satisfication and retention rates. - -## Why use `instructor` - -`instructor` is fully compatible with Structured Outputs and provides three main benefits to developers. - -1. **Automatic Validation and Retries**: Regenerates LLM responses on Pydantic validation failures, ensuring data integrity. -2. **Real-time Streaming Validation**: Incrementally validates partial JSON against Pydantic models, enabling immediate use of validated properties. -3. **Provider-Agnostic API**: Switch between LLM providers and models with a single line of code. - -Let's see this in action below - -### Automatic Validation and Retries - -With `instructor`, all it takes is a simple Pydantic Schema and a validator for you to get the extracted names as an upper case value. - -```python -import instructor -from pydantic import BaseModel, field_validator - - -class User(BaseModel): - name: str - age: int - - @field_validator("name") - def ensure_uppercase(cls, v: str) -> str: - if not v.isupper(): - raise ValueError("All letters must be uppercase. Got: " + v) - return v - - -client = instructor.from_provider( - "openai/gpt-5-nano", mode=instructor.Mode.TOOLS_STRICT -) - -resp = client.create( - response_model=User, - messages=[ - { - "role": "user", - "content": "Extract the following user: Jason is 25 years old.", - } - ], - model="gpt-4o-mini", -) - -print(resp) -#> name='JASON' age=25 -``` - -This built-in retry logic allows for targeted correction to the generated response, ensuring that outputs are not only consistent with your schema but also correct for your use-case. This is invaluable in building reliable LLM systems. - -### Real-time Streaming Validation - -A common use-case is to define a single schema and extract multiple instances of it. With `instructor`, doing this is relatively straightforward by using [our `create_iterable` method](../../concepts/lists.md). - -```python -client = instructor.from_provider( - "openai/gpt-5-nano", mode=instructor.Mode.TOOLS_STRICT -) - - -class User(BaseModel): - name: str - age: int - - -users = client.create_iterable( - model="gpt-4o-mini", - response_model=User, - messages=[ - { - "role": "system", - "content": "You are a perfect entity extraction system", - }, - { - "role": "user", - "content": (f"Extract `Jason is 10 and John is 10`"), - }, - ], -) - -for user in users: - print(user) - #> name='Jason' age=10 - #> name='John' age=10 -``` - -Other times, we might also want to stream out information as it's dynamically generated into some sort of frontend component With `instructor`, you'll be able to do just that [using the `create_partial` method](../../concepts/partial.md). - -```python -import instructor -from pydantic import BaseModel -from rich.console import Console - -client = instructor.from_provider( - "openai/gpt-5-nano", mode=instructor.Mode.TOOLS_STRICT -) - -text_block = """ -In our recent online meeting, participants from various backgrounds joined to discuss the upcoming tech conference. The names and contact details of the participants were as follows: - -- Name: John Doe, Email: johndoe@email.com, Twitter: @TechGuru44 -- Name: Jane Smith, Email: janesmith@email.com, Twitter: @DigitalDiva88 -- Name: Alex Johnson, Email: alexj@email.com, Twitter: @CodeMaster2023 - -During the meeting, we agreed on several key points. The conference will be held on March 15th, 2024, at the Grand Tech Arena located at 4521 Innovation Drive. Dr. Emily Johnson, a renowned AI researcher, will be our keynote speaker. - -The budget for the event is set at $50,000, covering venue costs, speaker fees, and promotional activities. Each participant is expected to contribute an article to the conference blog by February 20th. - -A follow-up meeting is scheduled for January 25th at 3 PM GMT to finalize the agenda and confirm the list of speakers. -""" - - -class User(BaseModel): - name: str - email: str - twitter: str - - -class MeetingInfo(BaseModel): - users: list[User] - date: str - location: str - budget: int - deadline: str - - -extraction_stream = client.create_partial( - model="gpt-4o-mini", - response_model=MeetingInfo, - messages=[ - { - "role": "user", - "content": f"Get the information about the meeting and the users {text_block}", - }, - ], - stream=True, -) - - -console = Console() - -for extraction in extraction_stream: - obj = extraction.model_dump() - console.clear() - console.print(obj) -``` - -This will output the following - -![Structured Output Extraction](./img/Structured_Output_Extraction.gif) - -### Provider-Agnostic API - -With `instructor`, switching between different providers is easy due to our unified API. - -For example, the switch from OpenAI to Anthropic requires only three adjustments - -1. Import the Anthropic client -2. Use `from_anthropic` instead of `from_openai` -3. Update the model name (e.g., from gpt-4o-mini to claude-3-5-sonnet) - -This makes it incredibly flexible for users looking to migrate and test different providers for their use cases. Let's see this in action with an example below. - -```python -import instructor -from pydantic import BaseModel - -client = instructor.from_provider("openai/gpt-5-nano") - - -class User(BaseModel): - name: str - age: int - - -resp = client.create( - model="gpt-4o-mini", - response_model=User, - messages=[ - { - "role": "user", - "content": "Extract the user from the string belo - Chris is a 27 year old engineer in San Francisco", - } - ], - max_tokens=100, -) - -print(resp) -#> name='Chris' age=27 -``` - -Now let's see how we can achieve the same with Anthropic. - -```python hl_lines="2 5 14" -import instructor -from pydantic import BaseModel - -client = instructor.from_provider("anthropic/claude-3-5-haiku-latest") # (2)! - - -class User(BaseModel): - name: str - age: int - - -resp = client.create( - model="claude-3-5-sonnet-20240620", # (3)! - response_model=User, - messages=[ - { - "role": "user", - "content": "Extract the user from the string belo - Chris is a 27 year old engineer in San Francisco", - } - ], - max_tokens=100, -) - -print(resp) -#> name='Chris' age=27 -``` - -1. Import the Anthropic client -2. Use `from_anthropic` instead of `from_openai` -3. Update the model name to `claude-3-5-sonnet-20240620` - -## Conclusion - -While OpenAI's Structured Outputs shows promise, it has key limitations. The system lacks support for extra JSON fields to provide output examples, default value factories, and pattern matching in defined schemas. These constraints limit developers' ability to express complex return types, potentially impacting application performance and flexibility. - -If you're interested in Structured Outputs, `instructor` addresses these critical issues. It provides automatic retries, real-time input validation, and multi-provider integration, allowing developers to more effectively implement Structured Outputs in their AI projects. - -if you haven't given `instructor` a shot, try it today! diff --git a/참고/instructor-main/docs/blog/posts/introduction.md b/참고/instructor-main/docs/blog/posts/introduction.md deleted file mode 100644 index 942fffa..0000000 --- a/참고/instructor-main/docs/blog/posts/introduction.md +++ /dev/null @@ -1,224 +0,0 @@ ---- -authors: -- jxnl -categories: -- Pydantic -comments: true -date: 2023-09-11 -description: Learn how Pydantic simplifies working with LLMs and structured JSON outputs - in Python, enhancing developer experience and code organization. -draft: false -tags: -- Pydantic -- LLMs -- Python -- OpenAI -- JSON ---- - -# Generating Structured Output / JSON from LLMs - -Language models have seen significant growth. Using them effectively often requires complex frameworks. This post discusses how Instructor simplifies this process using Pydantic. - - - -## The Problem with Existing LLM Frameworks - -Current frameworks for Language Learning Models (LLMs) have complex setups. Developers find it hard to control interactions with language models. Some frameworks require complex JSON Schema setups. - -## The OpenAI Function Calling Game-Changer - -OpenAI's Function Calling feature provides a constrained interaction model. However, it has its own complexities, mostly around JSON Schema. - -## Why Pydantic? - -Instructor uses Pydantic to simplify the interaction between the programmer and the language model. - -- **Widespread Adoption**: Pydantic is a popular tool among Python developers. -- **Simplicity**: Pydantic allows model definition in Python. -- **Framework Compatibility**: Many Python frameworks already use Pydantic. - -```python -import pydantic -import instructor - -# Enables the response_model -client = instructor.from_provider("openai/gpt-5-nano") - - -class UserDetail(pydantic.BaseModel): - name: str - age: int - - def introduce(self): - return f"Hello I'm {self.name} and I'm {self.age} years old" - - -user: UserDetail = client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": "Extract Jason is 25 years old"}, - ], -) -``` - -## Simplifying Validation Flow with Pydantic - -Pydantic validators simplify features like re-asking or self-critique. This makes these tasks less complex compared to other frameworks. - -```python -from typing_extensions import Annotated -from pydantic import BaseModel, BeforeValidator -from instructor import llm_validator - - -class QuestionAnswerNoEvil(BaseModel): - question: str - answer: Annotated[ - str, - BeforeValidator(llm_validator("don't say objectionable things")), - ] -``` - -## The Modular Approach - -Pydantic allows for modular output schemas. This leads to more organized code. - -### Composition of Schemas - -```python -class UserDetails(BaseModel): - name: str - age: int - - -class UserWithAddress(UserDetails): - address: str -``` - -### Defining Relationships - -```python -class UserDetail(BaseModel): - id: int - age: int - name: str - friends: List[int] - - -class UserRelationships(BaseModel): - users: List[UserDetail] -``` - -### Using Enums - -```python -from enum import Enum, auto - - -class Role(Enum): - PRINCIPAL = auto() - TEACHER = auto() - STUDENT = auto() - OTHER = auto() - - -class UserDetail(BaseModel): - age: int - name: str - role: Role -``` - -### Flexible Schemas - -```python -from typing import List - - -class Property(BaseModel): - key: str - value: str - - -class UserDetail(BaseModel): - age: int - name: str - properties: List[Property] -``` - -### Chain of Thought - -```python -class TimeRange(BaseModel): - chain_of_thought: str - start_time: int - end_time: int - - -class UserDetail(BaseModel): - id: int - age: int - name: str - work_time: TimeRange - leisure_time: TimeRange -``` - -## Language Models as Microservices - -The architecture resembles FastAPI. Most code can be written as Python functions that use Pydantic objects. This eliminates the need for prompt chains. - -### FastAPI Stub - -```python -import fastapi -from pydantic import BaseModel - -class UserDetails(BaseModel): - name: str - age: int - -app = fastapi.FastAPI() - -@app.get("/user/{user_id}", response_model=UserDetails) -async def get_user(user_id: int) -> UserDetails: - return ... -``` - -### Using Instructor as a Function - -```python -def extract_user(str) -> UserDetails: - return client.chat.completions( - response_model=UserDetails, - messages=[] - ) -``` - -### Response Modeling - -```python -class MaybeUser(BaseModel): - result: Optional[UserDetail] - error: bool - message: Optional[str] -``` - -## Conclusion - -Instructor, with Pydantic, simplifies interaction with language models. It is usable for both experienced and new developers. - -## Related Concepts - -- [Getting Started Guide](../../index.md) - Learn how to install and use Instructor -- [Model Providers](../../integrations/index.md) - Explore supported LLM providers -- [Validation Context](../../concepts/reask_validation.md) - Understand how to validate LLM outputs -- [Response Models](../../concepts/models.md) - Deep dive into defining structured outputs - -## See Also - -- [Why Instructor is the Best Library](best_framework.md) - Learn about Instructor's philosophy and advantages -- [Structured Outputs and Prompt Caching with Anthropic](structured-output-anthropic.md) - See how Instructor works with Claude -- [Chain of Density Tutorial](../../tutorials/6-chain-of-density.ipynb) - Learn advanced prompting techniques - -If you enjoy the content or want to try out `instructor` please check out the [github](https://github.com/jxnl/instructor) and give us a star! \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/jinja-proposal.md b/참고/instructor-main/docs/blog/posts/jinja-proposal.md deleted file mode 100644 index e4c0ebb..0000000 --- a/참고/instructor-main/docs/blog/posts/jinja-proposal.md +++ /dev/null @@ -1,242 +0,0 @@ ---- -authors: -- jxnl -categories: -- LLM Techniques -comments: true -date: 2024-09-19 -description: Explore the integration of Jinja templating in the Instructor for enhanced - formatting, validation, versioning, and secure logging. -draft: false -tags: -- Jinja -- Templating -- Pydantic -- API Development -- Data Validation ---- - -# Instructor Proposal: Integrating Jinja Templating - -As the creator of Instructor, I've always aimed to keep our product development streamlined and avoid unnecessary complexity. However, I'm now convinced that it's time to incorporate better templating into our data structure, specifically by integrating Jinja. - -This decision serves multiple purposes: - -1. It addresses the growing complexity in my prompt formatting needs -2. It allows us to differentiate ourselves from the standard library while adding proven utility. -3. It aligns with the practices I've consistently employed in both production and client code. -4. It provides an opportunity to introduce API changes that have been tested in private versions of Instructor. - -## Why Jinja is the Right Choice - -1. **Formatting Capabilities** - - Prompt formatting complexity has increased. - - List iteration and conditional implementation are necessary for formatting. - - This improves chunk generation, few shots, and dynamic rules. - -2. **Validation** - - Jinja template variables serve rendering and validation purposes. - - Pydantic's validation context allows access to template variables in validation functions. - -3. **Versioning and Logging** - - Render variable separation enhances prompt versioning and logging. - - Template variable diffing simplifies prompt change comparisons. - -By integrating Jinja into Instructor, we're not just adding a feature; we're enhancing our ability to handle complex formatting, improve validation processes, and streamline our versioning and logging capabilities. This addition will significantly boost the power and flexibility of Instructor, making it an even more robust tool for our users. - -## Enhancing Formatting Capabilities - -In Instructor, we propose implementing a new `context` keyword in our create methods. This addition will allow users to render the prompt using a provided context, leveraging Jinja's templating capabilities. Here's how it would work: - -1. Users pass a `context` dictionary to the create method. -2. The prompt template, written in Jinja syntax, is defined in the `content` field of the message. -3. Instructor renders the prompt using the provided context, filling in the template variables. - -This approach offers these benefits: - -- Separation of prompt structure and dynamic content -- Management of complex prompts with conditionals and loops -- Reusability of prompt templates across different contexts - -Let's look at an example to illustrate this feature: - -```python -client.create( - model="gpt-4o", - messages=[ - { - "role": "user", - "content": """ - You are a {{ role }} tasks with the following question - - - {{ question }} - - - Use the following context to answer the question, make sure to return [id] for every citation: - - - {% for chunk in context %} - - {{ chunk.id }} - {{ chunk.text }} - - {% endfor %} - - - {% if rules %} - Make sure to follow these rules: - - {% for rule in rules %} - * {{ rule }} - {% endfor %} - {% endif %} - """, - }, - ], - context={ - "role": "professional educator", - "question": "What is the capital of France?", - "context": [ - {"id": 1, "text": "Paris is the capital of France."}, - {"id": 2, "text": "France is a country in Europe."}, - ], - "rules": ["Use markdown."], - }, -) -``` - -## Validation - -Let's consider a scenario where we redact words from text. By using `ValidationInfo` to access context and passing it to the validator and template, we can implement a system for handling sensitive information. This approach allows us to: - -1. Validate input to ensure it doesn't contain banned words. -2. Redact patterns using regular expressions. -3. Provide instructions to the language model about word usage restrictions. - -Here's an example demonstrating this concept using Pydantic validators: - -```python -from pydantic import BaseModel, ValidationInfo, field_validator - -class Response(BaseModel): - text: str - - @field_validator('text') - @classmethod - def no_banned_words(cls, v: str, info: ValidationInfo): - context = info.context - if context: - banned_words = context.get('banned_words', set()) - banned_words_found = [word for word in banned_words if word.lower() in v.lower()] - if banned_words_found: - raise ValueError(f"Banned words found in text: {', '.join(banned_words_found)}, rewrite it but just without the banned words") - return v - - @field_validator('text') - @classmethod - def redact_regex(cls, v: str, info: ValidationInfo): - context = info.context - if context: - redact_patterns = context.get('redact_patterns', []) - for pattern in redact_patterns: - v = re.sub(pattern, '****', v) - return v - -response = client.create( - model="gpt-4o", - response_model=Response, - messages=[ - { - "role": "user", - "content": """ - Write about a {{ topic }} - - {% if banned_words %} - You must not use the following banned words: - - - {% for word in banned_words %} - * {{ word }} - {% endfor %} - - {% endif %} - """ - }, - ], - context={ - "topic": "jason and now his phone number is 123-456-7890" - "banned_words": ["jason"], - "redact_patterns": [ - r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b", # Phone number pattern - r"\b\d{3}-\d{2}-\d{4}\b", # SSN pattern - ], - }, - max_retries=3, -) - -print(response.text) -# > While i can't say his name anymore, his phone number is **** -``` - -## Better Versioning and Logging - -With the separation of prompt templates and variables, we gain several advantages: - -1. Version Control: We can now version the templates and retrieve the appropriate one for a given prompt. This allows for better management of template history, diffing and comparison. - -2. Enhanced Logging: The separation facilitates structured logging, enabling easier debugging and integration with various logging sinks, databases, and observability tools like OpenTelemetry. - -3. Security: Sensitive information in variables can be handled separately from the templates, allowing for better access control and data protection. - -This separation of concerns adheres to best practices in software design, resulting in a more maintainable, scalable, and robust system for managing prompts and their associated data. - -### Side effect of Context also being Pydantic Models - -Since they are just python objects we can use Pydantic models to validate the context and also control how they are rendered, so even secret information can be dynamically rendered! -Consider using secret string to pass in sensitive information to the llm. - -```python -from pydantic import BaseModel, SecretStr - - -class UserContext(BaseModel): - name: str - address: SecretStr - - -class Address(BaseModel): - street: SecretStr - city: str - state: str - zipcode: str - - -def normalize_address(address: Address): - context = UserContext(username="scolvin", address=address) - address = client.create( - model="gpt-4o", - messages=[ - { - "role": "user", - "content": "{{ user.name }} is `{{ user.address.get_secret_value() }}`, normalize it to an address object", - }, - ], - context={"user": context}, - ) - print(context) - #> UserContext(username='jliu', address="******") - print(address) - #> Address(street='******', city="Toronto", state="Ontario", zipcode="M5A 0J3") - logger.info( - f"Normalized address: {address}", - extra={"user_context": context, "address": address}, - ) - return address -``` - -This approach offers several advantages: - -1. Secure logging: You can confidently log your template variables without risking the exposure of sensitive information. -2. Type safety: Pydantic models provide type checking and validation, reducing the risk of errors. -3. Flexibility: You can easily control how different types of data are displayed or used in templates. \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/langsmith.md b/참고/instructor-main/docs/blog/posts/langsmith.md deleted file mode 100644 index 175684b..0000000 --- a/참고/instructor-main/docs/blog/posts/langsmith.md +++ /dev/null @@ -1,176 +0,0 @@ ---- -authors: -- jxnl -categories: -- LLM Techniques -comments: true -date: 2024-02-18 -description: Explore how LangSmith enhances OpenAI clients with seamless LLM observability - and the `instructor` package for question classification. -draft: false -tags: -- LangSmith -- OpenAI -- LLM -- Python -- API Development ---- - -# Seamless Support with Langsmith - -Its a common misconception that LangChain's [LangSmith](https://www.langchain.com/langsmith) is only compatible with LangChain's models. In reality, LangSmith is a unified DevOps platform for developing, collaborating, testing, deploying, and monitoring LLM applications. In this blog we will explore how LangSmith can be used to enhance the OpenAI client alongside `instructor`. - - - -## LangSmith - -In order to use langsmith, you first need to set your LangSmith API key. - -``` -export LANGCHAIN_API_KEY= -``` - -Next, you will need to install the LangSmith SDK: - -``` -pip install -U langsmith -pip install -U instructor -``` - -You can find this example in our [examples directory](../../examples/bulk_classification.md): - -```bash -# The example code is available in the examples directory -# See: https://python.useinstructor.com/examples/bulk_classification -``` - -In this example we'll use the `wrap_openai` function to wrap the OpenAI client with LangSmith. This will allow us to use LangSmith's observability and monitoring features with the OpenAI client. Then we'll use `instructor` to patch the client with the `TOOLS` mode. This will allow us to use `instructor` to add additional functionality to the client. We'll use [asyncio](./learn-async.md) to classify a list of questions. - -```python -import instructor -import asyncio - -from langsmith import traceable -from langsmith.wrappers import wrap_openai - -from openai import AsyncOpenAI -from pydantic import BaseModel, Field, field_validator -from typing import List -from enum import Enum - -# Wrap the OpenAI client with LangSmith -wrapped_client = wrap_openai(AsyncOpenAI()) - -# Create instructor client with LangSmith-wrapped client -# Note: When using LangSmith, you may need to pass the wrapped client -# For most cases, use: client = instructor.from_provider("openai/gpt-4o", mode=instructor.Mode.TOOLS) -client = instructor.from_provider("openai/gpt-4o", mode=instructor.Mode.TOOLS) - -# Rate limit the number of requests -sem = asyncio.Semaphore(5) - - -# Use an Enum to define the types of questions -class QuestionType(Enum): - CONTACT = "CONTACT" - TIMELINE_QUERY = "TIMELINE_QUERY" - DOCUMENT_SEARCH = "DOCUMENT_SEARCH" - COMPARE_CONTRAST = "COMPARE_CONTRAST" - EMAIL = "EMAIL" - PHOTOS = "PHOTOS" - SUMMARY = "SUMMARY" - - -# You can add more instructions and examples in the description -# or you can put it in the prompt in `messages=[...]` -class QuestionClassification(BaseModel): - """ - Predict the type of question that is being asked. - Here are some tips on how to predict the question type: - CONTACT: Searches for some contact information. - TIMELINE_QUERY: "When did something happen? - DOCUMENT_SEARCH: "Find me a document" - COMPARE_CONTRAST: "Compare and contrast two things" - EMAIL: "Find me an email, search for an email" - PHOTOS: "Find me a photo, search for a photo" - SUMMARY: "Summarize a large amount of data" - """ - - # If you want only one classification, just change it to - # `classification: QuestionType` rather than `classifications: List[QuestionType]`` - chain_of_thought: str = Field( - ..., description="The chain of thought that led to the classification" - ) - classification: List[QuestionType] = Field( - description=f"An accuracy and correct prediction predicted class of question. Only allowed types: {[t.value for t in QuestionType]}, should be used", - ) - - @field_validator("classification", mode="before") - def validate_classification(cls, v): - # sometimes the API returns a single value, just make sure it's a list - if not isinstance(v, list): - v = [v] - return v - - -@traceable(name="classify-question") -async def classify(data: str) -> QuestionClassification: - """ - Perform multi-label classification on the input text. - Change the prompt to fit your use case. - - Args: - data (str): The input text to classify. - """ - async with sem: # some simple rate limiting - return data, await client.create( - model="gpt-4-turbo-preview", - response_model=QuestionClassification, - max_retries=2, - messages=[ - { - "role": "user", - "content": f"Classify the following question: {data}", - }, - ], - ) - - -async def main(questions: List[str]): - tasks = [classify(question) for question in questions] - - for task in asyncio.as_completed(tasks): - question, label = await task - resp = { - "question": question, - "classification": [c.value for c in label.classification], - "chain_of_thought": label.chain_of_thought, - } - resps.append(resp) - return resps - - -if __name__ == "__main__": - import asyncio - - questions = [ - "What was that ai app that i saw on the news the other day?", - "Can you find the trainline booking email?", - "what did I do on Monday?", - "Tell me about todays meeting and how it relates to the email on Monday", - ] - - resp = asyncio.run(main(questions)) - - for r in resp: - print("q:", r["question"]) - #> q: what did I do on Monday? - print("c:", r["classification"]) - #> c: ['SUMMARY'] -``` - -If you follow what we've done is wrapped the client and proceeded to quickly use asyncio to classify a list of questions. This is a simple example of how you can use LangSmith to enhance the OpenAI client. You can use LangSmith to monitor and observe the client, and use `instructor` to add additional functionality to the client. - -To take a look at trace of this run check out this shareable [link](https://smith.langchain.com/public/eaae9f95-3779-4bbb-824d-97aa8a57a4e0/r). - -![](./img/langsmith.png) diff --git a/참고/instructor-main/docs/blog/posts/learn-async.md b/참고/instructor-main/docs/blog/posts/learn-async.md deleted file mode 100644 index 24d0904..0000000 --- a/참고/instructor-main/docs/blog/posts/learn-async.md +++ /dev/null @@ -1,265 +0,0 @@ ---- -authors: -- jxnl -categories: -- LLM Techniques -comments: true -date: 2023-11-13 -description: "Master Python asyncio.gather and asyncio.as_completed for efficient concurrent LLM processing with Instructor. Learn async programming patterns, rate limiting, and performance optimization for AI applications." -draft: false -slug: learn-async -tags: -- asyncio -- asyncio.gather -- asyncio.as_completed -- OpenAI -- Python -- data processing -- async programming -- concurrent processing -- LLM optimization ---- - -# Mastering Python asyncio.gather and asyncio.as_completed for LLM Processing - -Learn how to use Python's `asyncio.gather` and `asyncio.as_completed` for efficient concurrent processing of Large Language Models (LLMs) with Instructor. This comprehensive guide covers async programming patterns, rate limiting strategies, and performance optimization techniques. - - - -!!! notes "Complete Example Code" - - You can find the complete working example on [GitHub](https://github.com/jxnl/instructor/blob/main/examples/learn-async/run.py) - -## Understanding asyncio.gather vs asyncio.as_completed - -Python's `asyncio` library provides two powerful methods for concurrent execution: - -- **`asyncio.gather`**: Executes all tasks concurrently and returns results in the same order as input -- **`asyncio.as_completed`**: Returns results as they complete, regardless of input order - -Both methods significantly outperform sequential processing, but they serve different use cases. - -## Complete Setup: Async LLM Processing - -Here's a complete, self-contained example showing how to set up async processing with Instructor: - -```python -import instructor -from pydantic import BaseModel - -# Set up the async client with Instructor -client = instructor.from_provider("openai/gpt-5-nano", async_client=True) - - -class Person(BaseModel): - name: str - age: int - occupation: str - - -async def extract_person(text: str) -> Person: - """Extract person information from text using LLM.""" - return await client.create( - model="gpt-4o-mini", - response_model=Person, - messages=[{"role": "user", "content": f"Extract person info: {text}"}], - ) - - -# Sample dataset -dataset = [ - "John Smith is a 30-year-old software engineer", - "Sarah Johnson is a 25-year-old data scientist", - "Mike Davis is a 35-year-old product manager", - "Lisa Wilson is a 28-year-old UX designer", - "Tom Brown is a 32-year-old DevOps engineer", - "Emma Garcia is a 27-year-old frontend developer", - "David Lee is a 33-year-old backend developer", -] -``` - -## Method 1: Sequential Processing (Baseline) - -```python -async def sequential_processing() -> List[Person]: - """Process items one by one - slowest method.""" - start_time = time.time() - persons = [] - - for text in dataset: - person = await extract_person(text) - persons.append(person) - print(f"Processed: {person.name}") - - end_time = time.time() - print(f"Sequential processing took: {end_time - start_time:.2f} seconds") - return persons - - -# Run sequential processing -# persons = await sequential_processing() -``` - -## Method 2: asyncio.gather - Concurrent Processing - -```python -async def gather_processing() -> List[Person]: - """Process all items concurrently and return in order.""" - start_time = time.time() - - # Create tasks for all items - tasks = [extract_person(text) for text in dataset] - - # Execute all tasks concurrently - persons = await asyncio.gather(*tasks) - - end_time = time.time() - print(f"asyncio.gather took: {end_time - start_time:.2f} seconds") - - # Results maintain original order - for person in persons: - print(f"Processed: {person.name}") - - return persons - - -# Run gather processing -# persons = await gather_processing() -``` - -## Method 3: asyncio.as_completed - Streaming Results - -```python -async def as_completed_processing() -> List[Person]: - """Process items concurrently and handle results as they complete.""" - start_time = time.time() - persons = [] - - # Create tasks for all items - tasks = [extract_person(text) for text in dataset] - - # Process results as they complete - for task in asyncio.as_completed(tasks): - person = await task - persons.append(person) - print(f"Completed: {person.name}") - - end_time = time.time() - print(f"asyncio.as_completed took: {end_time - start_time:.2f} seconds") - return persons - - -# Run as_completed processing -# persons = await as_completed_processing() -``` - -## Method 4: Rate-Limited Processing with Semaphores - -```python -async def rate_limited_extract_person( - text: str, semaphore: asyncio.Semaphore -) -> Person: - """Extract person info with rate limiting.""" - async with semaphore: - return await extract_person(text) - - -async def rate_limited_gather(concurrency_limit: int = 3) -> List[Person]: - """Process items with controlled concurrency using asyncio.gather.""" - start_time = time.time() - - # Create semaphore to limit concurrent requests - semaphore = asyncio.Semaphore(concurrency_limit) - - # Create rate-limited tasks - tasks = [rate_limited_extract_person(text, semaphore) for text in dataset] - - # Execute with rate limiting - persons = await asyncio.gather(*tasks) - - end_time = time.time() - print( - f"Rate-limited gather (limit={concurrency_limit}) took: {end_time - start_time:.2f} seconds" - ) - return persons - - -async def rate_limited_as_completed(concurrency_limit: int = 3) -> List[Person]: - """Process items with controlled concurrency using asyncio.as_completed.""" - start_time = time.time() - persons = [] - - # Create semaphore to limit concurrent requests - semaphore = asyncio.Semaphore(concurrency_limit) - - # Create rate-limited tasks - tasks = [rate_limited_extract_person(text, semaphore) for text in dataset] - - # Process results as they complete - for task in asyncio.as_completed(tasks): - person = await task - persons.append(person) - print(f"Rate-limited completed: {person.name}") - - end_time = time.time() - print( - f"Rate-limited as_completed (limit={concurrency_limit}) took: {end_time - start_time:.2f} seconds" - ) - return persons - - -# Run rate-limited processing -# persons = await rate_limited_gather(concurrency_limit=2) -# persons = await rate_limited_as_completed(concurrency_limit=2) -``` - -## Performance Comparison - -Here are typical performance results when processing 7 items: - -| Method | Execution Time | Concurrency | Use Case | -|--------|---------------|-------------|----------| -| Sequential | 6.17 seconds | 1 | Baseline | -| asyncio.gather | 0.85 seconds | 7 | Fast processing, ordered results | -| asyncio.as_completed | 0.95 seconds | 7 | Streaming results | -| Rate-limited gather | 3.04 seconds | 2 | API-friendly | -| Rate-limited as_completed | 3.26 seconds | 2 | Streaming + rate limiting | - -## When to Use Each Method - -### Use asyncio.gather when: -- You need results in the same order as input -- All tasks must complete successfully -- You want the fastest possible execution -- Memory usage isn't a concern - -### Use asyncio.as_completed when: -- You want to process results as they arrive -- Order doesn't matter -- You're streaming data to clients -- You want to handle large datasets efficiently - -### Use rate limiting when: -- Working with API rate limits -- Being respectful to external services -- Managing resource consumption -- Building production applications - -## Key Takeaways - -1. **asyncio.gather** is fastest for ordered results -2. **asyncio.as_completed** is best for streaming and large datasets -3. **Rate limiting** is essential for production applications -4. **Error handling** should be implemented for robustness -5. **Monitoring** helps optimize performance - -## Related Resources - -- [Python asyncio Documentation](https://docs.python.org/3/library/asyncio.html) -- [Real Python Async IO Tutorial](https://realpython.com/async-io-python/) -- [Instructor Documentation](https://python.useinstructor.com) -- [OpenAI Async API Guide](https://platform.openai.com/docs/guides/async) - ---- - -**Next Steps**: Learn about [error handling patterns](../../concepts/error_handling.md) or explore [rate limiting with tenacity](../../concepts/retrying.md) for production applications. \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/llm-as-reranker.md b/참고/instructor-main/docs/blog/posts/llm-as-reranker.md deleted file mode 100644 index 939b8d8..0000000 --- a/참고/instructor-main/docs/blog/posts/llm-as-reranker.md +++ /dev/null @@ -1,197 +0,0 @@ ---- -authors: - - jxnl -categories: - - LLM - - Pydantic -comments: true -date: 2024-10-23 -description: Learn how to use Instructor and Pydantic to create an LLM-based reranker for improving search results relevance. -draft: false -tags: - - LLM - - Pydantic - - Instructor - - Search Relevance - - Reranking ---- - -# Building an LLM-based Reranker for your RAG pipeline - -Are you struggling with irrelevant search results in your Retrieval-Augmented Generation (RAG) pipeline? - -Imagine having a powerful tool that can intelligently reassess and reorder your search results, significantly improving their relevance to user queries. - -In this blog post, we'll show you how to create an LLM-based reranker using Instructor and Pydantic. This approach will: - -- Enhance the accuracy of your search results -- Leverage the power of large language models (LLMs) -- Utilize structured outputs for precise information retrieval - -By the end of this tutorial, you'll be able to implement a llm reranker to label your synthetic data for fine-tuning a traditional reranker, or to build out an evaluation pipeline for your RAG system. Let's dive in! - - - -## Setting Up the Environment - -First, let's set up our environment with the necessary imports: - -```python -import instructor - -client = instructor.from_provider("openai/gpt-5-nano") -``` - -We're using the `instructor` library, which integrates seamlessly with OpenAI's API and Pydantic for structured outputs. - -## Defining the Reranking Models - -We'll use Pydantic to define our `Label` and `RerankedResults` models that structure the output of our LLM: - -Notice that not only do I reference the chunk_id in the label class, I also asked a language model to use chain of thought. This is very useful for using models like 4o Mini or Claude, but not necessarily if we plan to use the `o1-mini` and `o1-preview` models. - -```python -class Label(BaseModel): - chunk_id: int = Field(description="The unique identifier of the text chunk") - chain_of_thought: str = Field( - description="The reasoning process used to evaluate the relevance" - ) - relevancy: int = Field( - description="Relevancy score from 0 to 10, where 10 is most relevant", - ge=0, - le=10, - ) - - -class RerankedResults(BaseModel): - labels: list[Label] = Field(description="List of labeled and ranked chunks") - - @field_validator("labels") - @classmethod - def model_validate(cls, v: list[Label]) -> list[Label]: - return sorted(v, key=lambda x: x.relevancy, reverse=True) -``` - -These models ensure that our LLM's output is structured and includes a list of labeled chunks with their relevancy scores. The `RerankedResults` model includes a validator that automatically sorts the labels by relevancy in descending order. - -## Creating the Reranker Function - -Next, we'll create a function that uses our LLM to rerank a list of text chunks based on their relevance to a query: - -```python -def rerank_results(query: str, chunks: list[dict]) -> RerankedResults: - return client.create( - model="gpt-4o-mini", - response_model=RerankedResults, - messages=[ - { - "role": "system", - "content": """ - You are an expert search result ranker. Your task is to evaluate the relevance of each text chunk to the given query and assign a relevancy score. - - For each chunk: - 1. Analyze its content in relation to the query. - 2. Provide a chain of thought explaining your reasoning. - 3. Assign a relevancy score from 0 to 10, where 10 is most relevant. - - Be objective and consistent in your evaluations. - """, - }, - { - "role": "user", - "content": """ - {{ query }} - - - {% for chunk in chunks %} - - {{ chunk.text }} - - {% endfor %} - - - Please provide a RerankedResults object with a Label for each chunk. - """, - }, - ], - context={"query": query, "chunks": chunks}, - ) -``` - -This function takes a query and a list of text chunks as input, sends them to the LLM with a predefined prompt, and returns a structured `RerankedResults` object. Thanks to instructor we can use jinja templating to inject the query and chunks into the prompt by passing in the `context` parameter. - -## Testing the Reranker - -To test our LLM-based reranker, we can create a sample query and a list of text chunks. Here's an example of how to use the reranker: - -```python -def main(): - query = "What are the health benefits of regular exercise?" - chunks = [ - { - "id": 0, - "text": "Regular exercise can improve cardiovascular health and reduce the risk of heart disease.", - }, - { - "id": 1, - "text": "The price of gym memberships varies widely depending on location and facilities.", - }, - { - "id": 2, - "text": "Exercise has been shown to boost mood and reduce symptoms of depression and anxiety.", - }, - { - "id": 3, - "text": "Proper nutrition is essential for maintaining a healthy lifestyle.", - }, - { - "id": 4, - "text": "Strength training can increase muscle mass and improve bone density, especially important as we age.", - }, - ] - - results = rerank_results(query, chunks) - - print("Reranked results:") - for label in results.labels: - print(f"Chunk {label.chunk_id} (Relevancy: {label.relevancy}):") - print(f"Text: {chunks[label.chunk_id]['text']}") - print(f"Reasoning: {label.chain_of_thought}") - print() - - -if __name__ == "__main__": - main() -``` - -This test demonstrates how the reranker evaluates and sorts the chunks based on their relevance to the query. The full implementation can be found in the `examples/reranker/run.py` file. - -If you want to extend this example, you could use the `rerank_results` function to label synthetic data for fine-tuning a traditional reranker, or to build out an evaluation pipeline for your RAG system. - -Moreover, we could also add validators to the `Label.chunk_id` field to ensure that the chunk_id is present in the `chunks` list. This might be useful if labels are `uuids` or complex strings and we want to ensure that the chunk_id is a valid index for the chunks list. - -heres an example - -```python -class Label(BaseModel): - chunk_id: int = Field(description="The unique identifier of the text chunk") - ... - - @field_validator("chunk_id") - @classmethod - def validate_chunk_id(cls, v: int, info: ValidationInfo) -> int: - context = info.context - chunks = context["chunks"] - if v not in [chunk["id"] for chunk in chunks]: - raise ValueError( - f"Chunk with id {v} not found, must be one of {[chunk['id'] for chunk in chunks]}" - ) - return v -``` - -This will automatically check that the `chunk_id` is present in the `chunks` list and raise a `ValueError` if it is not, where `context` is the context dictionary that we passed into the `rerank_results` function. - -## See Also -- [RAG and Beyond](rag-and-beyond.md) - Comprehensive RAG guide -- [Validation Fundamentals](validation-part1.md) - Validate ranking scores -- [Performance Monitoring](logfire.md) - Track reranking performance diff --git a/참고/instructor-main/docs/blog/posts/llms-txt-adoption.md b/참고/instructor-main/docs/blog/posts/llms-txt-adoption.md deleted file mode 100644 index 12d58d4..0000000 --- a/참고/instructor-main/docs/blog/posts/llms-txt-adoption.md +++ /dev/null @@ -1,89 +0,0 @@ ---- -authors: - - jxnl -categories: - - Announcements -comments: true -date: 2025-03-19 -description: - Instructor adopts llms.txt to make documentation more accessible to AI language models. -draft: false -slug: instructor-adopts-llms-txt -tags: - - Documentation - - AI - - LLMs - - Standards ---- - -# Instructor Adopts llms.txt: Making Documentation AI-Friendly - -We're excited to announce that Instructor now implements the llms.txt specification! You can now find our llms.txt file at [python.useinstructor.com/llms.txt](https://python.useinstructor.com/llms.txt). This adoption marks an important step in making our documentation more accessible to AI language models. - - - -## What is llms.txt? - -The llms.txt specification, [developed by Jeremy Howard and the Answer.AI team](https://github.com/AnswerDotAI/llms-txt), addresses a critical challenge in AI-documentation interaction: context windows are too small for most websites, and HTML pages with navigation, ads, and JavaScript are difficult for LLMs to process effectively. - -Think of llms.txt as robots.txt for AI language models - a standardized way to help AI systems understand and navigate your documentation. While robots.txt tells search engines what they can index, llms.txt helps AI models find and understand the most relevant information about your project. - -## Why Instructor Adopted llms.txt - -As a library focused on structured outputs from LLMs, it made perfect sense for us to implement this standard. Here's why: - -1. **Better AI Integration**: Our users often interact with Instructor through AI coding assistants. Having a llms.txt file helps these tools better understand our documentation. - -2. **Cleaner Documentation Access**: Instead of parsing our full HTML documentation, AI models can now access clean markdown versions of our docs. - -3. **Supporting the Standard**: We believe in the importance of standardizing how AI models interact with documentation. By adopting llms.txt early, we're helping establish best practices for AI-friendly documentation. - -## What This Means for Users - -If you're using AI coding assistants like GitHub Copilot, Claude, or Cursor with Instructor, you should notice: - -- More accurate code suggestions -- Better understanding of Instructor's features -- More relevant documentation references - -For example, when you ask an AI assistant about Instructor's features, it can now directly access our markdown documentation through the llms.txt file, rather than trying to parse our HTML documentation. - -## How It Works - -Our llms.txt file provides: - -- A concise overview of Instructor -- Links to key documentation in markdown format -- Important notes about usage and best practices -- References to example code and tutorials - -AI models can use this information to better understand: - -- Core concepts of Instructor -- How to use our key features -- Best practices for implementation -- Where to find detailed documentation - -## Implementing llms.txt - -The llms.txt specification is gaining adoption, and we encourage other Python libraries and frameworks to implement it. Here's how you can add llms.txt to your project: - -1. Create a `/llms.txt` file in your documentation root -2. Follow the [standard format](https://github.com/AnswerDotAI/llms-txt#format) -3. Include key information and markdown links -4. Test with various AI assistants - -## Looking Forward - -This is just the beginning. As more projects adopt llms.txt, we expect to see: - -- Better AI-assisted coding experiences -- More standardized documentation access -- Improved AI understanding of codebases -- Enhanced collaboration between humans and AI - -We're excited to be part of establishing this standard and look forward to seeing how it evolves. If you're interested in learning more about llms.txt or want to discuss its implementation, reach out to us on [GitHub](https://github.com/instructor-ai/instructor) or [Twitter](https://x.com/jxnl.co). - -For more details about the llms.txt specification, check out the [official repository](https://github.com/AnswerDotAI/llms-txt) and join the discussion about making documentation more AI-friendly. - -Happy coding! \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/llms-txt-support.md b/참고/instructor-main/docs/blog/posts/llms-txt-support.md deleted file mode 100644 index f5d6ec0..0000000 --- a/참고/instructor-main/docs/blog/posts/llms-txt-support.md +++ /dev/null @@ -1,45 +0,0 @@ ---- -authors: - - jxnl -categories: - - Announcements -comments: true -date: 2025-08-29 -description: - Instructor now automatically generates llms.txt files for better AI documentation access. -draft: false -slug: llms-txt-support -tags: - - Documentation - - AI ---- - -# Instructor Now Supports llms.txt - -We've added automatic `llms.txt` generation to Instructor's documentation using the [`mkdocs-llmstxt`](https://github.com/pawamoy/mkdocs-llmstxt) plugin. - - - -## What is llms.txt? - -The [`llms.txt` specification](https://github.com/AnswerDotAI/llms-txt) helps AI coding assistants access clean documentation without parsing complex HTML. Think "robots.txt for LLMs." - -## What This Means - -Your AI coding assistant (Copilot, Claude, Cursor) now gets better access to: -- Getting started guides -- Core concepts and patterns -- Provider integration docs - -This should result in more accurate suggestions and better understanding of Instructor's features. - -## Implementation - -We're using the `mkdocs-llmstxt` plugin to automatically generate our `llms.txt` from our existing markdown documentation. Every time we update our docs, the `llms.txt` file stays current automatically. - -No manual maintenance, always up-to-date. - -## Resources - -- [llms.txt Specification](https://github.com/AnswerDotAI/llms-txt) -- [mkdocs-llmstxt Plugin](https://github.com/pawamoy/mkdocs-llmstxt) \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/logfire.md b/참고/instructor-main/docs/blog/posts/logfire.md deleted file mode 100644 index 48f196d..0000000 --- a/참고/instructor-main/docs/blog/posts/logfire.md +++ /dev/null @@ -1,286 +0,0 @@ ---- -authors: -- ivanleomk -- jxnl -categories: -- LLM Observability -comments: true -date: 2024-05-01 -description: Explore Logfire, an observability platform to enhance application performance - tracking with Pydantic, Instructor, and OpenAI integration. -draft: false -slug: instructor-logfire -tags: -- Logfire -- Pydantic -- OpenAI -- Instructor -- LLM Observability ---- - -## Introduction - -Logfire is a new observability platform coming from the creators of Pydantic. It integrates almost seamlessly with many of your favourite libraries such as Pydantic, HTTPx and Instructor. In this article, we'll show you how to use Logfire with Instructor to gain visibility into the performance of your entire application. - -We'll walk through the following examples - -1. Classifying scam emails using Instructor -2. Performing simple validation using the `llm_validator` -3. Extracting data into a markdown table from an infographic with GPT4V - - - -As usual, all of the code that we refer to here is provided in [examples/logfire](https://www.github.com/jxnl/instructor/tree/main/examples/logfire) for you to use in your projects. - -- `classify.py`: Email Classification Example -- `image.py` : GPT4-V Example -- `validate.py` : `llm_validator` example - -??? info "Configure Logfire" - - Before starting this tutorial, make sure that you've registered for a [Logfire](https://logfire.pydantic.dev/) account. You'll also need to create a project to track these logs. - -We'll need to install our dependencies and configure logfire auth before proceeding so simply run the commands below. Logfire will handle the authentication and configuration of your project. - -```bash -pip install logfire openai instructor pydantic pandas tabulate -logfire auth -``` - -## Classification - -Now that we've got Logfire setup, let's see how we can get it to help us track a simple classification job. - -Logfire is dead simple to integrate - all it takes is 2 lines of code and we have it setup. - -```python -from openai import OpenAI -import instructor -import logfire - - -openai_client = OpenAI() -logfire.configure(pydantic_plugin=logfire.PydanticPlugin(record="all")) # (1)! -logfire.instrument_openai(openai_client) # (2)! -client = instructor.from_provider("openai/gpt-4o") -``` - -1. We add Pydantic logging using `logfire`. Note that depending on your use-case, you can configure what you want to log with Pydantic -2. We use their openai_integration to configure logging for our client before using instructor on it - -In this example, we'll be looking at classifying emails as either spam or not spam. To do so, we can define a simple Pydantic model as seen below. - -```python -import enum - - -class Labels(str, enum.Enum): - """Enumeration for single-label text classification.""" - - SPAM = "spam" - NOT_SPAM = "not_spam" - - -class SinglePrediction(BaseModel): - """ - Class for a single class label prediction. - """ - - class_label: Labels -``` - -We can then use this in a generic instructor function as seen below that simply asks the model to classify text and return it in the form of a `SinglePrediction` Pydantic object. - -Logfire can help us to log this entire function, and what's happening inside it, even down to the model validation level by using their `logfire.instrument` decorator. - -```python -@logfire.instrument("classification", extract_args=True) # (1)! -def classify(data: str) -> SinglePrediction: - """Perform single-label classification on the input text.""" - return client.create( - model="gpt-4o-mini", - response_model=SinglePrediction, - messages=[ - { - "role": "user", - "content": f"Classify the following text: {data}", - }, - ], - ) -``` - -1. Logfire allows us to use the `logfire.instrument` decorator and tag a function to a specific name. - -Let's see what happens when we run this against a list of different emails - -```python -emails = [ - "Hello there I'm a Nigerian prince and I want to give you money", - "Meeting with Thomas has been set at Friday next week", - "Here are some weekly product updates from our marketing team", -] - -for email in emails: - classify(email) -``` - -There are a few important things here that the logs immediately give us - -1. The duration that each individual portion of our code took to run -2. The payload that we sent over to OpenAI -3. The exact arguments and results that were passed to each individual portion of our code at each step - -![Logfire Classification](img/classification-logfire.png) - -## LLM Validators - -For our second example, we'll use the inbuilt `llm_validator` that instructor provides out of the box to validate that our statements don't contain unsafe content that we might not want to serve to users. Let's start by defining a simple Pydantic Model that can do so and configure our logfire integration. - -```python -from typing import Annotated -from pydantic import BaseModel -from pydantic.functional_validators import AfterValidator -from instructor import llm_validator -import logfire -import instructor -from openai import OpenAI - -openai_client = OpenAI() -logfire.configure(pydantic_plugin=logfire.PydanticPlugin(record="all")) -logfire.instrument_openai(openai_client) -client = instructor.from_provider("openai/gpt-4o") - - -class Statement(BaseModel): - message: Annotated[ - str, - AfterValidator( - llm_validator("Don't allow any objectionable content", client=client) - ), - ] -``` - -We can then test out our new validator with a few sample statements to see how our validator is working in practice. - -```python -messages = [ - "I think we should always treat violence as the best solution", - "There are some great pastries down the road at this bakery I know", -] - -for message in messages: - try: - Statement(message=message) - except ValidationError as e: - print(e) -``` - -With Logfire, we can capture the entirety of the validation process. As seen below, we have access to not only the original input data, but also the schema that was being used, the errors that were thrown and even the exact field that threw the error. - -![Logfire Validation](img/validation-logfire.png) - -## Vision Models - -For our last example, let's see how we can use Logfire to extract structured data from an image using GPT-4V with OpenAI. We'll be using a simple bar graph here and using `GPT4V` to extract the data from the image from statista below and convert it into a markdown format. - -![Reference Image](img/statista-image.jpeg) - -What we want is an output of the combined numbers as seen below - -| Country | Total Skier Visits (M) | -| :------------ | ---------------------: | -| United States | 55.5 | -| Austria | 43.6 | -| France | 40.7 | -| Japan | 26.6 | -| Italy | 22.3 | -| Switzerland | 22 | -| Canada | 18.5 | -| China | 17.9 | -| Sweden | 9.2 | -| Germany | 7 | - -This is relatively simple with Pydantic. What we need to do is to define a custom type which will handle the conversion process as seen below - -```python -from pydantic import BeforeValidator, InstanceOf, WithJsonSchema - - -def md_to_df(data: Any) -> Any: - # Convert markdown to DataFrame - if isinstance(data, str): - return ( - pd.read_csv( - StringIO(data), # Process data - sep="|", - index_col=1, - ) - .dropna(axis=1, how="all") - .iloc[1:] - .applymap(lambda x: x.strip()) - ) - return data - - -MarkdownDataFrame = Annotated[ - InstanceOf[pd.DataFrame], # (1)! - BeforeValidator(md_to_df), # (2)! - WithJsonSchema( # (3)! - { - "type": "string", - "description": "The markdown representation of the table, each one should be tidy, do not try to join tables that should be separate", - } - ), -] -``` - -1. We indicate that the type of this type should be a pandas dataframe -2. We run a validation step to ensure that we can convert the input into a valid pandas dataframe and return a new pandas Dataframe for our model to use -3. We then override the type of the schema so that when we pass it to OpenAI, it knows to generate a table in a markdown format. - -We can then use this in a normal instructor call - -```python -import instructor -import logfire - - -client = instructor.from_provider("openai/gpt-4o", mode=instructor.Mode.MD_JSON) -logfire.configure(pydantic_plugin=logfire.PydanticPlugin(record="all")) -logfire.instrument_openai(client._client) - - -@logfire.instrument("extract-table", extract_args=True) -def extract_table_from_image(url: str) -> Iterable[Table]: - return client.create( - model="gpt-4-vision-preview", - response_model=Iterable[Table], - max_tokens=1800, - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Extract out a table from the image. Only extract out the total number of skiiers.", - }, - {"type": "image_url", "image_url": {"url": url}}, - ], - } - ], - ) -``` - -We can then call it as seen below - -```python -url = "https://cdn.statcdn.com/Infographic/images/normal/16330.jpeg" -tables = extract_table_from_image(url) -for table in tables: - print(table.caption, end="\n") - print(table.dataframe.to_markdown()) -``` - -Logfire is able to capture the stack track of the entire call as seen below, profile each part of our application and most importantly capture the raw inputs of the OpenAI call alongside any potential errors. - -![Logfire Image](img/image-logfire.png) \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/lseg-market-surveillance.md b/참고/instructor-main/docs/blog/posts/lseg-market-surveillance.md deleted file mode 100644 index e2cf86c..0000000 --- a/참고/instructor-main/docs/blog/posts/lseg-market-surveillance.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -authors: -- jxnl -categories: -- Production -- Financial Services -comments: true -date: 2025-09-11 -description: London Stock Exchange Group uses Instructor in production for AI-powered market surveillance, achieving 100% precision in detecting price-sensitive news -draft: false -tags: -- Production -- Finance -- Amazon Bedrock -- Market Surveillance -- Anthropic ---- - -# London Stock Exchange Group Powers Market Surveillance with Instructor - -London Stock Exchange Group (LSEG) has deployed Instructor in production to power their AI-driven market surveillance system, demonstrating the library's capability in mission-critical financial applications. - - - -## Production Impact at Scale - -LSEG processes over £1 trillion of securities annually from 400 members, requiring sophisticated market abuse detection systems. Their new AI-powered "Surveillance Guide" uses Instructor to integrate with Anthropic's Claude Sonnet 3.5 model through Amazon Bedrock. - -## Remarkable Results - -The system achieved exceptional performance metrics: -- **100% precision** in identifying non-sensitive news -- **100% recall** for detecting price-sensitive content -- Automated analysis of 250,000+ regulatory news articles -- Significant reduction in manual analyst workload - -## Technical Architecture - -LSEG's implementation leverages Instructor's structured output capabilities in their technical stack: - -- **Instructor library**: Seamless integration with Claude Sonnet 3.5 -- **Amazon Bedrock**: Scalable foundation model infrastructure -- **Custom Python pipelines**: Data processing and analysis - -The system processes regulatory news through a two-step classification approach, using Instructor to ensure reliable, structured responses from the LLM for downstream analysis. - -## Why This Matters - -This production deployment showcases Instructor being used where accuracy and reliability are paramount - financial regulatory compliance. The system helps analysts efficiently review trades flagged for potential market abuse by automatically analyzing news sensitivity and market impact. - -As Charles Kellaway from LSEG noted, the solution transforms market surveillance operations by reducing manual review time while improving consistency in price-sensitivity assessment. - -## Learn More - -Read the full case study: [How London Stock Exchange Group is detecting market abuse with their AI-powered Surveillance Guide on Amazon Bedrock](https://aws.amazon.com/blogs/machine-learning/how-london-stock-exchange-group-is-detecting-market-abuse-with-their-ai-powered-surveillance-guide-on-amazon-bedrock/) - -Ready to build your own production-ready structured output applications? [Get started with Instructor](../../getting-started.md). diff --git a/참고/instructor-main/docs/blog/posts/matching-language.md b/참고/instructor-main/docs/blog/posts/matching-language.md deleted file mode 100644 index f7cce08..0000000 --- a/참고/instructor-main/docs/blog/posts/matching-language.md +++ /dev/null @@ -1,270 +0,0 @@ ---- -authors: -- jxnl -categories: -- Pydantic -comments: true -date: 2024-03-28 -description: Explore techniques to ensure language models generate summaries that - match the source text's language using Pydantic and langdetect. -draft: false -slug: matching-language-summaries -tags: -- multilingual summarization -- language detection -- Pydantic -- langdetect -- language models ---- - -# Matching Language in Multilingual Summarization Tasks - -When asking language models to summarize text, there's a risk that the generated summary ends up in English, even if the source text is in another language. This is likely due to the instructions being provided in English, biasing the model towards English output. - -In this post, we explore techniques to ensure the language of the generated summary matches the language of the source text. We leverage Pydantic for data validation and the `langdetect` library for language identification. - - - -## The Problem - -Consider the following example where we ask a language model to summarize text in various languages: - -```txt -Լեզվական մոդելները վերջին տարիներին դարձել են ավելի հարուստ եւ կատարյալ, հնարավորություն ընձեռելով ստեղծել սահուն եւ բնական տեքստեր, ինչպես նաեւ գերազանց արդյունքներ ցուցաբերել մեքենայական թարգմանության, հարցերի պատասխանման եւ ստեղծագործ տեքստերի ստեղծման նման տարբեր առաջադրանքներում։ Այս մոդելները մշակվում են հսկայական տեքստային տվյալների հիման վրա եւ կարող են բռնել բնական լեզվի կառուցվածքն ու նրբությունները՝ հեղափոխություն առաջացնելով համակարգիչների եւ մարդկանց միջեւ հաղորդակցության ոլորտում։ - ---- - -Mga modelo ng wika ay naging mas sopistikado sa nagdaang mga taon, na nagbibigay-daan sa pagbuo ng mga natural at madaling basahing teksto, at nagpapakita ng mahusay na pagganap sa iba't ibang gawain tulad ng awtomatikong pagsasalin, pagsagot sa mga tanong, at pagbuo ng malikhain na teksto. Ang mga modelo na ito ay sinanay sa napakalaking mga dataset ng teksto at kayang hulihin ang istruktura at mga nuances ng natural na wika. Ang mga pagpapabuti sa mga modelo ng wika ay maaaring magdulot ng rebolusyon sa komunikasyon sa pagitan ng mga computer at tao, at inaasahan ang higit pang pag-unlad sa hinaharap. - ---- - -Ngaahi motuʻa lea kuo nau hoko ʻo fakaʻofoʻofa ange ʻi he ngaahi taʻu fakamuimui ni, ʻo fakafaingofuaʻi e fakatupu ʻo e ngaahi konga tohi ʻoku lelei mo fakanatula pea ʻoku nau fakahaaʻi ʻa e ngaahi ola lelei ʻi he ngaahi ngāue kehekehe ʻo hangē ko e liliu fakaʻētita, tali fehuʻi, mo e fakatupu ʻo e konga tohi fakaʻatamai. Ko e ako ʻa e ngaahi motuʻa ni ʻi he ngaahi seti ʻo e fakamatala tohi lahi pea ʻoku nau malava ʻo puke ʻa e fakafuofua mo e ngaahi meʻa iiki ʻo e lea fakanatula. ʻE lava ke fakatupu ʻe he ngaahi fakaleleiʻi ki he ngaahi motuʻa lea ha liliu lahi ʻi he fetu'utaki ʻi he vahaʻa ʻo e ngaahi komipiuta mo e kakai, pea ʻoku ʻamanaki ʻe toe fakalakalaka ange ia ʻi he kahaʻu. -``` - -If we use a simple instructor prompt, even when we ask for the language to be correct, we oftentimes will get English instead. - -??? note "Expand to see documents examples" - - Լեզվական մոդելները վերջին տարիներին դարձել են ավելի հարուստ եւ կատարյալ, հնարավորություն ընձեռելով ստեղծել սահուն եւ բնական տեքստեր, ինչպես նաեւ գերազանց արդյունքներ ցուցաբերել մեքենայական թարգմանության, հարցերի պատասխանման եւ ստեղծագործ տեքստերի ստեղծման նման տարբեր առաջադրանքներում։ Այս մոդելները մշակվում են հսկայական տեքստային տվյալների հիման վրա եւ կարող են բռնել բնական լեզվի կառուցվածքն ու նրբությունները՝ հեղափոխություն առաջացնելով համակարգիչների եւ մարդկանց միջեւ հաղորդակցության ոլորտում։ - - --- - - Mga modelo ng wika ay naging mas sopistikado sa nagdaang mga taon, na nagbibigay-daan sa pagbuo ng mga natural at madaling basahing teksto, at nagpapakita ng mahusay na pagganap sa iba't ibang gawain tulad ng awtomatikong pagsasalin, pagsagot sa mga tanong, at pagbuo ng malikhain na teksto. Ang mga modelo na ito ay sinanay sa napakalaking mga dataset ng teksto at kayang hulihin ang istruktura at mga nuances ng natural na wika. Ang mga pagpapabuti sa mga modelo ng wika ay maaaring magdulot ng rebolusyon sa komunikasyon sa pagitan ng mga computer at tao, at inaasahan ang higit pang pag-unlad sa hinaharap. - - --- - - Ngaahi motuʻa lea kuo nau hoko ʻo fakaʻofoʻofa ange ʻi he ngaahi taʻu fakamuimui ni, ʻo fakafaingofuaʻi e fakatupu ʻo e ngaahi konga tohi ʻoku lelei mo fakanatula pea ʻoku nau fakahaaʻi ʻa e ngaahi ola lelei ʻi he ngaahi ngāue kehekehe ʻo hangē ko e liliu fakaʻētita, tali fehuʻi, mo e fakatupu ʻo e konga tohi fakaʻatamai. Ko e ako ʻa e ngaahi motuʻa ni ʻi he ngaahi seti ʻo e fakamatala tohi lahi pea ʻoku nau malava ʻo puke ʻa e fakafuofua mo e ngaahi meʻa iiki ʻo e lea fakanatula. ʻE lava ke fakatupu ʻe he ngaahi fakaleleiʻi ki he ngaahi motuʻa lea ha liliu lahi ʻi he fetu'utaki ʻi he vahaʻa ʻo e ngaahi komipiuta mo e kakai, pea ʻoku ʻamanaki ʻe toe fakalakalaka ange ia ʻi he kahaʻu. - - --- - - Dil modelleri son yıllarda daha da gelişti, akıcı ve doğal metinler üretmeyi mümkün kılıyor ve makine çevirisi, soru cevaplama ve yaratıcı metin oluşturma gibi çeşitli görevlerde mükemmel performans gösteriyor. Bu modeller, devasa metin veri setlerinde eğitilir ve doğal dilin yapısını ve nüanslarını yakalayabilir. Dil modellerindeki iyileştirmeler, bilgisayarlar ve insanlar arasındaki iletişimde devrim yaratabilir ve gelecekte daha da ilerleme bekleniyor. - - --- - - Mô hình ngôn ngữ đã trở nên tinh vi hơn trong những năm gần đây, cho phép tạo ra các văn bản trôi chảy và tự nhiên, đồng thời thể hiện hiệu suất xuất sắc trong các nhiệm vụ khác nhau như dịch máy, trả lời câu hỏi và tạo văn bản sáng tạo. Các mô hình này được huấn luyện trên các tập dữ liệu văn bản khổng lồ và có thể nắm bắt cấu trúc và sắc thái của ngôn ngữ tự nhiên. Những cải tiến trong mô hình ngôn ngữ có thể mang lại cuộc cách mạng trong giao tiếp giữa máy tính và con người, và người ta kỳ vọng sẽ có những tiến bộ hơn nữa trong tương lai. - - --- - - Les modèles de langage sont devenus de plus en plus sophistiqués ces dernières années, permettant de générer des textes fluides et naturels, et de performer dans une variété de tâches telles que la traduction automatique, la réponse aux questions et la génération de texte créatif. Entraînés sur d'immenses ensembles de données textuelles, ces modèles sont capables de capturer la structure et les nuances du langage naturel, ouvrant la voie à une révolution dans la communication entre les ordinateurs et les humains. - - --- - - 近年来,语言模型变得越来越复杂,能够生成流畅自然的文本,并在机器翻译、问答和创意文本生成等各种任务中表现出色。这些模型在海量文本数据集上训练,可以捕捉自然语言的结构和细微差别。语言模型的改进有望彻底改变计算机和人类之间的交流方式,未来有望实现更大的突破。 - - --- - - In den letzten Jahren sind Sprachmodelle immer ausgefeilter geworden und können flüssige, natürlich klingende Texte generieren und in verschiedenen Aufgaben wie maschineller Übersetzung, Beantwortung von Fragen und Generierung kreativer Texte hervorragende Leistungen erbringen. Diese Modelle werden auf riesigen Textdatensätzen trainiert und können die Struktur und Nuancen natürlicher Sprache erfassen, was zu einer Revolution in der Kommunikation zwischen Computern und Menschen führen könnte. - - --- - - पिछले कुछ वर्षों में भाषा मॉडल बहुत अधिक परिष्कृत हो गए हैं, जो प्राकृतिक और प्रवाहमय पाठ उत्पन्न कर सकते हैं, और मशीन अनुवाद, प्रश्नोत्तर, और रचनात्मक पाठ उत्पादन जैसे विभिन्न कार्यों में उत्कृष्ट प्रदर्शन कर सकते हैं। ये मॉडल विशाल पाठ डेटासेट पर प्रशिक्षित होते हैं और प्राकृतिक भाषा की संरचना और बारीकियों को समझ सकते हैं। भाषा मॉडल में सुधार कंप्यूटर और मानव के बीच संवाद में क्रांति ला सकता है, और भविष्य में और प्रगति की उम्मीद है। - - --- - - 近年、言語モデルは非常に洗練され、自然で流暢なテキストを生成できるようになり、機械翻訳、質問応答、クリエイティブなテキスト生成など、様々なタスクで優れたパフォーマンスを発揮しています。これらのモデルは膨大なテキストデータセットで学習され、自然言語の構造とニュアンスを捉えることができます。言語モデルの改善により、コンピューターと人間のコミュニケーションに革命が起こる可能性があり、将来のさらなる進歩が期待されています。 - - -In this example, we'll do something very simple, asking for the language to be correct. And generating a base model that only asks for a summary. To test we will use the library `langdetect` to detect the language of the text. To challenge us even more, we'll limit ourselves using 3.5 rather than 4 in order to use a 'dumber' model. - -```python -from pydantic import BaseModel, Field -from instructor import patch -from openai import AsyncOpenAI -from langdetect import detect - -docs = # To see the text, expand the notes above. - -# Patch the OpenAI client to enable response_model -client = patch(AsyncOpenAI()) - - -class GeneratedSummary(BaseModel): - summary: str - -async def summarize_text(text: str): - response = await client.create( - model="gpt-3.5-turbo", - response_model=GeneratedSummary, - messages=[ - { - "role": "system", - "content": "Generate a concise summary in the language of the article. ", - }, - { - "role": "user", - "content": f"Summarize the following text in a concise way:\n{text}", - }, - ], - ) # type: ignore - return response.summary, text - - -if __name__ == "__main__": - import asyncio - - async def main(): - results = await asyncio.gather(*[summarize_text(doc) for doc in docs]) - for summary, doc in results: - source_lang = detect(doc) - target_lang = detect(summary) - print( - f"Source: {source_lang}, Summary: {target_lang}, Match: {source_lang == target_lang}" - ) - - asyncio.run(main()) - """ - Source: et, Summary: en, Match: False - Source: tl, Summary: tl, Match: True - Source: sw, Summary: en, Match: False - Source: tr, Summary: tr, Match: True - Source: vi, Summary: en, Match: False - Source: fr, Summary: fr, Match: True - Source: zh-cn, Summary: en, Match: False - Source: de, Summary: de, Match: True - Source: hi, Summary: en, Match: False - Source: ja, Summary: en, Match: False - """ -``` - -In this example, you'll notice that not all the languages are matching. Many of them respond in English, and so we get pretty terrible results. Only 3 out of 9 passed! - -## Reiterating instructions - -A simple trick that I found to work very well is to add a language detection attribute before the summary. - -```python hl_lines="2" -class GeneratedSummary(BaseModel): - detected_language: str = Field( - description="The language code of the original article. The summary must be generated in this same language.", - ) - summary: str -``` - -Just by adding this single attribute, we end up getting 100% correctness on language matches. If you want to see for yourself, checkout the complete script below - -```python -from pydantic import BaseModel, Field -from instructor import patch -from openai import AsyncOpenAI -from langdetect import detect - -docs = map( - lambda x: x.strip(), - """ -Լեզվական մոդելները վերջին տարիներին դարձել են ավելի հարուստ եւ կատարյալ, հնարավորություն ընձեռելով ստեղծել սահուն եւ բնական տեքստեր, ինչպես նաեւ գերազանց արդյունքներ ցուցաբերել մեքենայական թարգմանության, հարցերի պատասխանման եւ ստեղծագործ տեքստերի ստեղծման նման տարբեր առաջադրանքներում։ Այս մոդելները մշակվում են հսկայական տեքստային տվյալների հիման վրա եւ կարող են բռնել բնական լեզվի կառուցվածքն ու նրբությունները՝ հեղափոխություն առաջացնելով համակարգիչների եւ մարդկանց միջեւ հաղորդակցության ոլորտում։ - ---- - -Mga modelo ng wika ay naging mas sopistikado sa nagdaang mga taon, na nagbibigay-daan sa pagbuo ng mga natural at madaling basahing teksto, at nagpapakita ng mahusay na pagganap sa iba't ibang gawain tulad ng awtomatikong pagsasalin, pagsagot sa mga tanong, at pagbuo ng malikhain na teksto. Ang mga modelo na ito ay sinanay sa napakalaking mga dataset ng teksto at kayang hulihin ang istruktura at mga nuances ng natural na wika. Ang mga pagpapabuti sa mga modelo ng wika ay maaaring magdulot ng rebolusyon sa komunikasyon sa pagitan ng mga computer at tao, at inaasahan ang higit pang pag-unlad sa hinaharap. - ---- - -Ngaahi motuʻa lea kuo nau hoko ʻo fakaʻofoʻofa ange ʻi he ngaahi taʻu fakamuimui ni, ʻo fakafaingofuaʻi e fakatupu ʻo e ngaahi konga tohi ʻoku lelei mo fakanatula pea ʻoku nau fakahaaʻi ʻa e ngaahi ola lelei ʻi he ngaahi ngāue kehekehe ʻo hangē ko e liliu fakaʻētita, tali fehuʻi, mo e fakatupu ʻo e konga tohi fakaʻatamai. Ko e ako ʻa e ngaahi motuʻa ni ʻi he ngaahi seti ʻo e fakamatala tohi lahi pea ʻoku nau malava ʻo puke ʻa e fakafuofua mo e ngaahi meʻa iiki ʻo e lea fakanatula. ʻE lava ke fakatupu ʻe he ngaahi fakaleleiʻi ki he ngaahi motuʻa lea ha liliu lahi ʻi he fetu'utaki ʻi he vahaʻa ʻo e ngaahi komipiuta mo e kakai, pea ʻoku ʻamanaki ʻe toe fakalakalaka ange ia ʻi he kahaʻu. - ---- - -Dil modelleri son yıllarda daha da gelişti, akıcı ve doğal metinler üretmeyi mümkün kılıyor ve makine çevirisi, soru cevaplama ve yaratıcı metin oluşturma gibi çeşitli görevlerde mükemmel performans gösteriyor. Bu modeller, devasa metin veri setlerinde eğitilir ve doğal dilin yapısını ve nüanslarını yakalayabilir. Dil modellerindeki iyileştirmeler, bilgisayarlar ve insanlar arasındaki iletişimde devrim yaratabilir ve gelecekte daha da ilerleme bekleniyor. - ---- - -Mô hình ngôn ngữ đã trở nên tinh vi hơn trong những năm gần đây, cho phép tạo ra các văn bản trôi chảy và tự nhiên, đồng thời thể hiện hiệu suất xuất sắc trong các nhiệm vụ khác nhau như dịch máy, trả lời câu hỏi và tạo văn bản sáng tạo. Các mô hình này được huấn luyện trên các tập dữ liệu văn bản khổng lồ và có thể nắm bắt cấu trúc và sắc thái của ngôn ngữ tự nhiên. Những cải tiến trong mô hình ngôn ngữ có thể mang lại cuộc cách mạng trong giao tiếp giữa máy tính và con người, và người ta kỳ vọng sẽ có những tiến bộ hơn nữa trong tương lai. - ---- - -Les modèles de langage sont devenus de plus en plus sophistiqués ces dernières années, permettant de générer des textes fluides et naturels, et de performer dans une variété de tâches telles que la traduction automatique, la réponse aux questions et la génération de texte créatif. Entraînés sur d'immenses ensembles de données textuelles, ces modèles sont capables de capturer la structure et les nuances du langage naturel, ouvrant la voie à une révolution dans la communication entre les ordinateurs et les humains. - ---- - -近年来,语言模型变得越来越复杂,能够生成流畅自然的文本,并在机器翻译、问答和创意文本生成等各种任务中表现出色。这些模型在海量文本数据集上训练,可以捕捉自然语言的结构和细微差别。语言模型的改进有望彻底改变计算机和人类之间的交流方式,未来有望实现更大的突破。 - ---- - -In den letzten Jahren sind Sprachmodelle immer ausgefeilter geworden und können flüssige, natürlich klingende Texte generieren und in verschiedenen Aufgaben wie maschineller Übersetzung, Beantwortung von Fragen und Generierung kreativer Texte hervorragende Leistungen erbringen. Diese Modelle werden auf riesigen Textdatensätzen trainiert und können die Struktur und Nuancen natürlicher Sprache erfassen, was zu einer Revolution in der Kommunikation zwischen Computern und Menschen führen könnte. - ---- - -पिछले कुछ वर्षों में भाषा मॉडल बहुत अधिक परिष्कृत हो गए हैं, जो प्राकृतिक और प्रवाहमय पाठ उत्पन्न कर सकते हैं, और मशीन अनुवाद, प्रश्नोत्तर, और रचनात्मक पाठ उत्पादन जैसे विभिन्न कार्यों में उत्कृष्ट प्रदर्शन कर सकते हैं। ये मॉडल विशाल पाठ डेटासेट पर प्रशिक्षित होते हैं और प्राकृतिक भाषा की संरचना और बारीकियों को समझ सकते हैं। भाषा मॉडल में सुधार कंप्यूटर और मानव के बीच संवाद में क्रांति ला सकता है, और भविष्य में और प्रगति की उम्मीद है। - ---- - -近年、言語モデルは非常に洗練され、自然で流暢なテキストを生成できるようになり、機械翻訳、質問応答、クリエイティブなテキスト生成など、様々なタスクで優れたパフォーマンスを発揮しています。これらのモデルは膨大なテキストデータセットで学習され、自然言語の構造とニュアンスを捉えることができます。言語モデルの改善により、コンピューターと人間のコミュニケーションに革命が起こる可能性があり、将来のさらなる進歩が期待されています。 -""".split( - "---" - ), -) - -# Patch the OpenAI client to enable response_model -client = patch(AsyncOpenAI()) - - -class GeneratedSummary(BaseModel): - detected_language: str = Field( - description="The language code of the original article. The summary must be generated in this same language.", - ) - summary: str - - -async def summarize_text(text: str): - response = await client.create( - model="gpt-3.5-turbo", - response_model=GeneratedSummary, - messages=[ - { - "role": "system", - "content": "Generate a concise summary in the language of the article. ", - }, - { - "role": "user", - "content": f"Summarize the following text in a concise way:\n{text}", - }, - ], - ) # type: ignore - return response.summary, text - - -if __name__ == "__main__": - import asyncio - - async def main(): - results = await asyncio.gather(*[summarize_text(doc) for doc in docs]) - for summary, doc in results: - source_lang = detect(doc) - target_lang = detect(summary) - print( - f"Source: {source_lang}, Summary: {target_lang}, Match: {source_lang == target_lang}" - ) - - asyncio.run(main()) - """ - Source: et, Summary: et, Match: True - Source: tl, Summary: tl, Match: True - Source: sw, Summary: sw, Match: True - Source: tr, Summary: tr, Match: True - Source: vi, Summary: vi, Match: True - Source: fr, Summary: fr, Match: True - Source: zh-cn, Summary: zh-cn, Match: True - Source: de, Summary: de, Match: True - Source: hi, Summary: hi, Match: True - Source: ja, Summary: ja, Match: True - """ -``` \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/migrating-to-uv.md b/참고/instructor-main/docs/blog/posts/migrating-to-uv.md deleted file mode 100644 index e55d4e7..0000000 --- a/참고/instructor-main/docs/blog/posts/migrating-to-uv.md +++ /dev/null @@ -1,235 +0,0 @@ ---- -authors: - - ivanleomk -categories: - - UV -comments: true -date: 2024-12-26 -description: How we migrated from poetry to uv -draft: false -tags: - - Migrations ---- - -## Why we migrated to uv - -We recently migrated to uv from poetry because we wanted to benefit from it's many features such as - -- Easier dependency management with automatic caching built in -- Significantly faster CI/CD compared to poetry, especially when we use the `caching` functionality provided by the Astral team -- Cargo-style lockfile that makes it easier to adopt new PEP features as they come out - -We took around 1-2 days to handle the migration and we're happy with the results. On average, for CI/CD, we've seen a huge speed up for our jobs. - -Here are some timings for jobs that I took from our CI/CD runs. - -In general I'd say that we saw a ~3x speedup with approximately 67% reduction in time needed for the jobs once we implemented caching for the individual `uv` github actions. - - - -| Job | Time (Poetry) | Time (UV) | -| ---------------- | --------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | -| Ruff Formatting | [1m16s](https://github.com/instructor-ai/instructor/actions/runs/12386936314) | [28s](https://github.com/instructor-ai/instructor/actions/runs/12501982235) (-63%) | -| Type checking | [3m3s](https://github.com/instructor-ai/instructor/actions/runs/12488572568) | [39s](https://github.com/instructor-ai/instructor/actions/runs/12501974285) (-79%) | -| Test Python 3.9 | [1m21s](https://github.com/instructor-ai/instructor/actions/runs/12251767751/job/34177033359) | [32s](https://github.com/instructor-ai/instructor/actions/runs/12501974279/job/34880278051) (-61%) | -| Test Python 3.10 | [1m32s](https://github.com/instructor-ai/instructor/actions/runs/12251767751/job/34177033359) | [33s](https://github.com/instructor-ai/instructor/actions/runs/12501974279/job/34880278299) (-64%) | -| Test Python 3.11 | [3m19](https://github.com/instructor-ai/instructor/actions/runs/12251767751/job/34177034094) | [2m48s](https://github.com/instructor-ai/instructor/actions/runs/12501974279/job/34880278480) (-16%) | - -- Note that for 3.11 I subtracted 1m12 from the time because we added ~60 more tests for gemini so to make it a fair comparison I subtracted the time it took to run the gemini tests. - -Most of our heavier jobs like the `Test Python` jobs are running multiple LLM calls in parallel and so the caching speedups of UV have some reduced benefit there. - -## How we migrated - -The first thing we did was to use an automated tool to convert our poetry lockfile to a uv compatible lockfile. For this, I followed [this thread](https://x.com/tiangolo/status/1839686030007361803) by Sebastian Ramirez on how to do the conversions. - -**Step 1** : Use `uv` to run a `pdm` which will migrate your pyproject.toml and make sure to remove all of the `tool.poetry` sections. You can see the initial `pyproject.toml` [here](https://github.com/instructor-ai/instructor/blob/ad046fbca335b9133a704bed1900cda846caaf7c/pyproject.toml). - -``` -uvx pdm import pyproject.toml -``` - -Note that since you're using `uv`, make sure to also delete the `pdm` sections too and your optional groups - -```toml -# dependency versions for extras -fastapi = { version = ">=0.109.2,<0.116.0", optional = true } -redis = { version = "^5.0.1", optional = true } -diskcache = { version = "^5.6.3", optional = true } -... - - -[tool.poetry.extras] -anthropic = ["anthropic", "xmltodict"] -groq = ["groq"] -cohere = ["cohere"] -... - - -[tool.pdm.build] -includes = ["instructor"] -[build-system] -requires = ["pdm-backend"] -build-backend = "pdm.backend" -``` - -**Step 2** : Once you've done so, since you're no longer using `poetry`, you need to update the build system. If you just delete it, you'll end up using `setuptools` by default and that will throw an error if you've declared your license using `license = {text = "MIT"}`. So you need to add the following to your `pyproject.toml`. - -This is documented in this UV issue [here](https://github.com/astral-sh/uv/issues/9513) which documents a bug with setuptools not being able to handle Metadata 2.4 keys and so you need to use `hatchling` as your build backend. - -```toml -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" -``` - -**Step 3** : Once you've done so, run uv sync to generate your `uv.lock` file to make sure you don't have any dependency issues. - -### New Commands to know - -Now that we migrated over from `poetry` to `uv`, there are a few new commands that you'll need to use. - -1. `uv sync --all-extras --group `: This should install all the dependencies for the project using `uv`, make sure to install the specific dependencies that you'd like to install. If you're writing docs for instance, you would run `uv sync --all-extras --group docs` - -2. `uv run ` : This runs the specific command using the virtual environment you've created. When running our CI pipeline, we use this to ensure we're using the right environment for our commands. - -## Migrating Your Workflows - -We had a few workflows that were using `poetry` and so we needed to update them to use `uv` instead. As seen below there are a few main changes you'll need to make to your relevant workflow - -```yaml -name: Test -on: - pull_request: - push: - branches: - - main - -jobs: - release: - runs-on: ubuntu-latest - - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11"] - - steps: - - uses: actions/checkout@v2 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} # (1)! - - - name: Cache Poetry virtualenv - uses: actions/cache@v2 - with: - path: ~/.cache/pypoetry/virtualenvs - key: ${{ runner.os }}-poetry-${{ hashFiles('**/poetry.lock') }} - restore-keys: | - ${{ runner.os }}-poetry- - - - name: Install Poetry - uses: snok/install-poetry@v1.3.1 # (2)! - - - name: Install dependencies - run: poetry install --with dev,anthropic # (3)! - - - name: Run tests - if: matrix.python-version != '3.11' - run: poetry run pytest tests/ -k 'not llm and not openai and not gemini and not anthropic and not cohere and not vertexai' && poetry run pytest tests/llm/test_cohere - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - - - name: Run Gemini Tests - run: poetry run pytest tests/llm/test_gemini # (4)! - env: - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - - - name: Generate coverage report - if: matrix.python-version == '3.11' - run: | - poetry run coverage run -m pytest tests/ -k "not docs and not anthropic and not gemini and not cohere and not vertexai and not fireworks" - poetry run coverage report - poetry run coverage html - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} -``` - -1. We switched over to using `uv` to install python - -2. We switch over to using astral's `astral-sh/setup-uv@v4` action to install `uv` - -3. Using `uv sync` was significantly faster than poetry install and with the cache I imagine it was even faster - -4. Instead of using `poetry run`, we use `uv run` which will start up the python virtual environment with the deps and then run the command you pass in. - -We then modified the workflow to the following yml config - -```yaml -name: Test -on: - pull_request: - push: - branches: - - main - -jobs: - release: - runs-on: ubuntu-latest - - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11"] - - steps: - - uses: actions/checkout@v2 - - name: Install uv - uses: astral-sh/setup-uv@v4 - with: - enable-cache: true # (1)! - - - name: Set up Python - run: uv python install ${{ matrix.python-version }} - - - name: Install the project - run: uv sync --all-extras - - name: Run tests - if: matrix.python-version != '3.11' - run: uv run pytest tests/ -k 'not llm and not openai and not gemini and not anthropic and not cohere and not vertexai' # (2)! - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }} - - - name: Run Gemini Tests - if: matrix.python-version == '3.11' - run: uv run pytest tests/llm/test_gemini - env: - GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - - - name: Generate coverage report - if: matrix.python-version == '3.11' - run: | - uv run coverage run -m pytest tests/ -k "not docs and not anthropic and not gemini and not cohere and not vertexai and not fireworks" - uv run coverage report - uv run coverage html - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} -``` - -1. Don't forget to enable the cache so that your jobs are faster - -2. Using `uv run` here is important because if you just run `pytest` it won't run the tests in your virtual environment causing them to fail. - -And that was basically it! Most of the migration work was really trying to figure out what was causing the tests to fail and then slowly fixing them. We were able to easily upgrade many of our existing dependencies and make sure that everything was working as expected. - -We also just did our first release with uv and it was a success! - -## Conclusion - -We're happy with the results and we're glad to have migrated to uv. It's been a smooth transition and we've been able to see a significant speedup in our CI/CD jobs. We're looking forward to continue using uv moving forward diff --git a/참고/instructor-main/docs/blog/posts/mkdocs-llmstxt-plugin-integration.md b/참고/instructor-main/docs/blog/posts/mkdocs-llmstxt-plugin-integration.md deleted file mode 100644 index b7a9bcf..0000000 --- a/참고/instructor-main/docs/blog/posts/mkdocs-llmstxt-plugin-integration.md +++ /dev/null @@ -1,133 +0,0 @@ ---- -authors: - - jxnl -categories: - - Technical - - Documentation -comments: true -date: 2025-08-29 -description: - Deep dive into how we integrated the mkdocs-llmstxt plugin to automatically generate llms.txt files for better AI documentation consumption. -draft: false -slug: mkdocs-llmstxt-plugin-integration -tags: - - MkDocs - - Plugins - - Documentation - - AI - - Automation ---- - -# Automating llms.txt Generation with mkdocs-llmstxt Plugin - -Today we integrated the `mkdocs-llmstxt` plugin into Instructor's documentation pipeline. This powerful plugin automatically generates `llms.txt` files from our MkDocs documentation, making our comprehensive guides instantly accessible to AI language models. - - - -## About the mkdocs-llmstxt Plugin - -The [`mkdocs-llmstxt` plugin](https://github.com/pawamoy/mkdocs-llmstxt) by Timothée Mazzucotelli is a brilliant solution to a common problem: how do you keep an `llms.txt` file synchronized with your evolving documentation? - -### Key Features - -**Automatic Generation**: The plugin generates `llms.txt` files directly from your MkDocs source files during the build process. No manual maintenance required. - -**Flexible Section Control**: You can specify exactly which parts of your documentation to include: - -```yaml -plugins: - - llmstxt: - sections: - Getting Started: - - index.md: Introduction to structured outputs - - installation.md: Setup instructions - Core Concepts: - - concepts/*.md -``` - -**Clean Markdown Conversion**: The plugin converts your documentation to clean, LLM-friendly markdown format, removing HTML artifacts and navigation elements. - -**Customizable Descriptions**: You can provide both short and long descriptions of your project, giving AI models the context they need. - -## Our Implementation - -Here's how we configured the plugin for Instructor: - -```yaml -plugins: - - llmstxt: - markdown_description: > - Instructor is a Python library that makes it easy to work with structured outputs - from large language models (LLMs). Built on top of Pydantic, it provides a simple, - type-safe way to extract structured data from LLM responses across multiple providers - including OpenAI, Anthropic, Google, and many others. - sections: - Getting Started: - - index.md: Introduction to structured outputs with LLMs - - getting-started.md: Quick start guide - - installation.md: Installation instructions - Core Concepts: - - concepts/*.md - Integrations: - - integrations/*.md -``` - -### Why These Sections? - -We carefully selected these sections because they provide AI models with the essential information needed to understand and use Instructor: - -- **Getting Started**: Core concepts and installation -- **Core Concepts**: Deep dive into features like validation, streaming, and patterns -- **Integrations**: Provider-specific guidance for OpenAI, Anthropic, Google, and others - -## Technical Benefits - -### Build Integration - -The plugin seamlessly integrates into our existing MkDocs build pipeline. Every time we deploy documentation updates, the `llms.txt` file is automatically regenerated with the latest content. - -### Content Freshness - -Unlike manually maintained `llms.txt` files, our generated version is always up-to-date. When we add new integration guides or update existing concepts, the changes are automatically reflected. - -### Glob Pattern Support - -The plugin supports glob patterns like `concepts/*.md`, making it easy to include entire directories without manually listing each file. - -## Plugin Architecture - -The `mkdocs-llmstxt` plugin works by: - -1. **Parsing Configuration**: Reading your `sections` configuration during the MkDocs build -2. **File Processing**: Converting specified markdown files to clean, LLM-friendly format -3. **Content Assembly**: Combining sections with metadata into the standard llms.txt format -4. **Output Generation**: Writing the final `llms.txt` file to your site root - -## Installation and Setup - -Adding the plugin to your own MkDocs project is straightforward: - -```bash -pip install mkdocs-llmstxt -``` - -Then add it to your `mkdocs.yml`: - -```yaml -site_url: https://your-site.com/ # Required for the plugin - -plugins: - - llmstxt: - markdown_description: Description of your project - sections: - Documentation: - - docs/*.md -``` - -## Resources - -- [mkdocs-llmstxt Plugin](https://github.com/pawamoy/mkdocs-llmstxt) -- [llms.txt Specification](https://github.com/AnswerDotAI/llms-txt) -- [Instructor Documentation](https://python.useinstructor.com/) - -Special thanks to Timothée Mazzucotelli for creating this excellent plugin! diff --git a/참고/instructor-main/docs/blog/posts/multimodal-gemini.md b/참고/instructor-main/docs/blog/posts/multimodal-gemini.md deleted file mode 100644 index 85f9bcf..0000000 --- a/참고/instructor-main/docs/blog/posts/multimodal-gemini.md +++ /dev/null @@ -1,223 +0,0 @@ ---- -authors: - - ivanleomk -categories: - - Gemini - - Multimodal -comments: true -date: 2024-10-23 -description: Learn how to use Google's Gemini model for multimodal structured extraction of YouTube videos, extracting structured recommendations for tourist destinations. -draft: false -tags: - - Gemini - - Multimodal AI - - Travel Recommendations - - Pydantic - - Python ---- - -# Structured Outputs with Multimodal Gemini - -In this post, we'll explore how to use Google's Gemini model with Instructor to analyze [travel videos](https://www.youtube.com/watch?v=_R8yhW_H9NQ) and extract structured recommendations. This powerful combination allows us to process multimodal inputs (video) and generate structured outputs using Pydantic models. This post was done in collaboration with [Kino.ai](https://kino.ai), a company that uses instructor to do structured extraction from multimodal inputs to improve search for film makers. - -## Setting Up the Environment - -First, let's set up our environment with the necessary libraries: - -```python -``` - - - -## Defining Our Data Models - -We'll use Pydantic to define our data models for tourist destinations and recommendations: - -```python -class TouristDestination(BaseModel): - name: str - description: str - location: str - - -class Recommendations(BaseModel): - chain_of_thought: str - description: str - destinations: list[TouristDestination] -``` - -## Initializing the Gemini Client - -Next, we'll set up our Gemini client using Instructor: - -```python -client = instructor.from_provider("google/gemini-2.5-flash") -) -``` - -## Uploading and Processing the Video - -To analyze a video, we first need to upload it: - -```python -file = genai.upload_file("./takayama.mp4") -``` - -Then, we can process the video and extract recommendations: - -```python -resp = client.create( - messages=[ - { - "role": "user", - "content": ["What places do they recommend in this video?", file], - } - ], - response_model=Recommendations, -) - -print(resp) -``` - -??? note "Expand to see Raw Results" - - ```python - Recomendations( - chain_of_thought='The video recommends visiting Takayama city, in the Hida Region, Gifu Prefecture. The - video suggests visiting the Miyagawa Morning Market, to try the Sarubobo good luck charms, and to enjoy the - cookie cup espresso, made by Koma Coffee. Then, the video suggests visiting a traditional Japanese Cafe, - called Kissako Katsure, and try their matcha and sweets. Afterwards, the video suggests to visit the Sanmachi - Historic District, where you can find local crafts and delicious foods. The video recommends trying Hida Wagyu - beef, at the Kin no Kotte Ushi shop, or to have a sit-down meal at the Kitchen Hida. Finally, the video - recommends visiting Shirakawa-go, a World Heritage Site in Gifu Prefecture.', - description='This video recommends a number of places to visit in Takayama city, in the Hida Region, Gifu - Prefecture. It shows some of the local street food and highlights some of the unique shops and restaurants in - the area.', - destinations=[ - TouristDestination( - name='Takayama', - description='Takayama is a city at the base of the Japan Alps, located in the Hida Region of - Gifu.', - location='Hida Region, Gifu Prefecture' - ), - TouristDestination( - name='Miyagawa Morning Market', - description="The Miyagawa Morning Market, or the Miyagawa Asai-chi in Japanese, is a market that - has existed officially since the Edo Period, more than 100 years ago. It's open every single day, rain or - shine, from 7am to noon.", - location='Hida Takayama' - ), - TouristDestination( - name='Nakaya - Handmade Hida Sarubobo', - description='The Nakaya shop sells handcrafted Sarubobo good luck charms.', - location='Hida Takayama' - ), - TouristDestination( - name='Koma Coffee', - description="Koma Coffee is a shop that has been in business for about 50 or 60 years, and they - serve coffee in a cookie cup. They've been serving coffee for about 10 years.", - location='Hida Takayama' - ), - TouristDestination( - name='Kissako Katsure', - description='Kissako Katsure is a traditional Japanese style cafe, called Kissako, and the name - means would you like to have some tea. They have a variety of teas and sweets.', - location='Hida Takayama' - ), - TouristDestination( - name='Sanmachi Historic District', - description='Sanmachi Dori is a Historic Merchant District in Takayama, all of the buildings here - have been preserved to look as they did in the Edo Period.', - location='Hida Takayama' - ), - TouristDestination( - name='Suwa Orchard', - description='The Suwa Orchard has been in business for more than 50 years.', - location='Hida Takayama' - ), - TouristDestination( - name='Kitchen HIDA', - description='Kitchen HIDA is a restaurant with a 50 year history, known for their Hida Beef dishes - and for using a lot of local ingredients.', - location='Hida Takayama' - ), - TouristDestination( - name='Kin no Kotte Ushi', - description='Kin no Kotte Ushi is a shop known for selling Beef Sushi, especially Hida Wagyu Beef - Sushi. Their sushi is medium rare.', - location='Hida Takayama' - ), - TouristDestination( - name='Shirakawa-go', - description='Shirakawa-go is a World Heritage Site in Gifu Prefecture.', - location='Gifu Prefecture' - ) - ] - ) - ``` - -The Gemini model analyzes the video and provides structured recommendations. Here's a summary of the extracted information: - -1. **Takayama City**: The main destination, located in the Hida Region of Gifu Prefecture. -2. **Miyagawa Morning Market**: A historic market open daily from 7am to noon. -3. **Nakaya Shop**: Sells handcrafted Sarubobo good luck charms. -4. **Koma Coffee**: A 50-60 year old shop famous for serving coffee in cookie cups. -5. **Kissako Katsure**: A traditional Japanese cafe offering various teas and sweets. -6. **Sanmachi Historic District**: A preserved merchant district from the Edo Period. -7. **Suwa Orchard**: A 50+ year old orchard business. -8. **Kitchen HIDA**: A restaurant with a 50-year history, known for Hida Beef dishes. -9. **Kin no Kotte Ushi**: A shop specializing in Hida Wagyu Beef Sushi. -10. **Shirakawa-go**: A World Heritage Site in Gifu Prefecture. - -## Limitations, Challenges, and Future Directions - -While the current approach demonstrates the power of multimodal AI for video analysis, there are several limitations and challenges to consider: - -1. **Lack of Temporal Information**: Our current method extracts overall recommendations but doesn't provide timestamps for specific mentions. This limits the ability to link recommendations to exact moments in the video. - -2. **Speaker Diarization**: The model doesn't distinguish between different speakers in the video. Implementing speaker diarization could provide valuable context about who is making specific recommendations. - -3. **Content Density**: Longer or more complex videos might overwhelm the model, potentially leading to missed information or less accurate extractions. - -### Future Explorations - -To address these limitations and expand the capabilities of our video analysis system, here are some promising areas to explore: - -1. **Timestamp Extraction**: Enhance the model to provide timestamps for each recommendation or point of interest mentioned in the video. This could be achieved by: - - ```python - class TimestampedRecommendation(BaseModel): - timestamp: str - timestamp_format: Literal["HH:MM", "HH:MM:SS"] # Helps with parsing - recommendation: str - - - class EnhancedRecommendations(BaseModel): - destinations: list[TouristDestination] - timestamped_mentions: list[TimestampedRecommendation] - ``` - -2. **Speaker Diarization**: Implement speaker recognition to attribute recommendations to specific individuals. This could be particularly useful for videos featuring multiple hosts or interviewees. - -3. **Segment-based Analysis**: Process longer videos in segments to maintain accuracy and capture all relevant information. This approach could involve: - - - Splitting the video into smaller chunks - - Analyzing each chunk separately - - Aggregating and deduplicating results - -4. **Multi-language Support**: Extend the model's capabilities to accurately analyze videos in various languages and capture culturally specific recommendations. - -5. **Visual Element Analysis**: Enhance the model to recognize and describe visual elements like landmarks, food dishes, or activities shown in the video, even if not explicitly mentioned in the audio. - -6. **Sentiment Analysis**: Incorporate sentiment analysis to gauge the speaker's enthusiasm or reservations about specific recommendations. - -By addressing these challenges and exploring these new directions, we can create a more comprehensive and nuanced video analysis system, opening up even more possibilities for applications in travel, education, and beyond. - -## Related Documentation -- [Multimodal Concepts](../../concepts/multimodal.md) - Working with images, video, and audio -- [Google Integration](../../integrations/google.md) - Complete Gemini setup guide - -## See Also -- [OpenAI Multimodal](openai-multimodal.md) - Compare multimodal approaches -- [Anthropic Structured Output](structured-output-anthropic.md) - Alternative provider -- [Chat with PDFs using Gemini](chat-with-your-pdf-with-gemini.md) - Practical PDF processing diff --git a/참고/instructor-main/docs/blog/posts/native_caching.md b/참고/instructor-main/docs/blog/posts/native_caching.md deleted file mode 100644 index 73ef1b9..0000000 --- a/참고/instructor-main/docs/blog/posts/native_caching.md +++ /dev/null @@ -1,276 +0,0 @@ ---- -authors: -- jxnl -categories: -- Performance Optimization -- Cost Reduction -- API Efficiency -- Python Development -comments: true -date: 2025-01-08 -description: Instructor v1.9.1 introduces native caching support for all providers. Learn how to drastically reduce API costs and improve response times with built-in cache adapters. -draft: false -slug: native-caching-v1-9-1 -tags: -- Python -- Caching -- Performance Optimization -- API Cost Optimization -- LLM Applications -- Production Scaling -- from_provider ---- - -# Native Caching in Instructor v1.9.1: Zero-Configuration Performance Boost - -> **New in v1.9.1**: Instructor now ships with built-in caching support for all providers. Simply pass a cache adapter when creating your client to dramatically reduce API costs and improve response times. - -Starting with Instructor v1.9.1, we've introduced native caching support that makes optimization effortless. Instead of implementing complex caching decorators or wrapper functions, you can now pass a cache adapter directly to `from_provider()` and automatically cache all your structured LLM calls. - -## The Game Changer: Built-in Caching - -Before v1.9.1, caching required custom decorators and manual implementation. Now, it's as simple as: - -```python -from instructor import from_provider -from instructor.cache import AutoCache - -# Works with any provider - caching flows through automatically -client = from_provider("openai/gpt-4o", cache=AutoCache(maxsize=1000)) - -# Your normal calls are now cached automatically -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -first = client.create( - messages=[{"role": "user", "content": "Extract: John is 25"}], response_model=User -) - -second = client.create( - messages=[{"role": "user", "content": "Extract: John is 25"}], response_model=User -) - -# second call was served from cache - same result, zero cost! -assert first.name == second.name -``` - -## Universal Provider Support - -The beauty of native caching is that it works with **every provider** through the same simple API: - -```python -from instructor.cache import AutoCache, DiskCache - -# Works with OpenAI -openai_client = from_provider("openai/gpt-5-nano", cache=AutoCache()) - -# Works with Anthropic -anthropic_client = from_provider("anthropic/claude-3-haiku", cache=AutoCache()) - -# Works with Google -google_client = from_provider("google/gemini-pro", cache=DiskCache()) - -# Works with any provider in the ecosystem -groq_client = from_provider("groq/llama-3.1-8b", cache=AutoCache()) -``` - -No provider-specific configuration needed. The cache parameter flows through `**kwargs` to all underlying implementations automatically. - -## Built-in Cache Adapters - -Instructor v1.9.1 ships with two production-ready cache implementations: - -### 1. AutoCache - In-Process LRU Cache - -Perfect for single-process applications and development: - -```python -from instructor.cache import AutoCache - -# Thread-safe in-memory cache with LRU eviction -cache = AutoCache(maxsize=1000) -client = from_provider("openai/gpt-4o", cache=cache) -``` - -**When to use**: -- Development and testing -- Single-process applications -- When you need maximum speed (200,000x+ faster cache hits) -- Applications where cache persistence isn't required - -### 2. DiskCache - Persistent Storage - -Ideal when you need cache persistence across sessions: - -```python -from instructor.cache import DiskCache - -# Persistent disk-based cache -cache = DiskCache(directory=".instructor_cache") -client = from_provider("anthropic/claude-3-sonnet", cache=cache) -``` - -**When to use**: -- Applications that restart frequently -- Development workflows where you want to preserve cache between sessions -- When working with expensive or time-intensive API calls -- Local applications with moderate performance requirements - -## Smart Cache Key Generation - -Instructor automatically generates intelligent cache keys that include: - -- **Provider/model name** - Different models get different cache entries -- **Complete message history** - Full conversation context is hashed -- **Response model schema** - Any changes to your Pydantic model automatically bust the cache -- **Mode configuration** - JSON vs Tools mode changes are tracked - -This means when you update your Pydantic model (adding fields, changing descriptions, etc.), the cache automatically invalidates old entries - no stale data! - -```python -from instructor.cache import make_cache_key - -# Generate deterministic cache key -key = make_cache_key( - messages=[{"role": "user", "content": "hello"}], - model="gpt-3.5-turbo", - response_model=User, - mode="TOOLS", -) -print(key) # SHA-256 hash: 9b8f5e2c8c9e... -``` - -## Custom Cache Implementations - -Want Redis, Memcached, or a custom backend? Simply inherit from `BaseCache`: - -```python -from instructor.cache import BaseCache -import redis - - -class RedisCache(BaseCache): - def __init__(self, host="localhost", port=6379, **kwargs): - self.redis = redis.Redis(host=host, port=port, **kwargs) - - def get(self, key: str): - value = self.redis.get(key) - return value.decode() if value else None - - def set(self, key: str, value, ttl: int | None = None): - if ttl: - self.redis.setex(key, ttl, value) - else: - self.redis.set(key, value) - - -# Use your custom cache -redis_cache = RedisCache(host="my-redis-server") -client = from_provider("openai/gpt-4o", cache=redis_cache) -``` - -The `BaseCache` interface is intentionally minimal - just implement `get()` and `set()` methods and you're ready to go. - -## Time-to-Live (TTL) Support - -Control cache expiration with per-call TTL overrides: - -```python -# Cache this result for 1 hour -result = client.create( - messages=[{"role": "user", "content": "Generate daily report"}], - response_model=Report, - cache_ttl=3600, # 1 hour in seconds -) -``` - -TTL support depends on your cache backend: -- **AutoCache**: TTL is ignored (no expiration) -- **DiskCache**: Full TTL support with automatic expiration -- **Custom backends**: Implement TTL handling in your `set()` method - -## Migration from Manual Caching - -If you were using custom caching decorators, migrating is straightforward: - -**Before v1.9.1**: -```python -@functools.cache -def extract_user(text: str) -> User: - return client.create( - messages=[{"role": "user", "content": text}], response_model=User - ) -``` - -**With v1.9.1**: -```python -# Remove decorator, add cache to client -client = from_provider("openai/gpt-4o", cache=AutoCache()) - - -def extract_user(text: str) -> User: - return client.create( - messages=[{"role": "user", "content": text}], response_model=User - ) -``` - -No more function-level caching logic - just create your client with caching enabled and all calls benefit automatically. - -## Real-World Performance Impact - -Native caching delivers the same dramatic performance improvements you'd expect: - -- **AutoCache**: 200,000x+ speed improvement for cache hits -- **DiskCache**: 5-10x improvement with persistence benefits -- **Cost Reduction**: 50-90% API cost savings depending on cache hit rate - -For a comprehensive deep-dive into caching strategies and performance analysis, check out our [complete caching guide](caching.md). - -## Getting Started - -Ready to enable native caching? Here's your quick start: - -1. **Upgrade to v1.9.1+**: - ```bash - pip install "instructor>=1.9.1" - ``` - -2. **Choose your cache backend**: - ```python - from instructor.cache import AutoCache, DiskCache - - # For development/single-process - cache = AutoCache(maxsize=1000) - - # For persistence - cache = DiskCache(directory=".cache") - ``` - -3. **Add cache to your client**: - ```python - from instructor import from_provider - - client = from_provider("your/favorite/model", cache=cache) - ``` - -4. **Use normally - caching happens automatically**: - ```python - result = client.create( - messages=[{"role": "user", "content": "your prompt"}], response_model=YourModel - ) - ``` - -## Learn More - -For detailed information about cache design, custom implementations, and advanced patterns, visit our [Caching Concepts](../../concepts/caching.md) documentation. - -The native caching feature represents our commitment to making high-performance LLM applications simple and accessible. No more complex caching logic - just fast, cost-effective structured outputs out of the box. - ---- - -*Have questions about native caching or want to share your use case? Join the discussion in our [GitHub repository](https://github.com/jxnl/instructor) or check out the [complete documentation](../../concepts/caching.md).* \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/open_source.md b/참고/instructor-main/docs/blog/posts/open_source.md deleted file mode 100644 index 1c3e7c5..0000000 --- a/참고/instructor-main/docs/blog/posts/open_source.md +++ /dev/null @@ -1,263 +0,0 @@ ---- -authors: -- jxnl -categories: -- API Development -comments: true -date: 2024-03-07 -description: Discover how Instructor integrates with OpenAI and local LLMs for structured - outputs using Pydantic and JSON schema. -draft: false -slug: open-source-local-structured-output-pydantic-json-openai -tags: -- OpenAI -- Pydantic -- LLMs -- Structured Outputs -- API Integration ---- - -# Structured Output for Open Source and Local LLMs - -Instructor has expanded its capabilities for language models. It started with API interactions via the OpenAI SDK, using [Pydantic](https://pydantic-docs.helpmanual.io/) for structured data validation. Now, Instructor supports multiple models and platforms. - -The integration of [JSON mode](../../concepts/patching.md#json-mode) improved adaptability to vision models and open source alternatives. This allows support for models from [GPT](https://openai.com/api/) and [Mistral](https://mistral.ai) to models on [Ollama](https://ollama.ai) and [Hugging Face](https://huggingface.co/models), using [llama-cpp-python](../../integrations/llama-cpp-python.md). - -Instructor now works with cloud-based APIs and local models for structured data extraction. Developers can refer to our guide on [Patching](../../concepts/patching.md) for information on using JSON mode with different models. - -For learning about Instructor and Pydantic, we offer a course on [Steering language models towards structured outputs](https://www.wandb.courses/courses/steering-language-models). - -The following sections show examples of Instructor's integration with platforms and local setups for structured outputs in AI projects. - - - - -## Exploring Different OpenAI Clients with Instructor - -OpenAI clients offer functionalities for different needs. We explore clients integrated with Instructor, providing structured outputs and capabilities. Examples show how to initialize and patch each client. - -## Local Models - -### Ollama: A New Frontier for Local Models - -Ollama enables structured outputs with local models using JSON schema. See our [Ollama documentation](../../integrations/ollama.md) for details. - -For setup and features, refer to the documentation. The [Ollama website](https://ollama.ai/download) provides resources, models, and support. - -``` -ollama run llama2 -``` - -```python -from openai import OpenAI -from pydantic import BaseModel -import instructor - - -class UserDetail(BaseModel): - name: str - age: int - - -# enables `response_model` in create call -client = instructor.from_openai( - OpenAI( - base_url="http://localhost:11434/v1", - api_key="ollama", # required, but unused - ), - mode=instructor.Mode.JSON, -) - - -user = client.create( - model="llama2", - messages=[ - { - "role": "user", - "content": "Jason is 30 years old", - } - ], - response_model=UserDetail, -) - -print(user) -#> name='Jason' age=30 -``` - -### llama-cpp-python - -llama-cpp-python provides the `llama-cpp` model for structured outputs using JSON schema. It uses [constrained sampling](https://llama-cpp-python.readthedocs.io/en/latest/#json-schema-mode) and [speculative decoding](https://llama-cpp-python.readthedocs.io/en/latest/#speculative-decoding). An [OpenAI compatible client](https://llama-cpp-python.readthedocs.io/en/latest/#openai-compatible-web-server) allows in-process structured output without network dependency. - -Example of using llama-cpp-python for structured outputs: - - -```python -import llama_cpp -import instructor -from llama_cpp.llama_speculative import LlamaPromptLookupDecoding -from pydantic import BaseModel - - -llama = llama_cpp.Llama( - model_path="../../models/OpenHermes-2.5-Mistral-7B-GGUF/openhermes-2.5-mistral-7b.Q4_K_M.gguf", - n_gpu_layers=-1, - chat_format="chatml", - n_ctx=2048, - draft_model=LlamaPromptLookupDecoding(num_pred_tokens=2), - logits_all=True, - verbose=False, -) - - -create = instructor.patch( - create=llama.create_chat_completion_openai_v1, - mode=instructor.Mode.JSON_SCHEMA, -) - - -class UserDetail(BaseModel): - name: str - age: int - - -user = create( - messages=[ - { - "role": "user", - "content": "Extract `Jason is 30 years old`", - } - ], - response_model=UserDetail, -) - -print(user) -#> name='Jason' age=30 -``` - -## Alternative Providers - -### Groq - -Groq's platform, detailed further in our [Groq documentation](../../integrations/groq.md) and on [Groq's official documentation](https://groq.com/), offers a unique approach to processing with its tensor architecture. This innovation significantly enhances the performance of structured output processing. - -```bash -export GROQ_API_KEY="your-api-key" -``` - -```python -import os -from pydantic import BaseModel - -import groq -import instructor - - -client = groq.Groq( - api_key=os.environ.get("GROQ_API_KEY"), -) - -# By default, the patch function will patch the ChatCompletion.create and ChatCompletion.create methods -# to support the response_model parameter -client = instructor.from_openai(client, mode=instructor.Mode.MD_JSON) - - -# Now, we can use the response_model parameter using only a base model -# rather than having to use the OpenAISchema class -class UserExtract(BaseModel): - name: str - age: int - - -user: UserExtract = client.create( - model="mixtral-8x7b-32768", - response_model=UserExtract, - messages=[ - {"role": "user", "content": "Extract jason is 25 years old"}, - ], -) - -assert isinstance(user, UserExtract), "Should be instance of UserExtract" - -print(user) -#> name='jason' age=25 -``` - -### Together AI - -Together AI, when combined with Instructor, offers a seamless experience for developers looking to leverage structured outputs in their applications. For more details, refer to our [Together AI documentation](../../integrations/together.md) and explore the [patching guide](../../concepts/patching.md) to enhance your applications. - -```bash -export TOGETHER_API_KEY="your-api-key" -``` - -```python -import os -from pydantic import BaseModel - -import instructor -import openai - - -client = openai.OpenAI( - base_url="https://api.together.xyz/v1", - api_key=os.environ["TOGETHER_API_KEY"], -) - -client = instructor.from_openai(client, mode=instructor.Mode.TOOLS) - - -class UserExtract(BaseModel): - name: str - age: int - - -user: UserExtract = client.create( - model="mistralai/Mixtral-8x7B-Instruct-v0.1", - response_model=UserExtract, - messages=[ - {"role": "user", "content": "Extract jason is 25 years old"}, - ], -) - -assert isinstance(user, UserExtract), "Should be instance of UserExtract" - -print(user) -#> name='jason' age=25 -``` - -### Mistral - -For those interested in exploring the capabilities of Mistral Large with Instructor, we highly recommend checking out our comprehensive guide on [Mistral Large](../../integrations/mistral.md). - -```python -import instructor -from pydantic import BaseModel -from mistralai.client import MistralClient - - -client = MistralClient() - -patched_chat = instructor.from_openai( - create=client.chat, mode=instructor.Mode.TOOLS -) - - -class UserDetails(BaseModel): - name: str - age: int - - -resp = patched_chat( - model="mistral-large-latest", - response_model=UserDetails, - messages=[ - { - "role": "user", - "content": f'Extract the following entities: "Jason is 20"', - }, - ], -) - -print(resp) -#> name='Jason' age=20 -``` diff --git a/참고/instructor-main/docs/blog/posts/openai-distilation-store.md b/참고/instructor-main/docs/blog/posts/openai-distilation-store.md deleted file mode 100644 index 8154235..0000000 --- a/참고/instructor-main/docs/blog/posts/openai-distilation-store.md +++ /dev/null @@ -1,118 +0,0 @@ ---- -authors: -- jxnl -categories: -- OpenAI -comments: true -date: 2024-10-02 -description: Learn how to use OpenAI's API Model Distillation with Instructor to create - efficient, tailored models for your applications. -draft: false -tags: -- OpenAI -- API Model Distillation -- Instructor -- Machine Learning -- Data Processing ---- - -# OpenAI API Model Distillation with Instructor - -OpenAI has recently introduced a new feature called [API Model Distillation](https://openai.com/index/api-model-distillation/), which allows developers to create custom models tailored to their specific use cases. This feature is particularly powerful when combined with Instructor's structured output capabilities. In this post, we'll explore how to leverage API Model Distillation with Instructor to create more efficient and specialized models. - - - -## What is API Model Distillation? - -API Model Distillation is a process that allows you to create a smaller, more focused model based on the inputs and outputs of a larger model. This distilled model can be more efficient and cost-effective for specific tasks while maintaining high performance. - -## Using Instructor with API Model Distillation - -Instructor's integration with OpenAI's API makes it seamless to use API Model Distillation. Here's how you can get started, make sure you have the latest version of OpenAI! - -``` -pip install -U openai -``` - -```python -import instructor -from pydantic import BaseModel - -# Enable response_model and API Model Distillation -client = instructor.from_provider("openai/gpt-4o") - - -class UserDetail(BaseModel): - name: str - age: int - - def introduce(self): - return f"Hello, I'm {self.name} and I'm {self.age} years old" - - -# Use the store parameter to enable API Model Distillation -user: UserDetail = client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": "Extract Jason is 25 years old"}, - ], - store=True, # Enable API Model Distillation -) -``` - -In this example, we've added the `store=True` parameter to the `chat.completions.create` method. This enables API Model Distillation for this specific call. - -## Metadata and Proxy Kwargs - -One of the great advantages of using Instructor with API Model Distillation is that it automatically handles metadata and proxies kwargs to the underlying OpenAI API. This means you can use additional parameters supported by the [OpenAI API](https://platform.openai.com/docs/api-reference) without any extra configuration. - -For example, you can add metadata to your API calls: - -```python -user: UserDetail = client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": "Extract Jason is 25 years old"}, - ], - store=True, - metadata={"task": "user_extraction", "source": "customer_support_chat"}, -) -``` - -The `metadata` parameter will be automatically passed to the OpenAI API, allowing you to track and organize your API calls for distillation purposes. - - -## Completions Dashboard - -To better understand how API Model Distillation works with Instructor, let's take a look at the following diagram: - -![API Model Distillation with Instructor](./img/distil_openai.png) - -This image illustrates the process of API Model Distillation when using Instructor with OpenAI's API. It shows how the structured output from Instructor, combined with metadata and other parameters, feeds into the distillation process to create a specialized model tailored to your specific use case. - -The diagram highlights: - -1. The initial request with structured output using Instructor -2. The inclusion of metadata and additional parameters -3. The distillation process that creates a specialized model -4. The resulting distilled model that can be used for faster, more efficient responses - -This visual representation helps to clarify the flow and benefits of using API Model Distillation in conjunction with Instructor's capabilities. - - -## Benefits of Using Instructor with API Model Distillation - -1. **Structured Output**: Instructor's use of [Pydantic](https://docs.pydantic.dev/) models ensures that your distilled model produces structured, validated output. -2. **Simplified Integration**: The proxy kwargs feature means you can use all OpenAI API parameters without additional configuration. -3. **Improved Efficiency**: By distilling models for specific tasks, you can reduce latency and costs for your applications. -4. **Consistency**: Distilled models can provide more consistent outputs for specialized tasks. - -## Conclusion - -API Model Distillation with Instructor's structured output creates efficient, specialized models. Instructor's integration with OpenAI's API allows you to incorporate this feature into workflows, improving performance and cost-effectiveness of AI applications. - -Remember to check [OpenAI's documentation](https://platform.openai.com/docs) for the latest information on API Model Distillation and best practices for creating and using distilled models. - -For more information on using Instructor, visit the [Instructor GitHub repository](https://github.com/jxnl/instructor) and give it a star if you find it helpful! \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/openai-multimodal.md b/참고/instructor-main/docs/blog/posts/openai-multimodal.md deleted file mode 100644 index 7dc283e..0000000 --- a/참고/instructor-main/docs/blog/posts/openai-multimodal.md +++ /dev/null @@ -1,98 +0,0 @@ ---- -authors: - - jxnl -categories: - - OpenAI - - Audio -comments: true -date: 2024-10-17 -description: Explore the new audio capabilities in OpenAI's Chat Completions API using the gpt-4o-audio-preview model. -draft: false -tags: - - OpenAI - - Audio Processing - - API - - Machine Learning ---- - -# Audio Support in OpenAI's Chat Completions API - -OpenAI has recently introduced audio support in their Chat Completions API, opening up exciting new possibilities for developers working with audio and text interactions. This feature is powered by the new `gpt-4o-audio-preview` model, which brings advanced voice capabilities to the familiar Chat Completions API interface. - - - -## Key Features - -The new audio support in the Chat Completions API offers several compelling features: - -1. **Flexible Input Handling**: The API can now process any combination of text and audio inputs, allowing for more versatile applications. - -2. **Natural, Steerable Voices**: Similar to the Realtime API, developers can use prompting to shape various aspects of the generated audio, including language, pronunciation, and emotional range. - -3. **Tool Calling Integration**: The audio support seamlessly integrates with existing tool calling functionality, enabling complex workflows that combine audio, text, and external tools. - -## Practical Example - -To demonstrate how to use this new functionality, let's look at a simple example using the `instructor` library: - -```python -from pydantic import BaseModel -import instructor -from instructor.processing.multimodal import Audio - -client = instructor.from_provider("openai/gpt-5-nano") - - -class Person(BaseModel): - name: str - age: int - - -resp = client.create( - model="gpt-4o-audio-preview", - response_model=Person, - modalities=["text"], - audio={"voice": "alloy", "format": "wav"}, - messages=[ - { - "role": "user", - "content": [ - "Extract the following information from the audio", - Audio.from_path("./output.wav"), - ], - }, - ], -) - -print(resp) -# Expected output: Person(name='Jason', age=20) -``` - -In this example, we're using the `gpt-4o-audio-preview` model to extract information from an audio file. The API processes the audio input and returns structured data (a Person object with name and age) based on the content of the audio. - -## Use Cases - -The addition of audio support to the Chat Completions API enables a wide range of applications: - -1. **Voice-based Personal Assistants**: Create more natural and context-aware voice interfaces for various applications. - -2. **Audio Content Analysis**: Automatically extract information, sentiments, or key points from audio recordings or podcasts. - -3. **Language Learning Tools**: Develop interactive language learning applications that can process and respond to spoken language. - -4. **Accessibility Features**: Improve accessibility in applications by providing audio-based interactions and text-to-speech capabilities. - -## Considerations - -While this new feature is exciting, it's important to note that it's best suited for asynchronous use cases that don't require extremely low latencies. For more dynamic and real-time interactions, OpenAI recommends using their Realtime API. - -As with any AI-powered feature, it's crucial to consider ethical implications and potential biases in audio processing and generation. Always test thoroughly and consider the diversity of your user base when implementing these features. - -## Related Documentation -- [Multimodal Guide](../../concepts/multimodal.md) - Comprehensive multimodal reference -- [OpenAI Integration](../../integrations/openai.md) - Full OpenAI setup - -## See Also -- [Gemini Multimodal](multimodal-gemini.md) - Alternative multimodal approach -- [Prompt Caching](anthropic-prompt-caching.md) - Cache large audio files -- [Monitoring with Logfire](logfire.md) - Track multimodal processing diff --git a/참고/instructor-main/docs/blog/posts/pairwise-llm-judge.md b/참고/instructor-main/docs/blog/posts/pairwise-llm-judge.md deleted file mode 100644 index d7da031..0000000 --- a/참고/instructor-main/docs/blog/posts/pairwise-llm-judge.md +++ /dev/null @@ -1,152 +0,0 @@ ---- -authors: - - jxnl -categories: - - LLM - - Pydantic -comments: true -date: 2024-10-17 -description: Explore how to use Instructor and Pydantic to create a pairwise LLM judge for evaluating text relevance. -draft: false -tags: - - LLM - - Pydantic - - Instructor - - Text Relevance - - AI Evaluation ---- - -# Building a Pairwise LLM Judge with Instructor and Pydantic - -In this blog post, we'll explore how to create a pairwise LLM judge using Instructor and Pydantic. This judge will evaluate the relevance between a question and a piece of text, demonstrating a practical application of structured outputs in language model interactions. - -## Introduction - -Evaluating text relevance is a common task in natural language processing and information retrieval. By leveraging large language models (LLMs) and structured outputs, we can create a system that judges the similarity or relevance between a question and a given text. - - - -## Setting Up the Environment - -First, let's set up our environment with the necessary imports: - -```python -import instructor - -client = instructor.from_provider("openai/gpt-5-nano") -``` - -Here, we're using the `instructor` library, which integrates seamlessly with OpenAI's API and Pydantic for structured outputs. - -## Defining the Judgment Model - -We'll use Pydantic to define a `Judgment` model that structures the output of our LLM: - -```python -class Judgment(BaseModel): - thought: str = Field( - description="The step-by-step reasoning process used to analyze the question and text" - ) - justification: str = Field( - description="Explanation for the similarity judgment, detailing key factors that led to the conclusion" - ) - similarity: bool = Field( - description="Boolean judgment indicating whether the question and text are similar or relevant (True) or not (False)" - ) -``` - -This model ensures that our LLM's output is structured and includes a thought process, justification, and a boolean similarity judgment. - -## Creating the Judge Function - -Next, we'll create a function that uses our LLM to judge the relevance between a question and a text: - -```python -def judge_relevance(question: str, text: str) -> Judgment: - return client.chat.create( - model="gpt-4", - messages=[ - { - "role": "system", - "content": """ - You are tasked with comparing a question and a piece of text to determine if they are relevant to each other or similar in some way. Your goal is to analyze the content, context, and potential connections between the two. - - To determine if the question and text are relevant or similar, please follow these steps: - - 1. Carefully read and understand both the question and the text. - 2. Identify the main topic, keywords, and concepts in the question. - 3. Analyze the text for any mention of these topics, keywords, or concepts. - 4. Consider any potential indirect connections or implications that might link the question and text. - 5. Evaluate the overall context and purpose of both the question and the text. - - As you go through this process, please use a chain of thought approach. Write out your reasoning for each step inside tags. - - After your analysis, provide a boolean judgment on whether the question and text are similar or relevant to each other. Use "true" if they are similar or relevant, and "false" if they are not. - - Before giving your final judgment, provide a justification for your decision. Explain the key factors that led to your conclusion. - - Please ensure your analysis is thorough, impartial, and based on the content provided. - """, - }, - { - "role": "user", - "content": """ - Here is the question: - - - {{question}} - - - Here is the text: - - {{text}} - - """, - }, - ], - response_model=Judgment, - context={"question": question, "text": text}, - ) -``` - -This function takes a question and a text as input, sends them to the LLM with a predefined prompt, and returns a structured `Judgment` object. - -## Testing the Judge - -To test our pairwise LLM judge, we can create a set of test pairs and evaluate the judge's performance: - -```python -if __name__ == "__main__": - test_pairs = [ - { - "question": "What are the main causes of climate change?", - "text": "Global warming is primarily caused by human activities, such as burning fossil fuels, deforestation, and industrial processes. These activities release greenhouse gases into the atmosphere, trapping heat and leading to a rise in global temperatures.", - "is_similar": True, - }, - # ... (other test pairs) - ] - - score = 0 - for pair in test_pairs: - result = judge_relevance(pair["question"], pair["text"]) - if result.similarity == pair["is_similar"]: - score += 1 - - print(f"Score: {score}/{len(test_pairs)}") - #> Score 9/10 -``` - -This test loop runs the judge on each pair and compares the result to a predetermined similarity value, calculating an overall score. - -## Conclusion - -By combining Instructor, Pydantic, and OpenAI's language models, we've created a powerful tool for judging text relevance. This approach demonstrates the flexibility and power of structured outputs in LLM applications. - -The pairwise LLM judge we've built can be used in various scenarios, such as: - -1. Improving search relevance in information retrieval systems -2. Evaluating the quality of question-answering systems -3. Assisting in content recommendation algorithms -4. Automating parts of the content moderation process - -As you explore this technique, consider how you might extend or adapt it for your specific use cases. The combination of structured outputs and large language models opens up a world of possibilities for creating intelligent, interpretable AI systems. diff --git a/참고/instructor-main/docs/blog/posts/parea.md b/참고/instructor-main/docs/blog/posts/parea.md deleted file mode 100644 index f352474..0000000 --- a/참고/instructor-main/docs/blog/posts/parea.md +++ /dev/null @@ -1,166 +0,0 @@ ---- -authors: - - jxnl - - joschkabraun -categories: - - LLM Observability -comments: true -date: 2024-07-17 -description: - Explore how Parea enhances the OpenAI instructor, enabling better monitoring, - collaboration, and error tracking for LLM applications. -draft: false -tags: - - Parea - - OpenAI - - LLM - - instructor - - validation ---- - -# Parea for Observing, Testing & Fine-tuning of Instructor - -[Parea](https://www.parea.ai) is a platform that enables teams to monitor, collaborate, test & label for LLM applications. In this blog we will explore how Parea can be used to enhance the OpenAI client alongside `instructor` and debug + improve `instructor` calls. Parea has some features which makes it particularly useful for `instructor`: - -- it automatically groups any LLM calls due to reties under a single trace -- it automatically tracks any validation error counts & fields that occur when using `instructor` -- it provides a UI to label JSON responses by filling out a form instead of editing JSON objects - -??? info "Configure Parea" - - Before starting this tutorial, make sure that you've registered for a [Parea](https://www.parea.ai) account. You'll also need to create an [API key](https://docs.parea.ai/api-reference/authentication). - -## Example: Writing Emails with URLs from Instructor Docs - -We will demonstrate Parea by using `instructor` to write emails which only contain URLs from the `instructor` docs. We'll need to install our dependencies before proceeding so simply run the command below. - - - -```bash -pip install -U parea-ai instructor -``` - -Parea is dead simple to integrate - all it takes is 2 lines of code, and we have it setup. - -```python hl_lines="9 15-16" -import os - -import instructor -from dotenv import load_dotenv -from openai import OpenAI -from parea import Parea # (1)! - -load_dotenv() - -client = OpenAI() - -p = Parea(api_key=os.getenv("PAREA_API_KEY")) # (2)! -p.wrap_openai_client(client, "instructor") - -client = instructor.from_provider("openai/gpt-4o") -``` - -1. Import `Parea` from the `parea` module -2. Setup tracing using their native integration with `instructor` - -In this example, we'll be looking at writing emails which only contain links to the instructor docs. To do so, we can define a simple Pydantic model as seen below. - -```python -class Email(BaseModel): - subject: str - body: str = Field( - ..., - description="Email body, Should contain links to instructor documentation. ", - ) - - @field_validator("body") - def check_urls(cls, v): - urls = re.findall(r"https?://(?:[-\w.]|(?:%[\da-fA-F]{2}))+", v) - errors = [] - for url in urls: - if not url.startswith("https://python.useinstructor.com"): - errors.append( - f"URL {url} is not from useinstructor.com, Only include URLs that include use instructor.com. " - ) - response = requests.get(url) - if response.status_code != 200: - errors.append( - f"URL {url} returned status code {response.status_code}. Only include valid URLs that exist." - ) - elif "404" in response.text: - errors.append( - f"URL {url} contained '404' in the body. Only include valid URLs that exist." - ) - if errors: - raise ValueError("\n".join(errors)) - return -``` - -Now we can proceed to create an email using above Pydantic model. - -```python hl_lines="5-14" -email = client.messages.create( - model="gpt-3.5-turbo", - max_tokens=1024, - max_retries=3, - messages=[ # (1)! - { - "role": "user", - "content": "I'm responding to a student's question. Here is the link to the documentation: {{doc_link1}} and {{doc_link2}}", - } - ], - template_inputs={ - "doc_link1": "https://python.useinstructor.com/docs/tutorial/tutorial-1", - "doc_link2": "https://jxnl.github.io/docs/tutorial/tutorial-2", - }, - response_model=Email, -) -print(email) -``` - -1. Parea supports templated prompts via `{{...}}` syntax in the `messages` parameter. We can pass the template inputs as a dictionary to the `template_inputs` parameter. - -If you follow what we've done, Parea has wrapped the client, and we wrote an email with links from the instructor docs. - -## Validation Error Tracking - -To take a look at trace of this execution checkout the screenshot below. Noticeable: - -- left sidebar: all related LLM calls are grouped under a trace called `instructor` -- middle section: the root trace visualizes the `templated_inputs` as inputs and the created `Email` object as output -- bottom of right sidebar: any validation errors are captured and tracked as score for the trace which enables visualizing them in dashboards and filtering by them on tables - -![](./img/parea/trace.png) - -Above we can see that while the email was successfully created, there was a validation error which meant that additional cost & latency were introduced because of the initially failed validation. -Below we can see a visualization of the average validation error count for our instructor usage over time. - -![](./img/parea/validation-error-chart.png) - -## Label Responses for Fine-Tuning - -Sometimes you may want to let subject-matter experts (SMEs) label responses to use them for fine-tuning. Parea provides a way to do this via an annotation queue. Editing raw JSON objects to correct tool use & function calling responses can be error-prone, esp. for non-devs. For that purpose, Parea has a so-called [Form Mode](https://docs.parea.ai/manual-review/overview#labeling-function-calling-tool-use-responses) which allows the user to safely fill-out a form instead of editing the JSON object. The labeled data can then be exported and used for fine-tuning. - -![Form Mode](img/parea/form-mode.gif) - -??? info "Export Labeled Data & Fine-Tune" - - After labeling the data, you can export them as JSONL file: - - ```python hl_lines="5 6" - from parea import Parea - - p = Parea(api_key=os.getenv("PAREA_API_KEY")) - - dataset = p.get_collection(DATASET_ID) # (1)! - dataset.write_to_finetune_jsonl("finetune.jsonl") # (2)! - ``` - - 1. Replace `DATASET_ID` with the actual dataset ID - 2. Writes the dataset to a JSONL file - - Now we can use `instructor` to fine-tune the model: - - ```bash - instructor jobs create-from-file finetune.jsonl - ``` diff --git a/참고/instructor-main/docs/blog/posts/pydantic-is-still-all-you-need.md b/참고/instructor-main/docs/blog/posts/pydantic-is-still-all-you-need.md deleted file mode 100644 index 31d6a65..0000000 --- a/참고/instructor-main/docs/blog/posts/pydantic-is-still-all-you-need.md +++ /dev/null @@ -1,132 +0,0 @@ ---- -authors: -- jxnl -categories: -- Pydantic -comments: true -date: 2024-09-07 -description: Explore how Pydantic enhances structured outputs in LLM applications, - ensuring reliability and improved data management. -draft: false -slug: pydantic-is-still-all-you-need -tags: -- Pydantic -- Structured Outputs -- Data Validation -- LLM Techniques -- Performance Optimization ---- - -# Pydantic is Still All You Need: Reflections on a Year of Structured Outputs - -A year ago, I gave a talk titled "Pydantic: All You Need" that kickstarted my Twitter career. Today, I'm back to reaffirm that message and share what I've learned in the past year about using structured outputs with language models. - -[Watch the youtube video](https://www.youtube.com/watch?v=pZ4DIH2BVqg){ .md-button .md-button--primary } - - - -## The Problem with Unstructured Outputs - -Imagine hiring an intern to write an API that returns a string you have to JSON load into a dictionary and pray the data is still there. You'd probably fire them and replace them with GPT. Yet, many of us are content using LLMs in the same haphazard way. - -By not using schemas and structured responses, we lose compatibility, composability, and reliability when building tools that interact with external systems. But there's a better way. - -## The Power of Pydantic - -Pydantic, combined with function calling, offers a superior alternative for structured outputs. It allows for: - -- Nested objects and models for modular structures -- Validators to improve system reliability -- Cleaner, more maintainable code - -For more details on how Pydantic enhances data validation, check out our [Data Validation with Pydantic](../../concepts/models.md) guide. - -And here's the kicker: nothing's really changed in the past year. The core API is still just: - -```python -from instructor import from_openai - -client = from_openai(OpenAI()) - -response = client.create(model="gpt-3.5-turbo", response_model=User, messages=[...]) -``` - -## What's New in Pydantic? - -Since last year: - -- We've released version 1.0 -- Launched in 5 languages (Python, TypeScript, Ruby, Go, Elixir) -- Built a version in Rust -- Seen 40% month-over-month growth in the Python library - -We now support [Ollama](../../integrations/ollama.md), [llama-cpp-python](../../integrations/llama-cpp-python.md), [Anthropic](../../integrations/anthropic.md), [Cohere](../../integrations/cohere.md), [Google](../../integrations/google.md), [Vertex AI](../../integrations/vertex.md), and more. As long as language models support function calling capabilities, this API will remain standard. - -## Key Features - -1. **Streaming with Structure**: Get objects as they return, improving latency while maintaining structured output. Learn more about this in our [Streaming Support](../../concepts/partial.md) guide. - -2. **Partials**: Validate entire objects, enabling real-time rendering for generative UI without complex JSON parsing. See our [Partial](../../concepts/partial.md) documentation for implementation details. - -3. **Validators**: Add custom logic to ensure correct outputs, with the ability to retry on errors. Dive deeper into this topic in our [Reasking and Validation](../../concepts/reask_validation.md) guide. - -## Real-World Applications - -### Generation and Extraction - -Structured outputs shine in tasks like: - -- Generating follow-up questions in RAG applications -- Validating URLs in generated content -- Extracting structured data from transcripts or images - -For a practical example, see our [Structured Data Extraction from Images](../../examples/image_to_ad_copy.md) case study. - -### Search Queries - -For complex search scenarios: - -```python -class Search(BaseModel): - query: str - start_date: Optional[datetime] - end_date: Optional[datetime] - limit: Optional[int] - source: Literal["news", "social", "blog"] -``` - -This structure allows for more sophisticated search capabilities, handling queries like "What is the latest news from X?" that embeddings alone can't handle. - -## Lessons Learned - -1. Validation errors are crucial for improving system performance. -2. Not all language models support retry logic effectively yet. -3. Structured outputs benefit vision, text, RAG, and agent applications alike. - -## The Future of Programming with LLMs - -We're not changing the language of programming; we're relearning how to program with data structures. Structured outputs allow us to: - -- Own the objects we define -- Control the functions we implement -- Manage the control flow -- Own the prompts - -This approach makes Software 3.0 backwards compatible with existing software, demystifying language models and returning us to a more classical programming structure. - -## Wrapping Up - -Pydantic is still all you need for effective structured outputs with LLMs. It's not just about generating accurate responses; it's about doing so in a way that's compatible with our existing programming paradigms and tools. - -As we continue to refine AI language models, keeping these principles in mind will lead to more robust, maintainable, and powerful applications. The future of AI isn't just about what the models can do, but how seamlessly we can integrate them into our existing software ecosystems. - -For more advanced use cases and integrations, check out our [examples](../../examples/index.md) section, which covers various LLM providers and specialized implementations. - -## Related Documentation -- [Instructor Philosophy](../../concepts/philosophy.md) - Why we chose Pydantic -- [Validation Guide](../../concepts/validation.md) - Practical validation techniques - -## See Also -- [Validation Deep Dive](validation-part1.md) - Advanced validation patterns -- [Best Framework Comparison](best_framework.md) - Why Instructor stands out -- [Introduction to Instructor](introduction.md) - Getting started guide diff --git a/참고/instructor-main/docs/blog/posts/rag-and-beyond.md b/참고/instructor-main/docs/blog/posts/rag-and-beyond.md deleted file mode 100644 index 930a332..0000000 --- a/참고/instructor-main/docs/blog/posts/rag-and-beyond.md +++ /dev/null @@ -1,246 +0,0 @@ ---- -authors: -- jxnl -categories: -- LLM Techniques -comments: true -date: 2023-09-17 -description: 'Explore how to enhance Retrieval Augmented Generation (RAG) with query - understanding for smarter search solutions. ' -draft: false -tags: -- RAG -- query understanding -- LLMs -- data modeling -- Pydantic ---- - -# RAG is more than just embedding search - -With the advent of large language models (LLM), retrieval augmented generation (RAG) has become a hot topic. However throughout the past year of [helping startups](https://jxnl.co) integrate LLMs into their stack I've noticed that the pattern of taking user queries, embedding them, and directly searching a vector store is effectively demoware. - -!!! note "What is RAG?" - - Retrieval augmented generation (RAG) is a technique that uses an LLM to generate responses, but uses a search backend to augment the generation. In the past year using text embeddings with a vector databases has been the most popular approach I've seen being socialized. - -
- ![RAG](img/dumb_rag.png) -
Simple RAG that embedded the user query and makes a search.
-
- -So let's kick things off by examining what I like to call the 'Dumb' RAG Model-a basic setup that's more common than you'd think. - - - -## The 'Dumb' RAG Model - -When you ask a question like, "what is the capital of France?" The RAG 'dumb' model embeds the query and searches in some unopinionated search endpoint. Limited to a single method API like `search(query: str) -> List[str]`. This is fine for simple queries, since you'd expect words like 'paris is the capital of france' to be in the top results of say, your wikipedia embeddings. - -### Why is this a problem? - -- **Query-Document Mismatch**: This model assumes that query embedding and the content embedding are similar in the embedding space, which is not always true based on the text you're trying to search over. Only using queries that are semantically similar to the content is a huge limitation! - -- **Monolithic Search Backend**: Assumes a single search backend, which is not always the case. You may have multiple search backends, each with their own API, and you want to route the query to vector stores, search clients, sql databases, and more. -- **Limitation of text search**: Restricts complex queries to a single string (`{query: str}`), sacrificing expressiveness, in using keywords, filters, and other advanced features. For example, asking `what problems did we fix last week` cannot be answered by a simple text search since documents that contain `problem, last week` are going to be present at every week. - -- **Limited ability to plan**: Assumes that the query is the only input to the search backend, but you may want to use other information to improve the search, like the user's location, or the time of day using the context to rewrite the query. For example, if you present the language model of more context it is able to plan a suite of queries to execute to return the best results. - -Now let's dive into how we can make it smarter with query understanding. This is where things get interesting. - -## Improving the RAG Model with Query Understanding - -!!! note "Shoutouts" -Much of this work has been inspired by / done in collab with a few of my clients at [new.computer](https://new.computer), [Metaphor Systems](https://metaphor.systems), and [Naro](https://narohq.com), go check them out! - -Ultimately what you want to deploy is a [system that understands](https://en.wikipedia.org/wiki/Query_understanding) how to take the query and rewrite it to improve precision and recall. - -
- ![RAG](img/query_understanding.png) -
Query Understanding system routes to multiple search backends.
-
- -Not convinced? Let's move from theory to practice with a real-world example. First up, Metaphor Systems. - -## Whats instructor? - -Instructor uses Pydantic to simplify the interaction between the programmer and language models via the function calling API. - -- **Widespread Adoption**: Pydantic is a popular tool among Python developers. -- **Simplicity**: Pydantic allows model definition in Python. -- **Framework Compatibility**: Many Python frameworks already use Pydantic. - -## Case Study 1: Metaphor Systems - -Take [Metaphor Systems](https://metaphor.systems), which turns natural language queries into their custom search-optimized query. If you take a look web UI you'll notice that they have an auto-prompt option, which uses function calls to further optimize your query using a language model, and turns it into a fully specified metaphor systems query. - -
-![Metaphor Systems](img/meta.png) -
Metaphor Systems UI
-
- -If we peek under the hood, we can see that the query is actually a complex object, with a date range, and a list of domains to search in. It's actually more complex than this but this is a good start. We can model this structured output in Pydantic using the instructor library - -```python -class DateRange(BaseModel): - start: datetime.date - end: datetime.date - - -class MetaphorQuery(BaseModel): - rewritten_query: str - published_daterange: DateRange - domains_allow_list: List[str] - - async def execute(): - return await metaphor.search(...) -``` - -Note how we model a rewritten query, range of published dates, and a list of domains to search in. This powerful pattern allows the user query to be restructured for better performance without the user having to know the details of how the search backend works. - -```python -import instructor - -# Enables response_model in the openai client -client = instructor.from_provider("openai/gpt-5-nano") - -query = client.create( - model="gpt-4", - response_model=MetaphorQuery, - messages=[ - { - "role": "system", - "content": "You're a query understanding system for the Metafor Systems search engine. Here are some tips: ...", - }, - {"role": "user", "content": "What are some recent developments in AI?"}, - ], -) -``` - -**Example Output** - -```json -{ - "rewritten_query": "novel developments advancements ai artificial intelligence machine learning", - "published_daterange": { - "start": "2023-09-17", - "end": "2021-06-17" - }, - "domains_allow_list": ["arxiv.org"] -} -``` - -This isn't just about adding some date ranges. It's about nuanced, tailored searches, that are deeply integrated with the backend. Metaphor Systems has a whole suite of other filters and options that you can use to build a powerful search query. They can even use some chain of thought prompting to improve how they use some of these advanced features. - -```python -class DateRange(BaseModel): - start: datetime.date - end: datetime.date - chain_of_thought: str = Field( - None, - description="Think step by step to plan what is the best time range to search in", - ) -``` - -Now, let's see how this approach can help model an agent like personal assistant. - -## Case Study 2: Personal Assistant - -Another great example of this multiple dispatch pattern is a personal assistant. You might ask, "What do I have today?", from a vague query you might want events, emails, reminders etc. That data will likely exist in multiple backends, but what you want is one unified summary of results. Here you can't assume that text of those documents are all embedded in a search backend. There might be a calendar client, email client, across personal and profession accounts. - -```python -class ClientSource(enum.Enum): - GMAIL = "gmail" - CALENDAR = "calendar" - - -class SearchClient(BaseModel): - query: str - keywords: List[str] - email: str - source: ClientSource - start_date: datetime.date - end_date: datetime.date - - async def execute(self) -> str: - if self.source == ClientSource.GMAIL: - ... - elif self.source == ClientSource.CALENDAR: - ... - - -class Retrieval(BaseModel): - queries: List[SearchClient] - - async def execute(self) -> str: - return await asyncio.gather(*[query.execute() for query in self.queries]) -``` - -Now we can call this with a simple query like "What do I have today?" and it will try to async dispatch to the correct backend. It's still important to prompt the language model well, but we'll leave that for another day. - -```python -import instructor - -# Enables response_model in the openai client -client = instructor.from_provider("openai/gpt-5-nano") - -retrieval = client.create( - model="gpt-4", - response_model=Retrieval, - messages=[ - {"role": "system", "content": "You are Jason's personal assistant."}, - {"role": "user", "content": "What do I have today?"}, - ], -) -``` - -**Example Output** - -```json -{ - "queries": [ - { - "query": None, - "keywords": None, - "email": "jason@example.com", - "source": "gmail", - "start_date": "2023-09-17", - "end_date": None - }, - { - "query": None, - "keywords": ["meeting", "call", "zoom"]]], - "email": "jason@example.com", - "source": "calendar", - "start_date": "2023-09-17", - "end_date": None - - } - ] -} -``` - -Notice that we have a list of queries that route to different search backends (email and calendar). We can even dispatch them async to be as performance as possible. Not only do we dispatch to different backends (that we have no control over), but you are likely going to render them to the user differently as well. Perhaps you want to summarize the emails in text, but you want to render the calendar events as a list that they can scroll across on a mobile app. - -!!! Note "Can I used framework X?" -I get this question a lot, but it's just code. Within these dispatches you can do whatever you want. You can use `input()` to ask the user for more information, make a post request, call a Langchain agent or LLamaindex query engine to get more information. The sky is the limit. - -Both of these examples showcase how both search providers and consumers can use `instructor` to model their systems. This is a powerful pattern that allows you to build a system that can be used by anyone, and can be used to build an LLM layer, from scratch, in front of any arbitrary backend. - -## Conclusion - -This is not about fancy embedding tricks, it's just plain old information retrieval and query understanding. The beauty of instructor is that it simplifies modeling the complex and lets you define the output of the language model, the prompts, and the payload we send to the backend in a single place. - -## What's Next? - -Here I want to show that `instructor` isn’t just about data extraction. It’s a powerful framework for building a data model and integrating it with your LLM. Structured output is just the beginning - the untapped goldmine is skilled use of tools and APIs. - -## Related Documentation -- [Validation Concepts](../../concepts/validation.md) - Validate RAG outputs - -## See Also -- [LLM as Reranker](llm-as-reranker.md) - Improve search relevance -- [Citation Extraction](citations.md) - Verify sources -- [PDF Processing](chat-with-your-pdf-with-gemini.md) - Document handling - -If you enjoy the content or want to try out `instructor` please check out the [github](https://github.com/jxnl/instructor) and give us a star! \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/rag-timelines.md b/참고/instructor-main/docs/blog/posts/rag-timelines.md deleted file mode 100644 index 4c767aa..0000000 --- a/참고/instructor-main/docs/blog/posts/rag-timelines.md +++ /dev/null @@ -1,105 +0,0 @@ ---- -authors: - - jxnl -categories: - - LLM Techniques -comments: true -date: 2024-06-06 -description: - Explore enhancing RAG systems with time filters using Instructor and - Pydantic for accurate, relevant data retrieval. -draft: false -tags: - - RAG - - Time Filters - - Pydantic - - Instructor - - LLM Techniques ---- - -# Enhancing RAG with Time Filters Using Instructor - -Retrieval-augmented generation (RAG) systems often need to handle queries with time-based constraints, like "What new features were released last quarter?" or "Show me support tickets from the past week." Effective time filtering is crucial for providing accurate, relevant responses. - -Instructor is a Python library that simplifies integrating large language models (LLMs) with data sources and APIs. It allows defining structured output models using Pydantic, which can be used as prompts or to parse LLM outputs. - - - -## Modeling Time Filters - -To handle time filters, we can define a Pydantic model representing a time range: - -```python -from datetime import datetime -from typing import Optional -from pydantic import BaseModel - - -class TimeFilter(BaseModel): - start_date: Optional[datetime] = None - end_date: Optional[datetime] = None -``` - -The `TimeFilter` model can represent an absolute date range or a relative time range like "last week" or "previous month." - -We can then combine this with a search query string: - -```python -class SearchQuery(BaseModel): - query: str - time_filter: TimeFilter -``` - -## Prompting the LLM - -Using Instructor, we can prompt the LLM to generate a `SearchQuery` object based on the user's query: - -```python -import instructor - -client = instructor.from_provider("openai/gpt-5-nano") - -response = client.create( - model="gpt-4o", - response_model=SearchQuery, - messages=[ - { - "role": "system", - "content": "You are a query generator for customer support tickets. The current date is 2024-02-17", - }, - { - "role": "user", - "content": "Show me customer support tickets opened in the past week.", - }, - ], -) - -# Example response: -{ - "query": "Show me customer support tickets opened in the past week.", - "time_filter": { - "start_date": "2024-02-10T00:00:00", - "end_date": "2024-02-17T00:00:00", - }, -} -``` - -## Nuances in dates and timezones - -When working with time-based queries, it's important to consider the nuances of dates, timezones, and publication times. Depending on the data source, the user's location, and when the content was originally published, the definition of "past week" or "last month" may vary. - -To handle this, you'll want to design your `TimeFilter` model to intelligently reason about these relative time periods. This could involve: - -- Defaulting to the user's local timezone if available, or using a consistent default like UTC -- Defining clear rules for how to calculate the start and end of relative periods like "week" or "month" - - e.g. does "past week" mean the last 7 days or the previous Sunday-Saturday range? -- Allowing for flexibility in how users specify dates (exact datetimes, just dates, natural language phrases) -- Validating and normalizing user input to fit the expected `TimeFilter` format -- Considering the original publication timestamp of the content, not just the current date - - e.g. "articles published in the last month" should look at the publish date, not the query date - -By building this logic into the `TimeFilter` model, you can abstract away the complexity and provide a consistent interface for the rest of your RAG system to work with standardized absolute datetime ranges - -Of course, there may be edge cases or ambiguities that are hard to resolve programmatically. In these situations, you may need to prompt the user for clarification or make a best guess based on the available information. The key is to strive for a balance of flexibility and consistency in how you handle time-based queries, factoring in publication dates when relevant. - -By modeling time filters with Pydantic and leveraging Instructor, RAG systems can effectively handle time-based queries. Clear prompts, careful model design, and appropriate parsing strategies enable accurate retrieval of information within specific time frames, enhancing the system's overall relevance and accuracy. diff --git a/참고/instructor-main/docs/blog/posts/semantic-validation-structured-outputs.md b/참고/instructor-main/docs/blog/posts/semantic-validation-structured-outputs.md deleted file mode 100644 index 5dd2bf0..0000000 --- a/참고/instructor-main/docs/blog/posts/semantic-validation-structured-outputs.md +++ /dev/null @@ -1,348 +0,0 @@ ---- -authors: -- jxnl -categories: -- Validation -- Pydantic -- LLMs -comments: true -date: 2025-05-20 -description: Learn how semantic validation with LLMs can ensure your structured outputs meet complex, subjective, and contextual criteria beyond what traditional rule-based validation can achieve. -draft: false -tags: -- Semantic Validation -- Structured Outputs -- LLM Validator -- Pydantic -- Data Quality ---- - -# Understanding Semantic Validation with Structured Outputs - -> Semantic validation uses LLMs to evaluate content against complex, subjective, and contextual criteria that would be difficult to implement with traditional rule-based validation approaches. - -As LLMs become increasingly integrated into production systems, ensuring the quality and safety of their outputs is paramount. Traditional validation methods relying on explicit rules can't keep up with the complexity and nuance of natural language. With the release of Instructor's semantic validation capabilities, we now have a powerful way to validate structured outputs against sophisticated criteria. - - - -## Beyond Rule-Based Validation - -Traditional validation approaches focus on verifying that data conforms to certain rules-ensuring that: - -- A field has the correct type (`int`, `str`, etc.) -- A value falls within predefined ranges (e.g., `age >= 0`) -- A pattern matches expected formats (e.g., email regex) - -These approaches work well for structured data with clear constraints but fall short when validating natural language against less precise criteria like: - -- "Content must be family-friendly" -- "Description must be professional and free of hyperbole" -- "Criticism must be constructive and respectful" -- "Message must adhere to community guidelines" - -This is where semantic validation with LLMs comes in. - -## What is Semantic Validation? - -Semantic validation uses an LLM to interpret and evaluate text against natural language criteria. Instead of writing explicit rules, you express validation requirements in plain language, and the LLM determines whether content meets those requirements. - -Let's see how this works with Instructor's `llm_validator`: - -```python -from typing import Annotated -from pydantic import BaseModel, BeforeValidator -import instructor -from instructor import llm_validator - -# Initialize client -client = instructor.from_provider("openai/gpt-5-nano") - - -class ProductDescription(BaseModel): - name: str - description: Annotated[ - str, - BeforeValidator( - llm_validator( - """The description must be: - 1. Professional and factual - 2. Free of excessive hyperbole or unsubstantiated claims - 3. Between 50-200 words in length - 4. Written in third person (no "you" or "your") - 5. Free of spelling and grammar errors""", - client=client, - ) - ), - ] -``` - -What makes this approach powerful is that we're leveraging the LLM's understanding of language and context to perform validation that would be extremely difficult to implement with traditional approaches. - -## When to Use Semantic Validation - -Semantic validation shines in situations where: - -1. **Criteria is complex or subjective**: "Ensure this content is respectful" requires understanding nuance that's difficult to capture in rules. - -2. **Context matters**: "The summary must accurately reflect the key findings" requires comparing multiple pieces of content. - -3. **The rules are constantly evolving**: Harmful content strategies change as bad actors adapt, making static rules obsolete quickly. - -4. **Human-like judgment is required**: "This product description should be compelling without being misleading" requires nuanced evaluation. - -## Real-World Examples - -### Content Moderation - -One of the most obvious applications is content moderation. Companies need to ensure user-generated content meets community guidelines without being overly restrictive: - -```python -class UserComment(BaseModel): - user_id: str - content: Annotated[ - str, - BeforeValidator( - llm_validator( - """Content must comply with community guidelines: - - No hate speech, harassment, or discrimination - - No explicit sexual or violent content - - No promotion of illegal activities - - No sharing of personal information - - No spamming or excessive self-promotion""", - client=client, - ) - ), - ] -``` - -### Tone and Style Enforcement - -Organizations often need to maintain a consistent tone and style in their communications: - -```python -class CompanyAnnouncement(BaseModel): - title: str - content: Annotated[ - str, - BeforeValidator( - llm_validator( - "The announcement must maintain a professional, positive tone without being overly informal or using slang", - client=client, - ) - ), - ] -``` - -### Fact-Checking - -For applications where factual accuracy is critical: - -```python -class FactCheckedClaim(BaseModel): - claim: str - is_accurate: bool - supporting_evidence: list[str] - - @classmethod - def validate_claim(cls, text: str) -> "FactCheckedClaim": - return client.create( - response_model=cls, - messages=[ - { - "role": "system", - "content": "You are a fact-checking system. Assess the factual accuracy of the claim.", - }, - {"role": "user", "content": "Fact check this claim: {{ claim }}"}, - ], - context={"claim": text}, - ) -``` - -## Beyond Field Validation: Model-Level Semantic Validation - -While field-level validation is powerful, sometimes we need to validate relationships between fields. This is where model-level semantic validation becomes useful: - -```python -class Report(BaseModel): - title: str - summary: str - key_findings: list[str] - - @model_validator(mode='after') - def validate_consistency(self): - # Semantic validation at the model level using Jinja templating - validation_result = client.create( - response_model=Validator, - messages=[ - { - "role": "system", - "content": "Validate that the summary accurately reflects the key findings.", - }, - { - "role": "user", - "content": """ - Please validate if this summary accurately reflects the key findings: - - Title: {{ title }} - Summary: {{ summary }} - - Key findings: - {% for finding in findings %} - - {{ finding }} - {% endfor %} - - Evaluate for consistency, completeness, and accuracy. - """, - }, - ], - context={ - "title": self.title, - "summary": self.summary, - "findings": self.key_findings, - }, - ) - - if not validation_result.is_valid: - raise ValueError(f"Consistency error: {validation_result.reason}") - - return self -``` - -## Technical Implementation - -Under the hood, the `llm_validator` uses a special `Validator` model that determines whether content meets the criteria and provides detailed error messages when it doesn't: - -```python -class Validator(BaseModel): - is_valid: bool - reason: Optional[str] = None - fixed_value: Optional[str] = None -``` - -When validation fails, the reason field contains a detailed explanation, which is perfect for both developers debugging issues and for automatic retry mechanisms. - -## Self-Healing with Retries - -One of the most powerful features of Instructor's validation system is its ability to automatically retry with error context: - -```python -try: - product = client.create( - response_model=ProductDescription, - messages=[ - {"role": "system", "content": "Generate a product description."}, - { - "role": "user", - "content": "Create a description for UltraClean 9000 Washing Machine", - }, - ], - max_retries=2, # Automatically retry up to 2 times with error context - ) - print("Success:", product.model_dump_json(indent=2)) -except Exception as e: - print(f"Failed after retries: {e}") - #> Failed after retries: name 'client' is not defined -``` - -With `max_retries` set, if the initial response fails validation, Instructor will automatically send the error context back to the LLM, giving it a chance to correct the issue. This creates a self-healing system that can recover from validation failures without developer intervention. - -## Performance and Cost Considerations - -Semantic validation adds an additional API call for each validation, which impacts: - -1. **Latency**: Each validation requires an LLM inference -2. **Cost**: More API calls mean higher usage costs -3. **Reliability**: Depends on LLM API availability - -For high-throughput applications, consider these strategies: - -- **Batch validations**: Validate multiple items in a single call where possible -- **Strategic placement**: Apply semantic validation at critical points rather than everywhere -- **Caching**: Cache validation results for identical or similar content -- **Use the right model**: `gpt-4o-mini` or similar models offer a good balance of capability and cost for many validation scenarios - -## Building a Layered Validation Strategy - -The most robust approach combines traditional validation with semantic validation: - -1. **Type validation**: Use Pydantic's built-in type validation as your first defense -2. **Rule-based validation**: Apply explicit rules where they make sense -3. **Semantic validation**: Reserve LLM-based validation for complex criteria - -This layered approach ensures you get the benefits of semantic validation without unnecessary API calls for simple validations. - -## Advanced Applications - -### Custom Guardrails Framework - -You can build a comprehensive guardrails framework by combining semantic validators: - -```python -def create_guarded_model(base_class, guardrails): - """Create a model with multiple semantic guardrails applied.""" - validators = {} - - for field_name, criteria in guardrails.items(): - validators[field_name] = Annotated[ - str, BeforeValidator(llm_validator(criteria, client=client)) - ] - - return create_model( - f"Guarded{base_class.__name__}", __base__=base_class, **validators - ) - - -# Usage -guardrails = { - "title": "Must be concise, descriptive, and free of clickbait", - "content": "Must follow community guidelines and be respectful", -} - -GuardedPost = create_guarded_model(Post, guardrails) -``` - -### Contextual Validation with External References - -For validations that require external knowledge: - -```python -class LegalCompliance(BaseModel): - document: str - compliance_status: Annotated[ - str, - BeforeValidator( - llm_validator( - """Check if this document complies with the provided guidelines. - Guidelines: {{ guidelines }}""", - client=client, - ) - ), - ] - - -# Usage -result = client.create( - response_model=LegalCompliance, - messages=[{"role": "user", "content": "Check this document: " + document_text}], - context={"guidelines": company_legal_guidelines}, -) -``` - -## Conclusion - -Semantic validation represents a significant advancement in ensuring the quality and safety of LLM outputs. By combining the flexibility of natural language criteria with the structured validation of Pydantic, we can build systems that are both powerful and safe. - -As these techniques mature, we can expect to see semantic validation become a standard part of AI application development, especially in regulated industries where output quality is critical. - -To get started with semantic validation in your projects, check out the [Semantic Validation documentation](https://python.useinstructor.com../../concepts/semantic_validation/.md) and explore the various examples and patterns. - -This approach isn't just a technical improvement-it's a fundamental shift in how we think about validation, moving from rigid rules to intelligent understanding of content and context. - -## Related Documentation -- [Validation Fundamentals](../../concepts/validation.md) - Core validation concepts -- [Semantic Validation](../../concepts/semantic_validation.md) - Using LLMs for validation - -## See Also -- [Validation Deep Dive](validation-part1.md) - Foundation validation concepts -- [Anthropic Prompt Caching](anthropic-prompt-caching.md) - Optimize validation costs -- [Monitoring with Logfire](logfire.md) - Track validation performance \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/situate-context.md b/참고/instructor-main/docs/blog/posts/situate-context.md deleted file mode 100644 index 271704d..0000000 --- a/참고/instructor-main/docs/blog/posts/situate-context.md +++ /dev/null @@ -1,193 +0,0 @@ ---- -authors: - - jxnl -categories: - - Anthropic - - LLM Techniques - - Python -comments: true -date: 2024-09-26 -description: - Learn to implement Anthropic's Contextual Retrieval with async processing - to enhance RAG systems and preserve crucial context efficiently. -draft: false -tags: - - Contextual Retrieval - - Async Processing - - RAG Systems - - Performance Optimization - - Document Chunking ---- - -# Implementing Anthropic's Contextual Retrieval with Async Processing - -Anthropic's [Contextual Retrieval](https://www.anthropic.com/blog/contextual-retrieval-for-rag) technique enhances RAG systems by preserving crucial context. - -This post examines the method and demonstrates an efficient implementation using async processing. We'll explore how to optimize your RAG applications with this approach, building on concepts from our [async processing guide](./learn-async.md). - - - -## Background: The Context Problem in RAG - -Anthropic identifies a key issue in traditional RAG systems: loss of context when documents are split into chunks. They provide an example: - -"Imagine you had a collection of financial information (say, U.S. SEC filings) embedded in your knowledge base, and you received the following question: 'What was the revenue growth for ACME Corp in Q2 2023?' - -A relevant chunk might contain the text: 'The company's revenue grew by 3% over the previous quarter.' However, this chunk on its own doesn't specify which company it's referring to or the relevant time period." - -## Anthropic's Solution: Contextual Retrieval - -Contextual Retrieval solves this by adding chunk-specific explanatory context before embedding. Anthropic's example: - -``` -original_chunk = "The company's revenue grew by 3% over the previous quarter." - -contextualized_chunk = "This chunk is from an SEC filing on ACME corp's performance in Q2 2023; the previous quarter's revenue was $314 million. The company's revenue grew by 3% over the previous quarter." -``` - -## Implementing Contextual Retrieval - -Anthropic uses Claude to generate context. They provide this prompt: - -``` - -{{WHOLE_DOCUMENT}} - -Here is the chunk we want to situate within the whole document - -{{CHUNK_CONTENT}} - -Please give a short succinct context to situate this chunk within the overall document for the purposes of improving search retrieval of the chunk. Answer only with the succinct context and nothing else. -``` - -## Performance Improvements - -Anthropic reports significant improvements: - -- Contextual Embeddings reduced top-20-chunk retrieval failure rate by 35% (5.7% → 3.7%). -- Combining Contextual Embeddings and Contextual BM25 reduced failure rate by 49% (5.7% → 2.9%). -- Adding reranking further reduced failure rate by 67% (5.7% → 1.9%). - -## Instructor implementation of Contextual Retrieval with Async Processing - -We can implement Anthropic's technique using async processing for improved efficiency: - -```python -from instructor import AsyncInstructor, Mode, patch -from anthropic import AsyncAnthropic -from pydantic import BaseModel, Field -import asyncio -from typing import List, Dict - - -class SituatedContext(BaseModel): - title: str = Field(..., description="The title of the document.") - context: str = Field( - ..., description="The context to situate the chunk within the document." - ) - - -client = AsyncInstructor( - create=patch( - create=AsyncAnthropic().beta.prompt_caching.messages.create, - mode=Mode.TOOLS, - ), - mode=Mode.TOOLS, -) - - -async def situate_context(doc: str, chunk: str) -> str: - response = await client.create( - model="claude-3-haiku-20240307", - max_tokens=1024, - temperature=0.0, - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "{{doc}}", - "cache_control": {"type": "ephemeral"}, - }, - { - "type": "text", - "text": "Here is the chunk we want to situate within the whole document\n{{chunk}}\nPlease give a short succinct context to situate this chunk within the overall document for the purposes of improving search retrieval of the chunk.\nAnswer only with the succinct context and nothing else.", - }, - ], - } - ], - response_model=SituatedContext, - context={"doc": doc, "chunk": chunk}, - ) - return response.context - - -def chunking_function(doc: str) -> List[str]: - chunk_size = 1000 - overlap = 200 - chunks = [] - start = 0 - while start < len(doc): - end = start + chunk_size - chunks.append(doc[start:end]) - start += chunk_size - overlap - return chunks - - -async def process_chunk(doc: str, chunk: str) -> Dict[str, str]: - context = await situate_context(doc, chunk) - return {"chunk": chunk, "context": context} - - -async def process(doc: str) -> List[Dict[str, str]]: - chunks = chunking_function(doc) - tasks = [process_chunk(doc, chunk) for chunk in chunks] - results = await asyncio.gather(*tasks) - return results - - -# Example usage -async def main(): - document = "Your full document text here..." - processed_chunks = await process(document) - for i, item in enumerate(processed_chunks): - print(f"Chunk {i + 1}:") - print(f"Text: {item['chunk'][:50]}...") - print(f"Context: {item['context']}") - print() - - -if __name__ == "__main__": - asyncio.run(main()) -``` - -## Key Features of This Implementation - -1. Async Processing: Uses `asyncio` for concurrent chunk processing. -2. Structured Output: Uses Pydantic models for type-safe responses. -3. Prompt Caching: Utilizes Anthropic's prompt caching for efficiency. -4. Chunking: Implements a basic chunking strategy with overlap. -5. Jinja2 templating: Uses Jinja2 templating to inject variables into the prompt. - -## Considerations from Anthropic's Article - -Anthropic mentions several implementation considerations: - -1. Chunk boundaries: Experiment with chunk size, boundary, and overlap. -2. Embedding model: They found Gemini and Voyage embeddings effective. -3. Custom contextualizer prompts: Consider domain-specific prompts. -4. Number of chunks: They found using 20 chunks most effective. -5. Evaluation: Always run evaluations on your specific use case. - -## Further Enhancements - -Based on Anthropic's suggestions: - -1. Implement dynamic chunk sizing based on content complexity. -2. Integrate with vector databases for efficient storage and retrieval. -3. Add error handling and retry mechanisms. -4. Experiment with different embedding models and prompts. -5. Implement a reranking step for further performance improvements. - -This implementation provides a starting point for leveraging Anthropic's Contextual Retrieval technique with the added efficiency of async processing. diff --git a/참고/instructor-main/docs/blog/posts/string-based-init.md b/참고/instructor-main/docs/blog/posts/string-based-init.md deleted file mode 100644 index 4031d09..0000000 --- a/참고/instructor-main/docs/blog/posts/string-based-init.md +++ /dev/null @@ -1,270 +0,0 @@ ---- -draft: false -date: 2024-04-20 -authors: - - jxnl -categories: - - Tutorial ---- - -# Unified Provider Interface with String-Based Initialization - -Instructor now offers a simplified way to initialize any supported LLM provider with a single consistent interface. This approach makes it easier than ever to switch between different LLM providers while maintaining the same structured output functionality you rely on. - -## The Problem - -As the number of LLM providers grows, so does the complexity of initializing and working with different client libraries. Each provider has its own initialization patterns, API structures, and quirks. This leads to code that isn't portable between providers and requires significant refactoring when you want to try a new model. - -## The Solution: String-Based Initialization - -We've introduced a new unified interface that allows you to initialize any supported provider with a simple string format: - -```python -import instructor -from pydantic import BaseModel - - -class UserInfo(BaseModel): - name: str - age: int - - -# Initialize any provider with a single consistent interface -client = instructor.from_provider("openai/gpt-4") -client = instructor.from_provider("anthropic/claude-3-sonnet") -client = instructor.from_provider("google/gemini-pro") -client = instructor.from_provider("mistral/mistral-large") -``` - -The `from_provider` function takes a string in the format `"provider/model-name"` and handles all the details of setting up the appropriate client with the right model. This provides several key benefits: - -- **Simplified Initialization**: No need to manually create provider-specific clients -- **Consistent Interface**: Same syntax works across all providers -- **Reduced Dependency Exposure**: You don't need to import specific provider libraries in your application code -- **Easy Experimentation**: Switch between providers with a single line change - -## Supported Providers - -The string-based initialization currently supports all major providers in the ecosystem: - -- OpenAI: `"openai/gpt-4"`, `"openai/gpt-4o"`, `"openai/gpt-5-nano"` -- Anthropic: `"anthropic/claude-3-opus-20240229"`, `"anthropic/claude-3-sonnet-20240229"`, `"anthropic/claude-3-5-haiku-latest"` -- Google Gemini: `"google/gemini-pro"`, `"google/gemini-pro-vision"` -- Mistral: `"mistral/mistral-small-latest"`, `"mistral/mistral-medium-latest"`, `"mistral/mistral-large-latest"` -- Cohere: `"cohere/command"`, `"cohere/command-r"`, `"cohere/command-light"` -- Perplexity: `"perplexity/sonar-small-online"`, `"perplexity/sonar-medium-online"` -- Groq: `"groq/llama2-70b-4096"`, `"groq/mixtral-8x7b-32768"`, `"groq/gemma-7b-it"` -- Writer: `"writer/palmyra-instruct"`, `"writer/palmyra-instruct-v2"` -- AWS Bedrock: `"bedrock/anthropic.claude-v2"`, `"bedrock/amazon.titan-text-express-v1"` -- Cerebras: `"cerebras/cerebras-gpt"`, `"cerebras/cerebras-gpt-2.7b"` -- Fireworks: `"fireworks/llama-v2-70b"`, `"fireworks/firellama-13b"` -- Vertex AI: `"vertexai/gemini-pro"`, `"vertexai/text-bison"` -- Google GenAI: `"genai/gemini-pro"`, `"genai/gemini-pro-vision"` - -Each provider will be initialized with sensible defaults, but you can also pass additional keyword arguments to customize the configuration. For model-specific details, consult each provider's documentation. - -## Async Support - -The unified interface fully supports both synchronous and asynchronous clients: - -```python -# Synchronous client (default) -client = instructor.from_provider("openai/gpt-4") - -# Asynchronous client -async_client = instructor.from_provider("anthropic/claude-3-sonnet", async_client=True) - -# Use like any other async client -response = await async_client.create( - response_model=UserInfo, - messages=[ - { - "role": "user", - "content": "Extract information about John who is 30 years old", - } - ], -) -``` - -## Mode Selection - -You can also specify which structured output mode to use with the provider: - -```python -import instructor -from instructor import Mode - -# Override the default mode for a provider -client = instructor.from_provider( - "anthropic/claude-3-sonnet", mode=Mode.TOOLS -) - -# Use JSON mode instead of the default tools mode -client = instructor.from_provider( - "mistral/mistral-large", mode=Mode.JSON_SCHEMA -) - -# Use reasoning tools instead of regular tools for Anthropic -client = instructor.from_provider( - "anthropic/claude-3-opus", mode=Mode.TOOLS -) -``` - -If not specified, each provider will use its recommended default mode: - -- OpenAI: `Mode.OPENAI_FUNCTIONS` -- Anthropic: `Mode.TOOLS` -- Google Gemini: `Mode.MD_JSON` -- Mistral: `Mode.TOOLS` -- Cohere: `Mode.TOOLS` -- Perplexity: `Mode.JSON` -- Groq: `Mode.GROQ_TOOLS` -- Writer: `Mode.MD_JSON` -- Bedrock: `Mode.TOOLS` (for Claude on Bedrock) -- Vertex AI: `Mode.TOOLS` - -You can always customize this based on your specific needs and model capabilities. - -## Error Handling - -The `from_provider` function includes robust error handling to help you quickly identify and fix issues: - -```python -# Missing dependency -try: - client = instructor.from_provider("anthropic/claude-3-sonnet") -except ImportError as e: - print("Error: Install the anthropic package first") - # pip install anthropic - -# Invalid provider format -try: - client = instructor.from_provider("invalid-format") -except ValueError as e: - print(e) # Model string must be in format "provider/model-name" - -# Unsupported provider -try: - client = instructor.from_provider("unknown/model") -except ValueError as e: - print(e) # Unsupported provider: unknown. Supported providers are: ... -``` - -The function validates the provider string format, checks if the provider is supported, and ensures the necessary packages are installed. - -## Environment Variables - -Like the native client libraries, `from_provider` respects environment variables set for each provider: - -```python -# Set environment variables -import os - -os.environ["OPENAI_API_KEY"] = "your-openai-key" -os.environ["ANTHROPIC_API_KEY"] = "your-anthropic-key" -os.environ["MISTRAL_API_KEY"] = "your-mistral-key" - -# No need to pass API keys directly -client = instructor.from_provider("openai/gpt-4") -``` - -## Troubleshooting - -Here are some common issues and solutions when using the unified provider interface: - -### Model Not Found Errors - -If you receive a 404 error, check that you're using the correct model name format: - -``` -Error code: 404 - {'type': 'error', 'error': {'type': 'not_found_error', 'message': 'model: claude-3-haiku'}} -``` - -For Anthropic models, always include the version date: -- ✅ Correct: `anthropic/claude-3-haiku-20240307` -- ❌ Incorrect: `anthropic/claude-3-haiku` - -### Provider-Specific Parameters - -Some providers require specific parameters for API calls: - -```python -# Anthropic requires max_tokens -anthropic_client = instructor.from_provider( - "anthropic/claude-3-5-haiku-latest", max_tokens=400 # Required for Anthropic -) - -# Use models with vision capabilities for multimodal content -gemini_client = instructor.from_provider( - "google/gemini-pro-vision" # Required for image processing -) -``` - -### Working Example - -Here's a complete example that demonstrates the automodel functionality with multiple providers: - -```python -import os -import asyncio -import instructor -from pydantic import BaseModel, Field - - -class UserInfo(BaseModel): - """User information extraction model.""" - - name: str = Field(description="The user's full name") - age: int = Field(description="The user's age in years") - occupation: str = Field(description="The user's job or profession") - - -async def main(): - # Test OpenAI - openai_client = instructor.from_provider("openai/gpt-5-nano") - openai_result = openai_client.create( - response_model=UserInfo, - messages=[ - {"role": "user", "content": "Jane Doe is a 28-year-old data scientist."} - ], - ) - print(f"OpenAI result: {openai_result.model_dump()}") - - # Test Anthropic with async client - if os.environ.get("ANTHROPIC_API_KEY"): - anthropic_client = instructor.from_provider( - model="anthropic/claude-3-5-haiku-latest", - async_client=True, - max_tokens=400, # Required for Anthropic - ) - anthropic_result = await anthropic_client.create( - response_model=UserInfo, - messages=[ - { - "role": "user", - "content": "John Smith is a 35-year-old software engineer.", - } - ], - ) - print(f"Anthropic result: {anthropic_result.model_dump()}") - - -if __name__ == "__main__": - asyncio.run(main()) -``` - -## Conclusion - -String-based initialization is a significant step toward making Instructor even more user-friendly and flexible. It reduces the learning curve for working with multiple providers and makes it easier than ever to experiment with different models. - -Benefits include: -- Simplified initialization with a consistent interface -- Automatic selection of appropriate default modes -- Support for both synchronous and asynchronous clients -- Clear error messages to quickly identify issues -- Respect for provider-specific environment variables -- Comprehensive model selection across the entire LLM ecosystem - -Whether you're building a new application or migrating an existing one, the unified provider interface offers a cleaner, more maintainable way to work with structured outputs across the LLM ecosystem. - -Try it today with `instructor.from_provider()` and check out the [complete example code](https://github.com/instructor-ai/instructor/tree/main/examples/automodel) in our repository! \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/structured-output-anthropic.md b/참고/instructor-main/docs/blog/posts/structured-output-anthropic.md deleted file mode 100644 index 044cf1d..0000000 --- a/참고/instructor-main/docs/blog/posts/structured-output-anthropic.md +++ /dev/null @@ -1,149 +0,0 @@ ---- -authors: - - jxnl -categories: - - Anthropic -comments: true -date: 2024-10-23 -description: Learn how to leverage Anthropic's Claude with Instructor for structured outputs and prompt caching, enhancing AI application development. -draft: false -tags: - - Anthropic - - API Development - - Pydantic - - Python - - LLM Techniques - - Prompt Caching ---- - -# Structured Outputs and Prompt Caching with Anthropic - -Anthropic's ecosystem now offers two powerful features for AI developers: structured outputs and prompt caching. These advancements enable more efficient use of large language models (LLMs). This guide demonstrates how to leverage these features with the Instructor library to enhance your AI applications. - -## Structured Outputs with Anthropic and Instructor - -Instructor now offers seamless integration with Anthropic's powerful language models, allowing developers to easily create structured outputs using Pydantic models. This integration simplifies the process of extracting specific information from AI-generated responses. - - - -To get started, you'll need to install Instructor with Anthropic support: - -```bash -pip install instructor[anthropic] -``` - -Here's a basic example of how to use Instructor with Anthropic: - -```python -from pydantic import BaseModel -from typing import List -import anthropic -import instructor - -# Patch the Anthropic client with Instructor -anthropic_client = instructor.from_anthropic(create=anthropic.Anthropic()) - - -# Define your Pydantic models -class Properties(BaseModel): - name: str - value: str - - -class User(BaseModel): - name: str - age: int - properties: List[Properties] - - -# Use the patched client to generate structured output -user_response = anthropic_client( - model="claude-3-7-sonnet-latest", - max_tokens=1024, - messages=[ - { - "role": "user", - "content": "Create a user for a model with a name, age, and properties.", - } - ], - response_model=User, -) - -print(user_response.model_dump_json(indent=2)) -""" -{ - "name": "John Doe", - "age": 30, - "properties": [ - { "name": "favorite_color", "value": "blue" } - ] -} -""" -``` - -This approach allows you to easily extract structured data from Claude's responses, making it simpler to integrate AI-generated content into your applications. - -## Prompt Caching: Boosting Performance and Reducing Costs - -Anthropic has introduced a new prompt caching feature that can significantly improve response times and reduce costs for applications dealing with large context windows. This feature is particularly useful when making multiple calls with similar large contexts over time. - -Here's how you can implement prompt caching with Instructor and Anthropic: - -```python -from pydantic import BaseModel - -# Set up the client with prompt caching -client = instructor.from_provider("anthropic/claude-3-5-haiku-latest") - - -# Define your Pydantic model -class Character(BaseModel): - name: str - description: str - - -# Load your large context -with open("./book.txt") as f: - book = f.read() - -# Make multiple calls using the cached context -for _ in range(2): - resp, completion = client.create_with_completion( - model="claude-3-7-sonnet-latest", - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "" + book + "", - "cache_control": {"type": "ephemeral"}, - }, - { - "type": "text", - "text": "Extract a character from the text given above", - }, - ], - }, - ], - response_model=Character, - max_tokens=1000, - ) -``` - -In this example, the large context (the book content) is cached after the first request and reused in subsequent requests. This can lead to significant time and cost savings, especially when working with extensive context windows. - -## Conclusion - -By combining Anthropic's Claude with Instructor's structured output capabilities and leveraging prompt caching, developers can create more efficient, cost-effective, and powerful AI applications. These features open up new possibilities for building sophisticated AI systems that can handle complex tasks with ease. - -As the AI landscape continues to evolve, staying up-to-date with the latest tools and techniques is crucial. We encourage you to explore these features and share your experiences with the community. Happy coding! - -## Related Documentation -- [How Patching Works](../../concepts/patching.md) - Understand provider integration -- [Anthropic Integration](../../integrations/anthropic.md) - Complete setup guide - -## See Also -- [Anthropic Prompt Caching](anthropic-prompt-caching.md) - Optimize Anthropic costs -- [Unified Provider Interface](announcing-unified-provider-interface.md) - Switch providers easily -- [Framework Comparison](best_framework.md) - Why Instructor excels diff --git a/참고/instructor-main/docs/blog/posts/tidy-data-from-messy-tables.md b/참고/instructor-main/docs/blog/posts/tidy-data-from-messy-tables.md deleted file mode 100644 index a130635..0000000 --- a/참고/instructor-main/docs/blog/posts/tidy-data-from-messy-tables.md +++ /dev/null @@ -1,145 +0,0 @@ ---- -title: Using Structured Outputs to convert messy tables into tidy data -description: With instructor, converting messy tables into tidy data is easy and fast -categories: - - Data Analysis - - Structured Outputs -date: 2024-11-21 -draft: false ---- - -# Using Structured Outputs to convert messy tables into tidy data - -## Why is this a problem? - -Messy data exports are a common problem. Whether it's multiple headers in the table, implicit relationships that make analysis a pain or even just merged cells, using `instructor` with structured outputs makes it easy to convert messy tables into tidy data, even if all you have is just an image of the table as we'll see below. - -Let's look at the following table as an example. It makes analysis unnecessarily difficult because it hides data relationships through empty cells and implicit repetition. If we were using it for data analysis, cleaning it manually would be a huge nightmare. - - - -![](./img/untidy_table.png) - -For example, the subject ID (321) and GTT date only appear in the first row, with blank cells below implying these values apply to the following rows. This format breaks most pandas operations - you can't simply group by subject ID or merge with other datasets without complex preprocessing to fill in these missing values. - -Instead, we have time series measurements spread across multiple rows, mixed data types in the insulin column (numbers and "lo off curve"), and repeated subject information hidden through empty cells. This means even simple operations like calculating mean glucose levels by time point or plotting glucose curves require data reshaping and careful handling of missing/special values. - -## Using Structured Outputs - -### Defining a custom type - -Using tools like instructor to automatically convert untidy data into tidy format can save hours of preprocessing and reduce errors in your analysis pipeline. - -Let's start by first defining a custom type that can parse the markdown table into a pandas dataframe. - -```python -from io import StringIO -from typing import Annotated, Any -from pydantic import BeforeValidator, PlainSerializer, InstanceOf, WithJsonSchema -import pandas as pd - - -def md_to_df(data: Any) -> Any: - # Convert markdown to DataFrame - if isinstance(data, str): - return ( - pd.read_csv( - StringIO(data), # Process data - sep="|", - index_col=1, - ) - .dropna(axis=1, how="all") - .iloc[1:] - .applymap(lambda x: x.strip()) - ) - return data - - -MarkdownDataFrame = Annotated[ - InstanceOf[pd.DataFrame], - BeforeValidator(md_to_df), - PlainSerializer(lambda df: df.to_markdown()), - WithJsonSchema( - { - "type": "string", - "description": "The markdown representation of the table, each one should be tidy, do not try to join tables that should be separate", - } - ), -] -``` - -### Extracting the table - -Then with this new custom data type, it becomes easy to just pass the image to the LLM and get a tidy dataframe in response. - -```python -import instructor -from pydantic import BaseModel - - -class Table(BaseModel): - caption: str - dataframe: MarkdownDataFrame # Custom type for handling tables - - -class TidyTables(BaseModel): - tables: list[Table] - - -# Patch the OpenAI client with instructor -client = instructor.from_provider("openai/gpt-5-nano") - - -def extract_table(image_path: str) -> TidyTables: - return client.create( - model="gpt-4o-mini", - messages=[ - { - "role": "user", - "content": [ - "Convert this untidy table to tidy format", - instructor.Image.from_path(image_path), - ], - } - ], - response_model=TidyTables, - ) - - -extracted_tables = extract_table("./untidy_table.png") -``` - -This then returns the following output for us as a single pandas dataframe which we can easily plot and do any sort of data analysis on. - -| ID | GTT date | GTT weight | time | glucose mg/dl | insulin ng/ml | Comment | -| --- | -------- | ---------- | ---- | ------------- | ------------- | ------------ | -| 321 | 2/9/15 | 24.5 | 0 | 99.2 | | lo off curve | -| 321 | 2/9/15 | 24.5 | 5 | 349.3 | 0.205 | | -| 321 | 2/9/15 | 24.5 | 15 | 286.1 | 0.129 | | -| 321 | 2/9/15 | 24.5 | 30 | 312 | 0.175 | | -| 321 | 2/9/15 | 24.5 | 60 | 99.9 | 0.122 | | -| 321 | 2/9/15 | 24.5 | 120 | 217.9 | | lo off curve | -| 322 | 2/9/15 | 18.9 | 0 | 185.8 | 0.251 | | -| 322 | 2/9/15 | 18.9 | 5 | 297.4 | 2.228 | | -| 322 | 2/9/15 | 18.9 | 15 | 439 | 2.078 | | -| 322 | 2/9/15 | 18.9 | 30 | 362.3 | 0.775 | | -| 322 | 2/9/15 | 18.9 | 60 | 232.7 | 0.5 | | -| 322 | 2/9/15 | 18.9 | 120 | 260.7 | 0.523 | | -| 323 | 2/9/15 | 24.7 | 0 | 198.5 | 0.151 | | -| 323 | 2/9/15 | 24.7 | 5 | 530.6 | | off curve lo | - -More importantly, we can also extract multiple tables from a single image. This would be useful in helping to segment and identify different sections of a messy report. With tidy data, we get the benefits of - -1. Each variable being its own column -2. Each observation being its own row -3. Each value having its own cell -4. Seamlessly working with pandas/numpy operations -5. Visualization libraries "just working" - -## Conclusion - -We can actually go one step further and make this even tidier by converting things like weight, glucose and insulin into a specific column called metric which would allow us to add arbitrary metrics to the table without having to change the schema or our plotting code. This is a huge productivity boost when doing complex data analysis. - -No more wrestling with complex data cleaning pipelines. Let the model handle the heavy lifting while you focus on analysis. With instructor, getting to that step just became a whole lot easier. - -Give `instructor` a try today and see how you can build reliable applications. Just run `pip install instructor` or check out our [Getting Started Guide](../../index.md) diff --git a/참고/instructor-main/docs/blog/posts/timestamp.md b/참고/instructor-main/docs/blog/posts/timestamp.md deleted file mode 100644 index 11bb33e..0000000 --- a/참고/instructor-main/docs/blog/posts/timestamp.md +++ /dev/null @@ -1,159 +0,0 @@ ---- -authors: -- jxnl -categories: -- Pydantic -comments: true -date: 2024-09-26 -description: Learn how to ensure consistent timestamp formats in video content using - Pydantic for effective parsing and validation. -draft: false -slug: consistent-timestamp-formats -tags: -- timestamp -- Pydantic -- data validation -- video processing -- NLP ---- - -# Ensuring Consistent Timestamp Formats with Language Models - -Gemini can Understand timestamps in language model outputs, but they can be inconsistent. Video content timestamps vary between HH:MM:SS and MM:SS formats, causing parsing errors and calculations. This post presents a technique to handle timestamps for clips and films without formatting issues. - -We combine Pydantic's data validation with custom parsing for consistent timestamp handling. You'll learn to process timestamps in any format, reducing errors in video content workflows. Kinda like how we ensured [matching language in multilingal summarization](./matching-language.md) by adding a simple field. - -The post provides a solution using Pydantic to improve timestamp handling in language model projects. This method addresses format inconsistencies and enables timestamp processing. - - - -## The Problem - -Consider a scenario where we're using a language model to generate timestamps for video segments. For shorter videos, timestamps might be in MM:SS format, while longer videos require HH:MM:SS. This inconsistency can lead to parsing errors and incorrect time calculations. - -Here's a simple example of how this problem might manifest: - -```python -class Segment(BaseModel): - title: str = Field(..., description="The title of the segment") - timestamp: str = Field(..., description="The timestamp of the event as HH:MM:SS") - - -# This might work for some cases, but fails for others: -# "2:00" could be interpreted as 2 minutes or 2 hours -# "1:30:00" doesn't fit the expected format -``` - -This approach doesn't account for the variability in timestamp formats and can lead to misinterpretations. - -## The Solution - -To address this issue, we can use a combination of Pydantic for data validation and a custom parser to handle different timestamp formats. Here's how we can implement this: - -1. Define the expected time formats -2. Use a custom validator to parse and normalize the timestamps -3. Ensure the output is always in a consistent format - -Let's look at the improved implementation: - -```python -from pydantic import BaseModel, Field, model_validator -from typing import Literal - - -class SegmentWithTimestamp(BaseModel): - title: str = Field(..., description="The title of the segment") - time_format: Literal["HH:MM:SS", "MM:SS"] = Field( - ..., description="The format of the timestamp" - ) - timestamp: str = Field( - ..., description="The timestamp of the event as either HH:MM:SS or MM:SS" - ) - - @model_validator(mode="after") - def parse_timestamp(self): - if self.time_format == "HH:MM:SS": - hours, minutes, seconds = map(int, self.timestamp.split(":")) - elif self.time_format == "MM:SS": - hours, minutes, seconds = 0, *map(int, self.timestamp.split(":")) - else: - raise ValueError("Invalid time format, must be HH:MM:SS or MM:SS") - - # Normalize seconds and minutes - total_seconds = hours * 3600 + minutes * 60 + seconds - hours, remainder = divmod(total_seconds, 3600) - minutes, seconds = divmod(remainder, 60) - - if hours > 0: - self.timestamp = f"{hours:02d}:{minutes:02d}:{seconds:02d}" - else: - self.timestamp = f"00:{minutes:02d}:{seconds:02d}" - - return self -``` - -This implementation offers several advantages: - -1. It explicitly defines the expected time format, reducing ambiguity. -2. The custom validator parses the input based on the specified format. -3. It normalizes all timestamps to a consistent HH:MM:SS format. -4. It handles edge cases, such as when minutes or seconds exceed 59. - -## Why This Works Better Than Alternatives - -You might wonder why we can't solve this problem with constrained sampling methods or JSON schema alone. The reason is that timestamp parsing often requires context-aware processing that goes beyond simple pattern matching. - -1. **Constrained sampling** might enforce a specific format, but it doesn't handle the conversion between different formats or normalization of times. - -2. **JSON schema** can validate the structure of the data, but it can't perform the complex parsing and normalization required for timestamps. - -Our approach combines the strengths of schema validation (using Pydantic) with custom logic to handle the intricacies of timestamp formatting. - -## Testing the Solution - -To ensure our implementation works as expected, we can create some test cases: - -```python -if __name__ == "__main__": - # Test cases for SegmentWithTimestamp - test_cases = [ - ( - SegmentWithTimestamp( - title="Introduction", time_format="MM:SS", timestamp="00:30" - ), - "00:00:30", - ), - ( - SegmentWithTimestamp( - title="Main Topic", time_format="HH:MM:SS", timestamp="00:15:45" - ), - "00:15:45", - ), - ( - SegmentWithTimestamp( - title="Conclusion", time_format="MM:SS", timestamp="65:00" - ), - "01:05:00", - ), - ] - - for input_data, expected_output in test_cases: - try: - assert input_data.timestamp == expected_output - print(f"Test passed: {input_data.timestamp} == {expected_output}") - except AssertionError: - print(f"Test failed: {input_data.timestamp} != {expected_output}") - - # Output: - # Test passed: 00:00:30 == 00:00:30 - # Test passed: 00:15:45 == 00:15:45 - # Test passed: 01:05:00 == 01:05:00 -``` - -These test cases demonstrate that our solution correctly handles different input formats and normalizes them to a consistent output format. - -## Conclusion - -Parsing and validation are needed when handling language model outputs. Its not about coercing language models, but building valid inputs into downstream systems. Combining Pydantic's validation with logic ensures handling across formats. This approach solves timestamp inconsistency and provides a framework for challenges in NLP tasks. - -When dealing with time-based data in language models, account for format variability and implement validation and normalization to maintain consistency. \ No newline at end of file diff --git a/참고/instructor-main/docs/blog/posts/using_json.md b/참고/instructor-main/docs/blog/posts/using_json.md deleted file mode 100644 index 02b733b..0000000 --- a/참고/instructor-main/docs/blog/posts/using_json.md +++ /dev/null @@ -1,113 +0,0 @@ ---- -authors: - - jxnl -categories: - - LLM Techniques -comments: true -date: 2024-06-15 -description: - Learn how to easily get structured JSON data from LLMs using the Instructor - library with Pydantic models in Python. -draft: false -slug: zero-cost-abstractions -tags: - - Instructor - - JSON - - LLM - - Pydantic - - Python ---- - -# Why Instructor is the best way to get JSON from LLMs - -Large Language Models (LLMs) like GPT are incredibly powerful, but getting them to return well-formatted JSON can be challenging. This is where the Instructor library shines. Instructor allows you to easily map LLM outputs to JSON data using Python type annotations and Pydantic models. - -Instructor makes it easy to get structured data like JSON from LLMs like GPT-3.5, GPT-4, GPT-4-Vision, and open-source models including [Mistral/Mixtral](../../integrations/together.md), [Ollama](../../integrations/ollama.md), and [llama-cpp-python](../../integrations/llama-cpp-python.md). - -It stands out for its simplicity, transparency, and user-centric design, built on top of Pydantic. Instructor helps you manage [validation context](../../concepts/reask_validation.md), retries with [Tenacity](../../concepts/retrying.md), and streaming [Lists](../../concepts/lists.md) and [Partial](../../concepts/partial.md) responses. - -- Instructor provides support for a wide range of programming languages, including: - - [Python](https://python.useinstructor.com) - - [TypeScript](https://js.useinstructor.com) - - [Ruby](https://ruby.useinstructor.com) - - [Go](https://go.useinstructor.com) - - [Elixir](https://hex.pm/packages/instructor) - - - -## The Simple Patch for JSON LLM Outputs - -Instructor works as a lightweight patch over the OpenAI Python SDK. To use it, you simply apply the patch to your OpenAI client: - -```python -import instructor - -client = instructor.from_provider("openai/gpt-5-nano") -``` - -Then, you can pass a `response_model` parameter to the `completions.create` or `chat.completions.create` methods. This parameter takes in a Pydantic model class that defines the JSON structure you want the LLM output mapped to. Just like `response_model` when using FastAPI. - -Here's an example of a `response_model` for a simple user profile: - -```python -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - email: str - - -client = instructor.from_provider("openai/gpt-5-nano") - -user = client.create( - model="gpt-3.5-turbo", - response_model=User, - messages=[ - { - "role": "user", - "content": "Extract the user's name, age, and email from this: John Doe is 25 years old. His email is john@example.com", - } - ], -) - -print(user.model_dump()) -#> { -# "name": "John Doe", -# "age": 25, -# "email": "john@example.com" -# } -``` - -Instructor extracts the JSON data from the LLM output and returns an instance of your specified Pydantic model. You can then use the `model_dump()` method to serialize the model instance to a JSON string. - -Some key benefits of Instructor: - -- Zero new syntax to learn - it builds on standard Python type hints -- Seamless integration with existing OpenAI SDK code -- Incremental, zero-overhead adoption path -- Direct access to the `messages` parameter for flexible prompt engineering -- Broad compatibility with any OpenAI SDK-compatible platform or provider - -## Pydantic: More Powerful than Plain Dictionaries - -You might be wondering, why use Pydantic models instead of just returning a dictionary of key-value pairs? While a dictionary could hold JSON data, Pydantic models provide several powerful advantages: - -1. Type validation: Pydantic models enforce the types of the fields. If the LLM returns an incorrect type (e.g. a string for an int field), it will raise a validation error. - -2. Field requirements: You can mark fields as required or optional. Pydantic will raise an error if a required field is missing. - -3. Default values: You can specify default values for fields that aren't always present. - -4. Advanced types: Pydantic supports more advanced field types like dates, UUIDs, URLs, lists, nested models, and more. - -5. Serialization: Pydantic models can be easily serialized to JSON, which is helpful for saving results or passing them to other systems. - -6. IDE support: Because Pydantic models are defined as classes, IDEs can provide autocompletion, type checking, and other helpful features when working with the JSON data. - -So while dictionaries can work for very simple JSON structures, Pydantic models are far more powerful for working with complex, validated JSON in a maintainable way. - -## JSON from LLMs Made Easy - -Instructor and Pydantic together provide a fantastic way to extract and work with JSON data from LLMs. The lightweight patching of Instructor combined with the powerful validation and typing of Pydantic models makes it easy to integrate JSON outputs into your LLM-powered applications. Give Instructor a try and see how much easier it makes getting JSON from LLMs! diff --git a/참고/instructor-main/docs/blog/posts/validation-part1.md b/참고/instructor-main/docs/blog/posts/validation-part1.md deleted file mode 100644 index 03ad5ad..0000000 --- a/참고/instructor-main/docs/blog/posts/validation-part1.md +++ /dev/null @@ -1,494 +0,0 @@ ---- -authors: -- jxnl -- ivanleomk -categories: -- Pydantic -- Data Validation -- Python -comments: true -date: 2023-10-23 -description: Explore dynamic, machine learning-driven validation using Python's Pydantic - and Instructor to enhance software reliability. -draft: false -tags: -- LLM Validation -- Pydantic -- Python -- Machine Learning -- Software Development ---- - -# Good LLM Validation is Just Good Validation - -> What if your validation logic could learn and adapt like a human, but operate at the speed of software? This is the future of validation and it's already here. - -Validation is the backbone of reliable software. But traditional methods are static, rule-based, and can't adapt to new challenges. This post looks at how to bring dynamic, machine learning-driven validation into your software stack using Python libraries like `Pydantic` and `Instructor`. We validate these outputs using a validation function which conforms to the structure seen below. - -```python -def validation_function(value): - if condition(value): - raise ValueError("Value is not valid") - return mutation(value) -``` - - - -## What is Instructor? - -`Instructor` helps to ensure you get the exact response type you're looking for when using openai's function call api. Once you've defined the `Pydantic` model for your desired response, `Instructor` handles all the complicated logic in-between - from the parsing/validation of the response to the automatic retries for invalid responses. This means that we can build in validators 'for free' and have a clear separation of concerns between the prompt and the code that calls openai. - -```python -import instructor # pip install instructor -from pydantic import BaseModel - -# This enables response_model keyword -# from client.chat.completions.create -client = instructor.from_provider("openai/gpt-5-nano") # (1)! - - -class UserDetail(BaseModel): - name: str - age: int - - -user: UserDetail = client.create( - model="gpt-3.5-turbo", - response_model=UserDetail, - messages=[ - {"role": "user", "content": "Extract Jason is 25 years old"}, - ], - max_retries=3, # (2)! -) - -assert user.name == "Jason" # (3)! -assert user.age == 25 -``` - -1. To simplify your work with OpenAI models and streamline the extraction of Pydantic objects from prompts, we - offer a patching mechanism for the `ChatCompletion` class. - -2. Invalid responses that fail to be validated successfully will trigger up to as many reattempts as you define. - -3. As long as you pass in a `response_model` parameter to the `ChatCompletion` api call, the returned object will always - be a validated `Pydantic` object. - -In this post, we'll explore how to evolve from static, rule-based validation methods to dynamic, machine learning-driven ones. You'll learn to use `Pydantic` and `Instructor` to leverage language models and dive into advanced topics like content moderation, validating chain of thought reasoning, and contextual validation. - -Let's examine how these approaches with an example. Imagine that you run a software company that wants to ensure you never serve hateful and racist content. This isn't an easy job since the language around these topics change very quickly and frequently. - -## Software 1.0: Introduction to Validations in Pydantic - -A simple method could be to compile a list of different words that are often associated with hate speech. For simplicity, let's assume that we've found that the words `Steal` and `Rob` are good predictors of hateful speech from our database. We can modify our validation structure above to accommodate this. - -This will throw an error if we pass in a string like `Let's rob the bank!` or `We should steal from the supermarkets`. - -Pydantic offers two approaches for this validation: using the `field_validator` decorator or the `Annotated` hints. - -### Using `field_validator` decorator - -We can use the `field_validator` decorator to define a validator for a field in Pydantic. Here's a quick example of how we might be able to do so. - -```python -from pydantic import BaseModel, ValidationError, field_validator - - -class UserMessage(BaseModel): - message: str - - @field_validator('message') - def message_cannot_have_blacklisted_words(cls, v: str) -> str: - for word in v.split(): # (1)! - if word.lower() in {'rob', 'steal'}: - raise ValueError(f"`{word}` was found in the message `{v}`") - return v - - -try: - UserMessage(message="This is a lovely day") - UserMessage(message="We should go and rob a bank") -except ValidationError as e: - print(e) - """ - 1 validation error for UserMessage - message - Value error, `rob` was found in the message `We should go and rob a bank` [type=value_error, input_value='We should go and rob a bank', input_type=str] - For further information visit https://errors.pydantic.dev/2.11/v/value_error - """ -``` - -1. We split the sentence into its individual words and iterate through each of the words. We then try to see if any of these - words are in our blacklist which in this case is just `rob` and `steal` - -Since the message `This is a lovely day` does not have any blacklisted words, no errors are thrown. However, in the given example above, the validation fails for the message `We should go and rob a bank` due to the presence of the word `rob` and the corresponding error message is displayed. - -``` -1 validation error for UserMessage -message - Value error, `rob` was found in the message `We should go and rob a bank` [type=value_error, input_value='We should go and rob a bank', input_type=str] - For further information visit https://errors.pydantic.dev/2.4/v/value_error -``` - -### Using `Annotated` - -Alternatively, you can use the `Annotated` function to perform the same validation. Here's an example where we utilise the same function we started with. - -```python -from pydantic import BaseModel, ValidationError -from typing import Annotated -from pydantic.functional_validators import AfterValidator - - -def message_cannot_have_blacklisted_words(value: str): - for word in value.split(): - if word.lower() in {'rob', 'steal'}: - raise ValueError(f"`{word}` was found in the message `{value}`") - return value - - -class UserMessage(BaseModel): - message: Annotated[str, AfterValidator(message_cannot_have_blacklisted_words)] - - -try: - UserMessage(message="This is a lovely day") - UserMessage(message="We should go and rob a bank") -except ValidationError as e: - print(e) - """ - 1 validation error for UserMessage - message - Value error, `rob` was found in the message `We should go and rob a bank` [type=value_error, input_value='We should go and rob a bank', input_type=str] - For further information visit https://errors.pydantic.dev/2.11/v/value_error - """ -``` - -This code snippet achieves the same validation result. If the user message contains any of the words in the blacklist, a `ValueError` is raised and the corresponding error message is displayed. - -``` -1 validation error for UserMessage -message - Value error, `rob` was found in the message `We should go and rob a bank` [type=value_error, input_value='We should go and rob a bank', input_type=str] - For further information visit https://errors.pydantic.dev/2.4/v/value_error -``` - -Validation is a fundamental concept in software development and remains the same when applied to AI systems. Existing programming concepts should be leveraged when possible instead of introducing new terms and standards. The underlying principles of validation remain unchanged. - -Suppose now that we've gotten a new message - `Violence is always acceptable, as long as we silence the witness`. Our original validator wouldn't throw any errors when passed this new message since it uses neither the words `rob` or `steal`. However, it's clear that it is not a message which should be published. How can we ensure that our validation logic can adapt to new challenges? - -## Software 3.0: Validation for LLMs or powered by LLMs - -Building upon the understanding of simple field validators, let's delve into probabilistic validation in software 3.0, (prompt engineering). We'll introduce an LLM-powered validator called `llm_validator` that uses a statement to verify the value. - -We can get around this by using the inbuilt `llm_validator` class from `Instructor`. - -```python -from instructor import llm_validator -from pydantic import BaseModel, ValidationError -from typing import Annotated -from pydantic.functional_validators import AfterValidator - - -class UserMessage(BaseModel): - message: Annotated[ - str, AfterValidator(llm_validator("don't say objectionable things")) - ] - - -try: - UserMessage( - message="Violence is always acceptable, as long as we silence the witness" - ) -except ValidationError as e: - print(e) - """ - 1 validation error for UserMessage - message - Assertion failed, The statement promotes violence, which is objectionable. [type=assertion_error, input_value='Violence is always accep... we silence the witness', input_type=str] - For further information visit https://errors.pydantic.dev/2.6/v/assertion_error - """ -``` - -This produces the following error message as seen below - -``` -1 validation error for UserMessage -message - Assertion failed, The statement promotes violence, which is objectionable. [type=assertion_error, input_value='Violence is always accep... we silence the witness', input_type=str] - For further information visit https://errors.pydantic.dev/2.4/v/assertion_error -``` - -The error message is generated by the language model (LLM) rather than the code itself, making it helpful for re-asking the model in a later section. To better understand this approach, let's see how to build an `llm_validator` from scratch. - -### Creating Your Own Field Level `llm_validator` - -Building your own `llm_validator` can be a valuable exercise to get started with `Instructor` and create custom validators. - -Before we continue, let's review the anatomy of a validator: - -```python -def validation_function(value): - if condition(value): - raise ValueError("Value is not valid") - return value -``` - -As we can see, a validator is simply a function that takes in a value and returns a value. If the value is not valid, it raises a `ValueError`. We can represent this using the following structure: - -```python -class Validation(BaseModel): - is_valid: bool = Field( - ..., description="Whether the value is valid based on the rules" - ) - error_message: Optional[str] = Field( - ..., - description="The error message if the value is not valid, to be used for re-asking the model", - ) -``` - -Using this structure, we can implement the same logic as before and utilize `Instructor` to generate the validation. - -```python -import instructor - -# Enables `response_model` and `max_retries` parameters -client = instructor.from_provider("openai/gpt-5-nano") - - -def validator(v): - statement = "don't say objectionable things" - resp = client.create( - model="gpt-3.5-turbo", - messages=[ - { - "role": "system", - "content": "You are a validator. Determine if the value is valid for the statement. If it is not, explain why.", - }, - { - "role": "user", - "content": f"Does `{v}` follow the rules: {statement}", - }, - ], - # this comes from client = instructor.from_provider("openai/gpt-5-nano") - response_model=Validation, # (1)! - ) - if not resp.is_valid: - raise ValueError(resp.error_message) - return v -``` - -1. The new parameter of `response_model` comes from `client = instructor.from_provider("openai/gpt-5-nano")` and does not exist in the original OpenAI SDK. This - allows us to pass in the `Pydantic` model that we want as a response. - -Now we can use this validator in the same way we used the `llm_validator` from `Instructor`. - -```python -class UserMessage(BaseModel): - message: Annotated[str, AfterValidator(validator)] -``` - -## Writing more complex validations - -### Validating Chain of Thought - -A popular way of prompting large language models nowadays is known as chain of thought. This involves getting a model to generate reasons and explanations for an answer to a prompt. - -We can utilise `Pydantic` and `Instructor` to perform a validation to check if the reasoning is reasonable, given both the answer and the chain of thought. To do this we can't build a field validator since we need to access multiple fields in the model. Instead we can use a model validator. - -```python -def validate_chain_of_thought(values): - chain_of_thought = values["chain_of_thought"] - answer = values["answer"] - resp = client.create( - model="gpt-3.5-turbo", - messages=[ - { - "role": "system", - "content": "You are a validator. Determine if the value is valid for the statement. If it is not, explain why.", - }, - { - "role": "user", - "content": f"Verify that `{answer}` follows the chain of thought: {chain_of_thought}", - }, - ], - # this comes from client = instructor.from_provider("openai/gpt-5-nano") - response_model=Validation, - ) - if not resp.is_valid: - raise ValueError(resp.error_message) - return values -``` - -We can then take advantage of the `model_validator` decorator to perform a validation on a subset of the model's data. - -> We're defining a model validator here which runs before `Pydantic` parses the input into its respective fields. That's why we have a **before** keyword used in the `model_validator` class. - -```python -from pydantic import BaseModel, model_validator - - -class AIResponse(BaseModel): - chain_of_thought: str - answer: str - - @model_validator(mode='before') - @classmethod - def chain_of_thought_makes_sense(cls, data: Any) -> Any: - # here we assume data is the dict representation of the model - # since we use 'before' mode. - return validate_chain_of_thought(data) -``` - -Now, when you create a `AIResponse` instance, the `chain_of_thought_makes_sense` validator will be invoked. Here's an example: - -```python -try: - resp = AIResponse(chain_of_thought="1 + 1 = 2", answer="The meaning of life is 42") -except ValidationError as e: - print(e) -``` - -If we create a `AIResponse` instance with an answer that does not follow the chain of thought, we will get an error. - -``` -1 validation error for AIResponse - Value error, The statement 'The meaning of life is 42' does not follow the chain of thought: 1 + 1 = 2. - [type=value_error, input_value={'chain_of_thought': '1 +... meaning of life is 42'}, input_type=dict] -``` - -### Validating Citations From Original Text - -Let's see a more concrete example. Let's say that we've asked our model a question about some text source and we want to validate that the generated answer is supported by the source. This would allow us to minimize hallucinations and prevent statements that are not backed by the original text. While we could verify this by looking up the original source manually, a more scalable approach is to use a validator to do this automatically. - -We can pass in additional context to our validation functions using the `model_validate` function in `Pydantic` so that our models have more information to work with when performing validation. This context is a normal python dictionary and can be accessed inside the `info` argument in our validator functions. - -```python -from pydantic import ValidationInfo, BaseModel, field_validator - - -class AnswerWithCitation(BaseModel): - answer: str - citation: str - - @field_validator('citation') - @classmethod - def citation_exists(cls, v: str, info: ValidationInfo): # (1)! - context = info.context - if context: - context = context.get('text_chunk') - if v not in context: - raise ValueError(f"Citation `{v}` not found in text chunks") - return v -``` - -1. This `info` object corresponds to the value of `context` that we pass into the `model_validate` function as seen below. - -We can then take our original example and test it against our new model - -```python -try: - AnswerWithCitation.model_validate( - {"answer": "Jason is a cool guy", "citation": "Jason is cool"}, - context={"text_chunk": "Jason is just a guy"}, # (1)! - ) -except ValidationError as e: - print(e) -``` - -1. This `context` object is just a normal python dictionary and can take in and store any arbitrary values - -This in turn generates the following error since `Jason is cool` does not exist in the text `Jason is just a guy`. - -``` -1 validation error for AnswerWithCitation -citation -Value error, Citation `Jason is cool` not found in text chunks [type=value_error, input_value='Jason is cool', input_type=str] - For further information visit https://errors.pydantic.dev/2.4/v/value_error -``` - -## Putting it all together with `client = instructor.from_provider("openai/gpt-5-nano")` - -To pass this context from the `client.chat.completions.create` call, `client = instructor.from_provider("openai/gpt-5-nano")` also passes the `context`, which will be accessible from the `info` argument in the decorated validator functions. - -```python -import instructor - -# Enables `response_model` and `max_retries` parameters -client = instructor.from_provider("openai/gpt-5-nano") - - -def answer_question(question: str, text_chunk: str) -> AnswerWithCitation: - return client.create( - model="gpt-3.5-turbo", - messages=[ - { - "role": "user", - "content": f"Answer the question: {question} with the text chunk: {text_chunk}", - }, - ], - response_model=AnswerWithCitation, - context={"text_chunk": text_chunk}, - ) -``` - -## Error Handling and Re-Asking - -Validators can ensure certain properties of the outputs by throwing errors, in an AI system we can use the errors and allow language model to self correct. Then by running `client = instructor.from_provider("openai/gpt-5-nano")` not only do we add `response_model` and `context` it also allows you to use the `max_retries` parameter to specify the number of times to try and self correct. - -This approach provides a layer of defense against two types of bad outputs: - -1. Pydantic Validation Errors (code or LLM-based) -2. JSON Decoding Errors (when the model returns an incorrect response) - -### Define the Response Model with Validators - -To keep things simple let's assume we have a model that returns a `UserModel` object. We can define the response model using Pydantic and add a field validator to ensure that the name is in uppercase. - -```python -from pydantic import BaseModel, field_validator - - -class UserModel(BaseModel): - name: str - age: int - - @field_validator("name") - @classmethod - def validate_name(cls, v): - if v.upper() != v: - raise ValueError("Name must be in uppercase.") - return v -``` - -This is where the `max_retries` parameter comes in. It allows the model to self correct and retry the prompt using the error message rather than the prompt. - -```python -model = client.create( - model="gpt-3.5-turbo", - messages=[ - {"role": "user", "content": "Extract jason is 25 years old"}, - ], - # Powered by client = instructor.from_provider("openai/gpt-5-nano") - response_model=UserModel, - max_retries=2, -) - -assert model.name == "JASON" -``` - -In this example, even though there is no code explicitly transforming the name to uppercase, the model is able to correct the output. - -## Conclusion - -From the simplicity of Pydantic and Instructor to the dynamic validation capabilities of LLMs, the landscape of validation is changing but without needing to introduce new concepts. It's clear that the future of validation is not just about preventing bad data but about allowing llms to understand the data and correcting it. - -If you enjoy the content or want to try out `Instructor` please check out the [github](https://github.com/jxnl/instructor) and give us a star! - -## Related Documentation -- [Core Validation Concepts](../../concepts/validation.md) - Learn about validation fundamentals -- [Reask Validation](../../concepts/reask_validation.md) - Handle validation failures gracefully - -## See Also -- [Semantic Validation with Structured Outputs](semantic-validation-structured-outputs.md) - Next evolution in validation -- [Why Bad Schemas Break LLMs](bad-schemas-could-break-llms.md) - Schema design best practices -- [Pydantic Is Still All You Need](pydantic-is-still-all-you-need.md) - Why Pydantic validation matters diff --git a/참고/instructor-main/docs/blog/posts/version-1.md b/참고/instructor-main/docs/blog/posts/version-1.md deleted file mode 100644 index 429d8d4..0000000 --- a/참고/instructor-main/docs/blog/posts/version-1.md +++ /dev/null @@ -1,274 +0,0 @@ ---- -authors: -- jxnl -categories: -- OpenAI -comments: true -date: 2024-04-01 -description: 'Introducing instructor 1.0.0: Simplified API for OpenAI with improved - typing support, validation, and streamlined usability.' -draft: false -slug: announce-instructor-v1 -tags: -- API Development -- OpenAI -- Data Validation -- Python -- LLM Techniques ---- - -# Announcing instructor=1.0.0 - -Over the past 10 months, we've build up instructor with the [principle](../../why.md) of 'easy to try, and easy to delete'. We accomplished this by patching the openai client with the `instructor` package and adding new arguments like `response_model`, `max_retries`, and `context`. As a result I truly believe isntructor is the [best way](./best_framework.md) to get structured data out of llm apis. - -But as a result, we've been a bit stuck on getting typing to work well while giving you more control at development time. I'm excited to launch version 1.0.0 which cleans up the api w.r.t. typing without compromising the ease of use. - - - -## Growth - -Over the past 10 months, we've enjoyed healthy growth with over 4000+ github stars and 100+ contributors, and more importantly, 120k monthly downloads, and 20k unique monthly visitors with 500k requests per month to our docs - -![downloads](./img/downloads.png) - -## Whats new? - -Honestly, nothing much, the simplest change you'll need to make is to replace `instructor.patch` with `instructor.from_openai`. - -```python -import instructor - -client = instructor.from_provider("openai/gpt-5-nano") -``` - -Except now, any default arguments you want to place into the `create` call will be passed to the client. via kwargs. - -IF you know you want to pass in temperature, seed, or model, you can do so. - -```python -import openai -import instructor - -client = instructor.from_openai( - openai.OpenAI(), model="gpt-4-turbo-preview", temperature=0.2 -) -``` - -Now, whenever you call `client.chat.completions.create` the `model` and `temperature` will be passed to the openai client! - -## No new Standards - -When I first started working on this project, my goal was to ensure that we weren't introducing any new standards. Instead, our focus was on maintaining compatibility with existing ones. By creating our own client, we can seamlessly proxy OpenAI's `chat.completions.create` and Anthropic's `messages.create` methods. This approach allows us to provide a smooth upgrade path for your client, enabling support for all the latest models and features as they become available. Additionally, this strategy safeguards us against potential downstream changes. - -```python -import openai -import anthropic -import litellm -import instructor -from typing import TypeVar - -T = TypeVar("T") - -# These are all ways to create a client -client = instructor.from_provider("openai/gpt-5-nano") -client = instructor.from_provider("anthropic/claude-3-5-haiku-latest") -client = instructor.from_litellm(litellm.completion) - -# all of these will route to the same underlying create function -# allow you to add instructor to try it out, while easily removing it -client.create(model="gpt-4", response_model=type[T]) -> T -client.create(model="gpt-4", response_model=type[T]) -> T -client.messages.create(model="gpt-4", response_model=type[T]) -> T -``` - -## Type are inferred correctly - -This was the dream of instructor but due to the patching of openai, it wasnt possible for me to get typing to work well. Now, with the new client, we can get typing to work well! We've also added a few `create_*` methods to make it easier to create iterables and partials, and to access the original completion. - -### Calling `create` - -```python -import instructor -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -client = instructor.from_provider("openai/gpt-5-nano") - -user = client.create( - model="gpt-4-turbo-preview", - messages=[ - {"role": "user", "content": "Create a user"}, - ], - response_model=User, -) -``` - -Now if you use a ID, you can see the type is correctly inferred. - -![type](./img/type.png) - -### Handling async: `await create` - -This will also work correctly with asynchronous clients. - -```python -import instructor -from pydantic import BaseModel - - -client = instructor.from_provider("openai/gpt-5-nano", async_client=True) - - -class User(BaseModel): - name: str - age: int - - -async def extract(): - return await client.create( - model="gpt-4-turbo-preview", - messages=[ - {"role": "user", "content": "Create a user"}, - ], - response_model=User, - ) -``` - -Notice that simply because we return the `create` method, the `extract()` function will return the correct user type. - -![async](./img/async_type.png) - -### Returning the original completion: `create_with_completion` - -You can also return the original completion object - -```python -import instructor -from pydantic import BaseModel - - -client = instructor.from_provider("openai/gpt-5-nano") - - -class User(BaseModel): - name: str - age: int - - -user, completion = client.create_with_completion( - model="gpt-4-turbo-preview", - messages=[ - {"role": "user", "content": "Create a user"}, - ], - response_model=User, -) -``` - -![with_completion](./img/with_completion.png) - - -### Streaming Partial Objects: `create_partial` - -In order to handle streams, we still support `Iterable[T]` and `Partial[T]` but to simply the type inference, we've added `create_iterable` and `create_partial` methods as well! - -```python -import instructor -from pydantic import BaseModel - - -client = instructor.from_provider("openai/gpt-5-nano") - - -class User(BaseModel): - name: str - age: int - - -user_stream = client.create_partial( - model="gpt-4-turbo-preview", - messages=[ - {"role": "user", "content": "Create a user"}, - ], - response_model=User, -) - -for user in user_stream: - print(user) - #> name=None age=None - #> name=None age=None - #> name='' age=None - #> name='John' age=None - #> name='John Doe' age=None - #> name='John Doe' age=None - #> name='John Doe' age=None - #> name='John Doe' age=None - #> name='John Doe' age=30 - #> name='John Doe' age=30 - # name=None age=None - # name='' age=None - # name='John' age=None - # name='John Doe' age=None - # name='John Doe' age=30 -``` - -Notice now that the type inferred is `Generator[User, None]` - -![generator](./img/generator.png) - -### Streaming Iterables: `create_iterable` - -We get an iterable of objects when we want to extract multiple objects. - -```python -import instructor -from pydantic import BaseModel - - -client = instructor.from_provider("openai/gpt-5-nano") - - -class User(BaseModel): - name: str - age: int - - -users = client.create_iterable( - model="gpt-4-turbo-preview", - messages=[ - {"role": "user", "content": "Create 2 users"}, - ], - response_model=User, -) - -for user in users: - print(user) - #> name='John Doe' age=30 - #> name='Jane Doe' age=28 - # User(name='John Doe', age=30) - # User(name='Jane Smith', age=25) -``` - -![iterable](./img/iterable.png) - -## Validation and Error Handling - -Instructor has always supported validation and error handling. But now, we've added a new `context` argument to the `create` call. This allows you to pass in a `ValidationContext` object which will be passed to the `response_model`. This allows you to add custom validation logic to the `response_model`. - -If you want to learn more check out the docs on [retrying](../../concepts/retrying.md) and [reasking](../../concepts/reask_validation.md) - -## Support in multiple languages - -While each flavor is different the core philosophy is the same. Keeping it as close as possible to the common api allows us to support all the same features in all the same languages by hooking into each libraries's popular validation libraries. - -Check out: - -- [JavaScript](https://github.com/instructor-ai/instructor-js) -- [Elixir](https://github.com/instructor-ai/instructor-elixir) -- [PHP](https://github.com/cognesy/instructor-php) - -If you're interested in contributing, check out the [contributing guide](../../contributing.md), and you want to create instructor in your language, let [me](https://twitter.com/jxnlco) know and I can help with promotion and connecting all the docs! diff --git a/참고/instructor-main/docs/blog/posts/why-care-about-mcps.md b/참고/instructor-main/docs/blog/posts/why-care-about-mcps.md deleted file mode 100644 index 2f959cc..0000000 --- a/참고/instructor-main/docs/blog/posts/why-care-about-mcps.md +++ /dev/null @@ -1,185 +0,0 @@ ---- -title: Understanding Model Context Protocol (MCP) -date: 2025-03-27 -description: A comprehensive look at the Model Context Protocol (MCP), its architecture, benefits, and comparison with OpenAPI -authors: - - ivanleomk -tags: - - LLM - - MCP - - Standards ---- - -# What is MCP - -With [OpenAI joining Anthropic in supporting the Model Context Protocol (MCP)](https://x.com/sama/status/1904957253456941061), we're witnessing a unified standard for language models to interact with external systems. This creates exciting opportunities for multi-LLM architectures where specialized AI applications work in parallel-discovering tools, handing off tasks, and accessing powerful capabilities through standardized interfaces. - - - -## What is MCP and Why Does It Matter? - -MCP is an open protocol developed by Anthropic that standardizes how AI models and applications interact with external tools, data sources, and systems. It solves the fragmentation problem where teams build custom implementations for AI integrations by providing a standardized interface layer. - -There are three components to the MCP ecosystem: - -1. **Hosts**: Programs like Claude Desktop, IDEs, or AI tools that want to access data via MCP clients -2. **Clients**: Protocol clients that maintain 1:1 connections with servers -3. **Servers**: Lightweight programs that each expose specific capabilities through the standardized Model Context Protocol - -![MCP Architecture](./img/mcp_architecture.png) - -When interacting with Clients, Hosts have access to two primary options: **Tools**, which are model-controlled functions that retrieve or modify data, and **Resources**, which are application-controlled data like files. - -There's also the intention of eventually allowing servers themselves to have the capability of requesting completions/approval from Clients and Hosts while executing their tasks [through the `sampling` endpoint](https://modelcontextprotocol.io/docs../../concepts/sampling.md). - -### The Integration Problem MCP Solves - -Before MCP, integrating AI applications with external tools and systems created what's known as an "M×N problem". If you have M different AI applications (Claude, ChatGPT, custom agents, etc.) and N different tools/systems (GitHub, Slack, Asana, databases, etc.), you would need to build M×N different integrations. This leads to duplicated effort across teams, inconsistent implementations, and a maintenance burden that grows quadratically. - -MCP transforms this into an "M+N problem". Tool creators build N MCP servers (one for each system), while application developers build M MCP clients (one for each AI application). The total integration work becomes M+N instead of M×N. - -This means a team can build a GitHub MCP server once, and it will work with any MCP-compatible client. Similarly, once you've built an MCP-compatible agent, it can immediately work with all existing MCP servers without additional integration work. - -## Market Signals: Growing Adoption - -The adoption curve for MCP has been remarkably steep since its introduction. [Almost 3000 community-built MCP servers have emerged in just a few months](https://smithery.ai), showing the strong developer interest in this standard. Major platforms like Zed, Cursor, Perser, and Windsurf have become MCP Hosts, integrating the protocol into their core offerings. Companies including Cloudflare have released official [MCP support with features such as OAuth](https://blog.cloudflare.com/remote-model-context-protocol-servers-mcp/) for developers to start building great applications. - -![MCP Stars Growth](./img/mcp_stars.webp) - -With both OpenAI and Anthropic supporting MCP, we now have a unified approach spanning the two most advanced AI model providers. This critical mass suggests MCP is positioned to become the dominant standard for AI tool integration. - -## MCP vs OpenAPI Specification - -While MCP and OpenAPI are both standards for API interfaces, they have different purposes and approaches. Here's a simplified comparison of the key differences: - -| Aspect | OpenAPI Specification | Model Context Protocol (MCP) | -| ----------------- | ---------------------------------------------------- | --------------------------------------------------------------------------------- | -| **Primary Users** | Human developers interacting with web APIs | AI models and agents discovering and using tools | -| **Architecture** | Centralized specification in a single JSON/YAML file | Distributed system with hosts, clients, and servers allowing dynamic discovery | -| **Use Cases** | Documenting RESTful services for human consumption | Enabling AI models to autonomously find and use tools with semantic understanding | - -These two standards serve complementary purposes in the modern tech ecosystem. While OpenAPI excels at documenting traditional web services for human developers, MCP is purpose-built for the emerging AI agent landscape, providing rich semantic context that makes tools discoverable and usable by language models. - -Most organizations will likely maintain both: OpenAPI specifications for their developer-facing services and MCP interfaces for AI-enabled applications, creating bridges between these worlds as needed. - -## Getting Started With MCP Development - -The learning curve for MCP is relatively gentle-many servers are less than 200 lines of code and can be built in under an hour. Here are several ways you can start using MCP in existing environments: - -### Claude Desktop - -Claude Desktop now supports MCP integrations, allowing Claude to access up-to-date information through tools. You can add these MCPs by going to Claude's Settings and editing the configuration. - -![Claude Desktop MCP Settings](./img/claude_desktop_screenshot.png) - -For example, you can install Firecrawl's MCP using the following configuration: - -```json -{ - "mcpServers": { - "mcp-server-firecrawl": { - "command": "npx", - "args": ["-y", "firecrawl-mcp"], - "env": { - "FIRECRAWL_API_KEY": "YOUR_API_KEY_HERE" - } - } - } -} -``` - -This allows Claude to crawl websites and get up-to-date information: - -![Claude Desktop Using MCP](./img/claude_desktop_mcp.png) - -### Cursor Integration - -Cursor provides support for MCPs through a simple configuration file. Create a `.cursor/mcp.json` file with your desired MCP servers: - -```json -{ - "mcpServers": { - "github": { - "command": "npx", - "args": ["-y", "@modelcontextprotocol/server-github"], - "env": { - "GITHUB_PERSONAL_ACCESS_TOKEN": "" - } - } - } -} -``` - -Enable the MCP option in Cursor Settings: - -![Cursor MCP Support](./img/cursor_mcp_support.png) - -Then use Cursor's Agent with your MCP servers: - -![Cursor MCP Agent](./img/cursor_mcp_agent.png) - -In the example above, I've provided a simple github MCP to ask some questions about the issues from the `instructor-ai` repository. But you can really do a lot more, for instance, you can provide a `puppeteer` MCP to allow your model to interact with a web browser for instance to see how your frontend code looks like when it gets rendered to fix it automatically. - -### OpenAI Agent SDK - -OpenAI's Agent SDK now supports MCP servers using the `MCPServer` class, allowing you to connect agents to local tools and resources: - -```python -import asyncio -import shutil - -from agents import Agent, Runner, trace -from agents.mcp import MCPServer, MCPServerStdio - - -async def run(mcp_server: MCPServer, directory_path: str): - agent = Agent( - name="Assistant", - instructions=f"Answer questions about the git repository at {directory_path}, use that for repo_path", - mcp_servers=[mcp_server], - ) - - question = input("Enter a question: ") - - print("\n" + "-" * 40) - print(f"Running: {question}") - result = await Runner.run(starting_agent=agent, input=question) - print(result.final_output) - - message = "Summarize the last change in the repository." - print("\n" + "-" * 40) - print(f"Running: {message}") - result = await Runner.run(starting_agent=agent, input=message) - print(result.final_output) - - -async def main(): - # Ask the user for the directory path - directory_path = input("Please enter the path to the git repository: ") - - async with MCPServerStdio( - cache_tools_list=True, # Cache the tools list, for demonstration - params={"command": "uvx", "args": ["mcp-server-git"]}, - ) as server: - with trace(workflow_name="MCP Git Example"): - await run(server, directory_path) - - -if __name__ == "__main__": - if not shutil.which("uvx"): - raise RuntimeError( - "uvx is not installed. Please install it with `pip install uvx`." - ) - - asyncio.run(main()) -``` - -This allows the agent to understand local git repositories: - -![Agent MCP Example](./img/agent_mcp_example.png) - -## Conclusion - -For developers and organizations, the question isn't if you should build for MCPs but when. As the ecosystem matures, early adopters will have a significant advantage in integrating AI capabilities into their existing systems and workflows. This is especially true with the upcoming MCP registry by Anthropic, incoming support for remote MCP server hosting, and OAuth integrations that will help build richer and more personal integrations. - -The standardization provided by MCP will likely drive the next wave of AI integration, making it possible to build complex, multi-agent systems that leverage the best capabilities from different providers through a unified interface. diff --git a/참고/instructor-main/docs/blog/posts/writer-support.md b/참고/instructor-main/docs/blog/posts/writer-support.md deleted file mode 100644 index 8f14865..0000000 --- a/참고/instructor-main/docs/blog/posts/writer-support.md +++ /dev/null @@ -1,161 +0,0 @@ ---- -authors: - - ivanleomk - - yanomaly -categories: - - Writer SDK -comments: true -date: 2024-11-19 -description: Announcing Writer integration with Instructor for structured outputs and enterprise AI workflows -draft: false -slug: writer-support -tags: - - Writer - - Enterprise AI - - Integrations ---- - -# Structured Outputs with Writer now supported - -> - -We're excited to announce that `instructor` now supports [Writer](https://writer.com)'s enterprise-grade LLMs, including their latest Palmyra X 004 model. This integration enables structured outputs and enterprise AI workflows with Writer's powerful language models. - -## Getting Started - -First, make sure that you've signed up for an account on [Writer](https://app.writer.com/aistudio/signup?utm_campaign=devrel) and obtained an API key using this [quickstart guide](https://dev.writer.com/api-guides/quickstart). Once you've done so, install `instructor` with Writer support by running `pip install instructor[writer]` in your terminal. - -Make sure to set the `WRITER_API_KEY` environment variable with your Writer API key or pass it as an argument to the `Writer` constructor. - - - -```python -import instructor -from pydantic import BaseModel - -# Initialize Writer client -client = instructor.from_provider("writer/claude-3-5-sonnet-20241022") - - -class User(BaseModel): - name: str - age: int - - -# Extract structured data -user = client.create( - model="palmyra-x-004", - messages=[{"role": "user", "content": "Extract: John is 30 years old"}], - response_model=User, -) - -print(user) -#> name='John' age=30 -``` - -!!! note - - If you'd like to use the Async version of the Writer client, you can do so by using `instructor.from_provider("writer/claude-3-5-sonnet-20241022")`. - -We also support streaming with the Writer client using our `create_partial` method. This allows you to process responses incrementally as they arrive. - -This is particularly valuable for maintaining responsive applications and delivering a smooth user experience, especially when dealing with larger responses so that users can see immediate results. - -```python -import instructor -from pydantic import BaseModel - -# Initialize Writer client -client = instructor.from_provider("writer/claude-3-5-sonnet-20241022") - - -text_block = """ -In our recent online meeting, participants from various backgrounds joined to discuss the upcoming tech conference. The names and contact details of the participants were as follows: - -- Name: John Doe, Email: johndoe@email.com, Twitter: @TechGuru44 -- Name: Jane Smith, Email: janesmith@email.com, Twitter: @DigitalDiva88 -- Name: Alex Johnson, Email: alexj@email.com, Twitter: @CodeMaster2023 - -During the meeting, we agreed on several key points. The conference will be held on March 15th, 2024, at the Grand Tech Arena located at 4521 Innovation Drive. Dr. Emily Johnson, a renowned AI researcher, will be our keynote speaker. - -The budget for the event is set at $50,000, covering venue costs, speaker fees, and promotional activities. Each participant is expected to contribute an article to the conference blog by February 20th. - -A follow-up meetingis scheduled for January 25th at 3 PM GMT to finalize the agenda and confirm the list of speakers. -""" - - -class User(BaseModel): - name: str - email: str - twitter: str - - -class MeetingInfo(BaseModel): - date: str - location: str - budget: int - deadline: str - - -PartialMeetingInfo = instructor.Partial[MeetingInfo] - - -extraction_stream = client.create( - model="palmyra-x-004", - messages=[ - { - "role": "user", - "content": f"Get the information about the meeting and the users {text_block}", - }, - ], - response_model=PartialMeetingInfo, - stream=True, -) # type: ignore - - -for obj in extraction_stream: - print(obj) - #> date='March 15th, 2024' location='' budget=None deadline=None - #> date='March 15th, 2024' location='Grand Tech Arena, 4521 Innovation' budget=None deadline=None - #> date='March 15th, 2024' location='Grand Tech Arena, 4521 Innovation Drive' budget=50000 eadline='February 20th' -``` - -As with all our integrations, `instructor` ships with the ability to automatically retry requests that happen due to schema validation without you having to do anything. - -```python -import instructor -from typing import Annotated -from pydantic import BaseModel, AfterValidator, Field - -# Initialize Writer client -client = instructor.from_provider("writer/claude-3-5-sonnet-20241022") - - -# Example of model, that may require usage of retries -def uppercase_validator(v): - if v.islower(): - raise ValueError("Name must be in uppercase") - return v - - -class User(BaseModel): - name: Annotated[str, AfterValidator(uppercase_validator)] = Field( - ..., description="The name of the user" - ) - age: int - - -user = client.create( - model="palmyra-x-004", - messages=[{"role": "user", "content": "Extract: jason is 12"}], - response_model=User, - max_retries=3, -) - -print(user) -#> name='JASON' age=12 -``` - -This was a sneak peek into the things that you can do with Writer and `instructor` - from classification of text to sentimen analysis and more. - -We're excited to see what you build with `instructor` and Writer. If you have any other questions about writer, do check out the [Writer Documentation](https://dev.writer.com/introduction) for the API sdk. diff --git a/참고/instructor-main/docs/blog/posts/youtube-flashcards.md b/참고/instructor-main/docs/blog/posts/youtube-flashcards.md deleted file mode 100644 index b47591b..0000000 --- a/참고/instructor-main/docs/blog/posts/youtube-flashcards.md +++ /dev/null @@ -1,386 +0,0 @@ ---- -authors: -- jxnl -- zilto -categories: -- Data Processing -comments: true -date: 2024-10-18 -description: Flashcard generator application with Instructor + Burr -draft: false -slug: youtube-flashcards -tags: -- instructor -- Burr -- OpenAI -- LLM -- observability ---- - -# Flashcard generator with Instructor + Burr - -Flashcards help break down complex topics and learn anything from biology to a new -language or lines for a play. This blog will show how to use LLMs to generate -flashcards and kickstart your learning! - -**Instructor** lets us get structured outputs from LLMs reliably, and [Burr](https://github.com/dagworks-inc/burr) helps -create an LLM application that's easy to understand and debug. It comes with **Burr UI**, -a free, open-source, and local-first tool for observability, annotations, and more! - - - -??? info - - This post expands on an earlier one: [Analyzing Youtube Transcripts with Instructor](./youtube-transcripts.md/). - - -## Generate flashcards using LLMs with Instructor - -```bash -pip install openai instructor pydantic youtube_transcript_api "burr[start]" -``` - -### 1. Define the LLM response model - -With `instructor`, you define Pydantic models that will serve as template for the LLM to -fill. - -Here, we define the `QuestionAnswer` model which will store the question, the answer, and -some metadata. Attributes without a default value will be generated by the LLM. - -```python hl_lines="10-11 23 24-27" -import uuid - -from pydantic import BaseModel, Field -from pydantic.json_schema import SkipJsonSchema - - -class QuestionAnswer(BaseModel): - question: str = Field(description="Question about the topic") - options: list[str] = Field( - description="Potential answers to the question.", min_items=3, max_items=5 - ) - answer_index: int = Field( - description="Index of the correct answer options (starting from 0).", ge=0, lt=5 - ) - difficulty: int = Field( - description="Difficulty of this question from 1 to 5, 5 being the most difficult.", - gt=0, - le=5, - ) - youtube_url: SkipJsonSchema[str | None] = None - id: uuid.UUID = Field(description="Unique identifier", default_factory=uuid.uuid4) -``` - -This examples shows several `instructor` features: - -- `Field` can have a `default` or `default_factory` value to prevent the LLM from - hallucinating the value - - `id` generates a unique id (`uuid`) -- The type annotation `SkipJsonSchema` also prevents the LLM from generating the value. - - `youtube_url` is set programmatically in the application. We don't want the LLM - to hallucinate it. -- `Field` can set constraints on what the LLM generates. - - `min_items=3, max_items=5` to limit the number of potential answers between 3 and 5 - - `ge=0, lt=5` to limit the difficulty between 0 and 5 with 5 being the most difficult - - -### 2. Retrieve the YouTube transcript - -We use `youtube-transcript-api` to get the full transcript of a video. - -```python -from youtube_transcript_api import YouTubeTranscriptApi - -youtube_url = "https://www.youtube.com/watch?v=hqutVJyd3TI" -_, _, video_id = youtube_url.partition("?v=") -segments = YouTubeTranscriptApi.get_transcript(video_id) -transcript = " ".join([s["text"] for s in segments]) -``` - -### 3. Generate question-answer pairs - -Now, to produce question-answer pairs: - -1. Create an `instructor` client by wrapping the OpenAI client -2. Use `.create_iterable()` on the `instructor_client` to generate multiple outputs from - the input -3. Specify `response_model=QuestionAnswer` to ensure outputs are `QuestionAnswer` objects -4. Use the `messages` to pass the task instructos via the `system` message, and the input - transcript via `user` message. - -```python hl_lines="4 10 12" -import instructor - -instructor_client = instructor.from_provider("openai/gpt-5-nano") - -system_prompt = """Analyze the given YouTube transcript and generate question-answer pairs -to help study and understand the topic better. Please rate all questions from 1 to 5 -based on their difficulty.""" - -response = instructor_client.create_iterable( - model="gpt-4o-mini", - response_model=QuestionAnswer, - messages=[ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": transcript}, - ], -) -``` - -This will return an generator that you can iterate over to access individual -`QuestionAnswer` objects. - -```python -print("Preview:\n") -count = 0 -for qna in response: - if count > 2: - break - print(qna.question) - print(qna.options) - print() - count += 1 - -""" -Preview: - -What is the primary purpose of the new OpenTelemetry instrumentation released with Burr? -['To reduce code complexity', 'To provide full instrumentation without changing code', 'To couple the project with OpenAI', 'To enhance customer support'] - -What do you need to install to use the OpenTelemetry instrumentation with Burr applications? -['Only OpenAI package', 'Specific OpenTelemetry instrumentation module', 'All available packages', 'No installation needed'] - -What advantage does OpenTelemetry provide in the context of instrumentation? -['It is vendor agnostic', 'It requires complex integration', 'It relies on specific vendors', 'It makes applications slower'] -""" -``` - - -## Create a flashcard application with Burr - -Burr uses `actions` and `transitions` to define complex applications while -preserving the simplicity of a flowchart for understanding and debugging. - - -### 1. Define `actions` - -Actions are what your application can do. The `@action` decorator specifies what values -can be read from or written to `State`. The decorated function takes a `State` as -first argument and return an updated `State` object. - -Next, we define three actions: - -- Process the user input to get the YouTube URL -- Get the YouTube transcript associated with the URL -- Generate question-answer pairs for the transcript - -Note that this is only a light refactor from the previous code snippets. - -```python -from burr.core import action, State - - -@action(reads=[], writes=["youtube_url"]) -def process_user_input(state: State, user_input: str) -> State: - """Process user input and update the YouTube URL.""" - youtube_url = ( - user_input # In practice, we would have more complex validation logic. - ) - return state.update(youtube_url=youtube_url) - - -@action(reads=["youtube_url"], writes=["transcript"]) -def get_youtube_transcript(state: State) -> State: - """Get the official YouTube transcript for a video given it's URL""" - youtube_url = state["youtube_url"] - - _, _, video_id = youtube_url.partition("?v=") - transcript = YouTubeTranscriptApi.get_transcript(video_id) - full_transcript = " ".join([entry["text"] for entry in transcript]) - - # store the transcript in state - return state.update(transcript=full_transcript, youtube_url=youtube_url) - - -@action(reads=["transcript", "youtube_url"], writes=["question_answers"]) -def generate_question_and_answers(state: State) -> State: - """Generate `QuestionAnswer` from a YouTube transcript using an LLM.""" - # read the transcript from state - transcript = state["transcript"] - youtube_url = state["youtube_url"] - - # create the instructor client - instructor_client = instructor.from_provider("openai/gpt-5-nano") - system_prompt = ( - "Analyze the given YouTube transcript and generate question-answer pairs" - " to help study and understand the topic better. Please rate all questions from 1 to 5" - " based on their difficulty." - ) - response = instructor_client.create_iterable( - model="gpt-4o-mini", - response_model=QuestionAnswer, - messages=[ - {"role": "system", "content": system_prompt}, - {"role": "user", "content": transcript}, - ], - ) - - # iterate over QuestionAnswer, add the `youtube_url`, and append to state - for qna in response: - qna.youtube_url = youtube_url - # `State` is immutable, so `.append()` returns a new object with the appended value - state = state.append(question_answers=qna) - - return state -``` - -### 2. Build the `Application` - -To create a Burr `Application`, we use the `ApplicationBuilder` object. - -Minimally, it needs to: - -- Use `.with_actions()` to define all possible actions. Simply pass the functions - decorated with `@action`. -- Use `.with_transitions()` to define possible transitions between actions. This is - done via tuples `(from_action, to_action)`. -- Use `.with_entrypoint()` to specify which action to run first. - - -```python -from burr.core import ApplicationBuilder - -app = ( - ApplicationBuilder() - .with_actions( - process_user_input, - get_youtube_transcript, - generate_question_and_answers, - ) - .with_transitions( - ("process_user_input", "get_youtube_transcript"), - ("get_youtube_transcript", "generate_question_and_answers"), - ("generate_question_and_answers", "process_user_input"), - ) - .with_entrypoint("process_user_input") - .build() -) -app.visualize() -``` - -![Burr application graph](./img/youtube-flashcards/flashcards.png) - -> You can always visualize the application graph to understand the logic's flow. - - -### 3. Launch the application - -Using `Application.run()` will make the application execute actions until a halt condition. -In this case, we halt before `process_user_input` to get the YouTube URL from the user. - -The method `.run()` returns a tuple `(action_name, result, state)`. In this case, we only -use the state to inspect the generated question-answer pairs. - -```python -action_name, result, state = app.run( - halt_before=["process_user_input"], - inputs={"user_input": "https://www.youtube.com/watch?v=hqutVJyd3TI"}, -) -print(state["question_answers"][0]) -``` - -You can create a simple local experience by using `.run()` in a `while` loop - -```python -while True: - user_input = input("Enter a YouTube URL (q to quit): ") - if user_input.lower() == "q": - break - - action_name, result, state = app.run( - halt_before=["process_user_input"], - inputs={"user_input": user_input}, - ) - print(f"{len(state['question_answers'])} question-answer pairs generated") -``` - - -## Next steps - -Now that you know how to use Instructor for reliable LLM outputs and Burr to -structure your application, many avenues open up depending on your goals! - - -### 1. Build complex agents - -Instructor improves the LLM's reasoning by providing structure. Nesting models and adding -constraints allow to [get facts with citations](../../examples/exact_citations.md) -or [extract a knowledge graph](../../examples/knowledge_graph.md) -in a few lines of code. Also, [retries](../../concepts/retrying.md) -enable the LLM to self-correct. - -Burr sets the boundaries between users, LLMs, and the rest of your system. You can add -`Condition` on transitions to create complex workflows that remain easy to reason about. - -### 2. Add Burr to your product - -Your Burr `Application` is a lightweight Python object. You can run it within a notebook, -via script, a web app (Streamlit, Gradio, etc.), or as a [web service](https://burr.dagworks.io/examples/deployment/web-server/) -(e.g., FastAPI). - -The `ApplicationBuilder` provides many features to productionize your app: - -- [Persistence](https://burr.dagworks.io../../concepts/state-persistence/.md): save and restore `State` - (e.g., store conversation history) -- [Observability](https://burr.dagworks.io../../concepts/additional-visibility/.md): log and monitor - application telemetry (e.g., LLM calls, number of tokens used, errors and retries) -- [Streaming and async](https://burr.dagworks.io../../concepts/streaming-actions/.md): create snappy - user interfaces by streaming LLM responses and running actions asynchronously. - -For example, you can log telemetry into Burr UI in a few lines of code. First, instrument the -OpenAI library. Then, add `.with_tracker()` the `ApplicationBuilder` with a project name and -enabling `use_otel_tracing=True`. - -```python hl_lines="5 19" -from burr.core import ApplicationBuilder -from opentelemetry.instrumentation.openai import OpenAIApiInstrumentor - -# instrument before importing instructor or creating the OpenAI client -OpenAIApiInstrumentor().instrument() - -app = ( - ApplicationBuilder() - .with_actions( - process_user_input, - get_youtube_transcript, - generate_question_and_answers, - ) - .with_transitions( - ("process_user_input", "get_youtube_transcript"), - ("get_youtube_transcript", "generate_question_and_answers"), - ("generate_question_and_answers", "process_user_input"), - ) - .with_tracker(project="youtube-qna", use_otel_tracing=True) - .with_entrypoint("process_user_input") - .build() -) -``` - -![telemetry](./img/youtube-flashcards/telemetry.gif) - -> Telemetry for our OpenAI API calls with Instructor. We see the prompt, the response model, and the response content. - -### 3. Annotate application logs - -Burr UI has a built-in annotation tool that allows you to label, rate, or comment on -logged data (e.g., user input, LLM response, content retrieved for RAG). This can be -useful to create test cases and evaluation datasets. - -![annotation tool](./img/youtube-flashcards/annotations.png) - - -## Conclusion - -We've shown how Instructor helps getting reliable outputs from LLMs and Burr provides -the right tools to build an application. Now it's your turn to start building! diff --git a/참고/instructor-main/docs/blog/posts/youtube-transcripts.md b/참고/instructor-main/docs/blog/posts/youtube-transcripts.md deleted file mode 100644 index 5765c1f..0000000 --- a/참고/instructor-main/docs/blog/posts/youtube-transcripts.md +++ /dev/null @@ -1,327 +0,0 @@ ---- -authors: -- jxnl -categories: -- Data Processing -comments: true -date: 2024-07-11 -description: Learn how to extract and summarize YouTube video transcripts into chapters - using Python and Pydantic for versatile applications. -draft: false -slug: youtube-transcripts -tags: -- YouTube -- transcripts -- Pydantic -- Python -- Data Processing ---- - -# Analyzing Youtube Transcripts with Instructor - -## Extracting Chapter Information - -!!! info "Code Snippets" - - As always, the code is readily available in our `examples/youtube` folder in our repo for your reference in the `run.py` file. - -In this post, we'll show you how to summarise Youtube video transcripts into distinct chapters using `instructor` before exploring some ways you can adapt the code to different applications. - -By the end of this article, you'll be able to build an application as per the video below. - -![](../../img/youtube.gif) - - - -Let's first install the required packages. - -```bash -pip install openai instructor pydantic youtube_transcript_api -``` - -!!! info "Quick Note" - - The video that we'll be using in this tutorial is [A Hacker's Guide To Language Models](https://www.youtube.com/watch?v=jkrNMKz9pWU) by Jeremy Howard. It has the video id of `jkrNMKz9pWU`. - -Next, let's start by defining a Pydantic Model for the structured chapter information that we want. - -```python -from pydantic import BaseModel, Field - - -class Chapter(BaseModel): - start_ts: float = Field( - ..., - description="Starting timestamp for a chapter.", - ) - end_ts: float = Field( - ..., - description="Ending timestamp for a chapter", - ) - title: str = Field( - ..., description="A concise and descriptive title for the chapter." - ) - summary: str = Field( - ..., - description="A brief summary of the chapter's content, don't use words like 'the speaker'", - ) -``` - -We can take advantage of `youtube-transcript-api` to extract out the transcript of a video using the following function - -```python -from youtube_transcript_api import YouTubeTranscriptApi - - -def get_youtube_transcript(video_id: str) -> str: - try: - transcript = YouTubeTranscriptApi.get_transcript(video_id) - return " ".join( - [f"ts={entry['start']} - {entry['text']}" for entry in transcript] - ) - except Exception as e: - print(f"Error fetching transcript: {e}") - return "" -``` - -Once we've done so, we can then put it all together into the following functions. - -```python hl_lines="30-31 38-48" -import instructor -from pydantic import BaseModel, Field -from youtube_transcript_api import YouTubeTranscriptApi - -# Set up OpenAI client -client = instructor.from_provider("openai/gpt-5-nano") - - -class Chapter(BaseModel): - start_ts: float = Field( - ..., - description="The start timestamp indicating when the chapter starts in the video.", - ) - end_ts: float = Field( - ..., - description="The end timestamp indicating when the chapter ends in the video.", - ) - title: str = Field( - ..., description="A concise and descriptive title for the chapter." - ) - summary: str = Field( - ..., - description="A brief summary of the chapter's content, don't use words like 'the speaker'", - ) - - -def get_youtube_transcript(video_id: str) -> str: - try: - transcript = YouTubeTranscriptApi.get_transcript(video_id) - return [f"ts={entry['start']} - {entry['text']}" for entry in transcript] - except Exception as e: - print(f"Error fetching transcript: {e}") - """ - Error fetching transcript: type object 'YouTubeTranscriptApi' has no attribute 'get_transcript' - """ - return "" - - -def extract_chapters(transcript: str): - return client.create_iterable( - model="gpt-4o", # You can experiment with different models - response_model=Chapter, - messages=[ - { - "role": "system", - "content": "Analyze the given YouTube transcript and extract chapters. For each chapter, provide a start timestamp, end timestamp, title, and summary.", - }, - {"role": "user", "content": transcript}, - ], - ) - - -if __name__ == "__main__": - transcripts = get_youtube_transcript("jkrNMKz9pWU") - - for transcript in transcripts[:2]: - print(transcript) - #> ts=0.539 - hi I am Jeremy Howard from fast.ai and - #> ts=4.62 - this is a hacker's guide to language - - formatted_transcripts = ''.join(transcripts) - chapters = extract_chapters(formatted_transcripts) - - for chapter in chapters: - print(chapter.model_dump_json(indent=2)) - """ - { - "start_ts": 0.0, - "end_ts": 30.0, - "title": "Introduction and Topic Overview", - "summary": "Introduction to the video, outlining the main topic of discussion." - } - """ - """ - { - "start_ts": 31.0, - "end_ts": 60.0, - "title": "Background Information", - "summary": "Background information relevant to the topic." - } - """ - """ - { - "start_ts": 61.0, - "end_ts": 120.0, - "title": "Key Concept Explanation", - "summary": "Detailed explanation of the key concepts." - } - """ - """ - { - "start_ts": 121.0, - "end_ts": 165.0, - "title": "Critical Analysis", - "summary": "Analysis and discussion of the critical aspects of the topic." - } - """ - """ - { - "start_ts": 166.0, - "end_ts": 210.0, - "title": "Examples and Case Studies", - "summary": "Presentation of examples and case studies related to the topic." - } - """ - """ - { - "start_ts": 211.0, - "end_ts": 240.0, - "title": "Conclusion and Final Thoughts", - "summary": "Conclusion of the video with final thoughts on the topic." - } - """ - """ - { - "start_ts": 9.72, - "end_ts": 65.6, - "title": "Understanding Language Models", - "summary": "Explains the code-first approach to using language models, suggesting prerequisites such as prior deep learning knowledge and recommends the course.fast.ai for in-depth learning." - } - """ - """ - { - "start_ts": 65.6, - "end_ts": 250.68, - "title": "Basics of Language Models", - "summary": "Covers the concept of language models, demonstrating how they predict the next word in a sentence, and showcases OpenAI's text DaVinci for creative brainstorming with examples." - } - """ - """ - { - "start_ts": 250.68, - "end_ts": 459.199, - "title": "How Language Models Work", - "summary": "Dives deeper into how language models like ULMfit and others were developed, their training on datasets like Wikipedia, and the importance of learning various aspects of the world to predict the next word effectively." - } - """ - # ... other chapters -``` - -## Alternative Ideas - -Now that we've seen a complete example of chapter extraction, let's explore some alternative ideas using different Pydantic models. These models can be used to adapt our YouTube transcript analysis for various applications. - -### 1. Study Notes Generator - -```python -from pydantic import BaseModel, Field -from typing import List - - -class Concept(BaseModel): - term: str = Field(..., description="A key term or concept mentioned in the video") - definition: str = Field( - ..., description="A brief definition or explanation of the term" - ) - - -class StudyNote(BaseModel): - timestamp: float = Field( - ..., description="The timestamp where this note starts in the video" - ) - topic: str = Field(..., description="The main topic being discussed at this point") - key_points: List[str] = Field(..., description="A list of key points discussed") - concepts: List[Concept] = Field( - ..., description="Important concepts mentioned in this section" - ) -``` - -This model structures the video content into clear topics, key points, and important concepts, making it ideal for revision and study purposes. - -### 2. Content Summarization - -```python -from pydantic import BaseModel, Field -from typing import List - - -class ContentSummary(BaseModel): - title: str = Field(..., description="The title of the video") - duration: float = Field( - ..., description="The total duration of the video in seconds" - ) - main_topics: List[str] = Field( - ..., description="A list of main topics covered in the video" - ) - key_takeaways: List[str] = Field( - ..., description="The most important points from the entire video" - ) - target_audience: str = Field( - ..., description="The intended audience for this content" - ) -``` - -This model provides a high-level overview of the entire video, perfect for quick content analysis or deciding whether a video is worth watching in full. - -### 3. Quiz Generator - -```python -from pydantic import BaseModel, Field -from typing import List - - -class QuizQuestion(BaseModel): - question: str = Field(..., description="The quiz question") - options: List[str] = Field( - ..., min_items=2, max_items=4, description="Possible answers to the question" - ) - correct_answer: int = Field( - ..., - ge=0, - lt=4, - description="The index of the correct answer in the options list", - ) - explanation: str = Field( - ..., description="An explanation of why the correct answer is correct" - ) - - -class VideoQuiz(BaseModel): - title: str = Field( - ..., description="The title of the quiz, based on the video content" - ) - questions: List[QuizQuestion] = Field( - ..., - min_items=5, - max_items=20, - description="A list of quiz questions based on the video content", - ) -``` - -This model transforms video content into an interactive quiz, perfect for testing comprehension or creating engaging content for social media. - -To use these alternative models, you would replace the `Chapter` model in our original code with one of these alternatives and adjust the system prompt in the `extract_chapters` function accordingly. - -## Conclusion - -The power of this approach lies in its flexibility. By defining the result of our function calls as Pydantic Models, we're able to quickly adapt code for a wide variety of applications whether it be generating quizzes, creating study materials or just optimizing for simple SEO. \ No newline at end of file diff --git a/참고/instructor-main/docs/cli/batch.md b/참고/instructor-main/docs/cli/batch.md deleted file mode 100644 index c66e614..0000000 --- a/참고/instructor-main/docs/cli/batch.md +++ /dev/null @@ -1,356 +0,0 @@ ---- -title: Managing Batch Jobs with Multi-Provider CLI -description: Learn how to create, list, cancel, and delete batch jobs using the unified Command Line Interface (CLI) across OpenAI and Anthropic providers. ---- - -# Using the Command Line Interface for Batch Jobs - -The instructor CLI provides comprehensive functionalities for managing batch jobs across multiple providers with a unified interface. This multi-provider support allows users to leverage the strengths of different AI providers for their batch processing needs. - -## Supported Providers - -- **OpenAI**: Utilizes OpenAI's robust batch processing capabilities with metadata support -- **Anthropic**: Leverages Anthropic's advanced language models with cancel/delete operations - -The CLI uses a unified `--provider` flag for all commands, with backward compatibility for legacy flags. - -```bash -$ instructor batch --help - - Usage: instructor batch [OPTIONS] COMMAND [ARGS]... - - Manage OpenAI Batch jobs - -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────╮ -│ cancel Cancel a batch job │ -│ create Create batch job using BatchProcessor │ -│ create-from-file Create a batch job from a file │ -│ delete Delete a completed batch job │ -│ download-file Download the file associated with a batch job │ -│ list See all existing batch jobs │ -│ results Retrieve results from a batch job │ -╰──────────────────────────────────────────────────────────────────────────────╯ -``` - -## Creating a Batch Job - -### List Jobs with Enhanced Display - -```bash -$ instructor batch list --help - - Usage: instructor batch list [OPTIONS] - - See all existing batch jobs - -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ --limit INTEGER Total number of batch jobs │ -│ to show │ -│ [default: 10] │ -│ --poll INTEGER Time in seconds to wait │ -│ for the batch job to │ -│ complete │ -│ [default: 10] │ -│ --screen --no-screen Enable or disable screen │ -│ output │ -│ [default: no-screen] │ -│ --live --no-live Enable live polling to │ -│ continuously update the │ -│ table │ -│ [default: no-live] │ -│ --provider TEXT Provider to use (e.g., │ -│ 'openai', 'anthropic') │ -│ [default: openai] │ -│ --use-anthropic --no-use-anthropic [DEPRECATED] Use --model │ -│ instead. Use Anthropic API │ -│ instead of OpenAI │ -│ [default: │ -│ no-use-anthropic] │ -│ --help Show this message and │ -│ exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ -``` - -The enhanced list command now shows rich information including timestamps, duration, and provider-specific metrics: - -```bash -$ instructor batch list --provider openai --limit 3 - - Openai Batch Jobs -┏━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━┳━━━━━━━┓ -┃ Batch ID ┃ Status ┃ Created ┃ Started ┃ Duration┃ Completed┃ Failed ┃ Total ┃ -┡━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━╇━━━━━━━┩ -│ batch_abc123... │ completed │ 07/07 │ 07/07 │ 2m │ 15 │ 0 │ 15 │ -│ │ │ 23:48 │ 23:48 │ │ │ │ │ -│ batch_def456... │ processing │ 07/07 │ 07/07 │ 45m │ 8 │ 0 │ 10 │ -│ │ │ 22:30 │ 22:31 │ │ │ │ │ -│ batch_ghi789... │ failed │ 07/07 │ N/A │ N/A │ 0 │ 5 │ 5 │ -│ │ │ 21:15 │ │ │ │ │ │ -└────────────────────┴────────────┴────────────┴────────────┴─────────┴──────────┴────────┴───────┘ - -$ instructor batch list --provider anthropic --limit 2 - - Anthropic Batch Jobs -┏━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━┳━━━━━━━━━━━━━┓ -┃ Batch ID ┃ Status ┃ Created ┃ Started ┃ Duration┃ Succeeded┃ Errored ┃ Processing ┃ -┡━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━╇━━━━━━━━━━━━━┩ -│ msgbatch_abc123... │ completed │ 07/08 │ 07/08 │ 1m │ 20 │ 0 │ 0 │ -│ │ │ 03:47 │ 03:47 │ │ │ │ │ -│ msgbatch_def456... │ processing │ 07/08 │ 07/08 │ 15m │ 5 │ 0 │ 10 │ -│ │ │ 03:30 │ 03:30 │ │ │ │ │ -└──────────────────────┴────────────┴────────────┴────────────┴─────────┴──────────┴─────────┴─────────────┘ -``` - -### Create From File with Metadata Support - -You can create batch jobs directly from pre-formatted .jsonl files with enhanced metadata support: - -```bash -$ instructor batch create-from-file --help - - Usage: instructor batch create-from-file [OPTIONS] - - Create a batch job from a file - -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ * --file-path TEXT File containing the │ -│ batch job requests │ -│ [default: None] │ -│ [required] │ -│ --model TEXT Model in format │ -│ 'provider/model-name' │ -│ (e.g., 'openai/gpt-4', │ -│ 'anthropic/claude-3-s… │ -│ [default: │ -│ openai/gpt-4o-mini] │ -│ --description TEXT Description/metadata │ -│ for the batch job │ -│ [default: Instructor │ -│ batch job] │ -│ --completion-window TEXT Completion window for │ -│ the batch job (OpenAI │ -│ only) │ -│ [default: 24h] │ -│ --use-anthropic --no-use-anthropic [DEPRECATED] Use │ -│ --model instead. Use │ -│ Anthropic API instead │ -│ of OpenAI │ -│ [default: │ -│ no-use-anthropic] │ -│ --help Show this message and │ -│ exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ -``` - -Example usage with metadata: - -```bash -# OpenAI batch with custom metadata -instructor batch create-from-file \ - --file-path batch_requests.jsonl \ - --model "openai/gpt-5-nano" \ - --description "Email classification batch - production v2.1" \ - --completion-window "24h" - -# Anthropic batch -instructor batch create-from-file \ - --file-path batch_requests.jsonl \ - --model "anthropic/claude-3-5-sonnet-20241022" \ - --description "Text analysis batch" -``` - -For creating .jsonl files, you can use the enhanced `BatchProcessor`: - -```python -from instructor.batch import BatchProcessor -from pydantic import BaseModel, Field -from typing import Literal - -class Classification(BaseModel): - label: Literal["SPAM", "NOT_SPAM"] = Field( - ..., description="Whether the email is spam or not" - ) - -# Create processor -processor = BatchProcessor("openai/gpt-5-nano", Classification) - -# Prepare message conversations -messages_list = [ - [ - {"role": "system", "content": "Classify the following email"}, - {"role": "user", "content": "Hello there I'm a Nigerian prince and I want to give you money"} - ], - [ - {"role": "system", "content": "Classify the following email"}, - {"role": "user", "content": "Meeting with Thomas has been set at Friday next week"} - ] -] - -# Create batch file -processor.create_batch_from_messages( - messages_list=messages_list, - file_path="batch_requests.jsonl", - max_tokens=100, - temperature=0.1 -) -``` - -## Job Management Operations - -### Cancelling a Batch Job - -Cancel running batch jobs across all providers: - -```bash -$ instructor batch cancel --help - - Usage: instructor batch cancel [OPTIONS] - - Cancel a batch job - -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ * --batch-id TEXT Batch job ID to cancel │ -│ [default: None] │ -│ [required] │ -│ --provider TEXT Provider to use (e.g., │ -│ 'openai', 'anthropic') │ -│ [default: openai] │ -│ --use-anthropic --no-use-anthropic [DEPRECATED] Use │ -│ --provider 'anthropic' │ -│ instead. Use Anthropic API │ -│ instead of OpenAI │ -│ [default: │ -│ no-use-anthropic] │ -│ --help Show this message and │ -│ exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ -``` - -Examples: - -```bash -# Cancel OpenAI batch -instructor batch cancel --batch-id batch_abc123 --provider openai - -# Cancel Anthropic batch -instructor batch cancel --batch-id msgbatch_def456 --provider anthropic -``` - -### Deleting a Batch Job - -Delete completed batch jobs (supported by Anthropic): - -```bash -$ instructor batch delete --help - - Usage: instructor batch delete [OPTIONS] - - Delete a completed batch job - -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ * --batch-id TEXT Batch job ID to delete [default: None] [required] │ -│ --provider TEXT Provider to use (e.g., 'openai', 'anthropic') │ -│ [default: openai] │ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ -``` - -Examples: - -```bash -# Delete Anthropic batch (supported) -instructor batch delete --batch-id msgbatch_abc123 --provider anthropic - -# Try to delete OpenAI batch (shows helpful message) -instructor batch delete --batch-id batch_ghi789 --provider openai -# Note: OpenAI does not support batch deletion via API -``` - -### Retrieving Batch Results - -Get structured results from completed batch jobs: - -```bash -$ instructor batch results --help - - Usage: instructor batch results [OPTIONS] - - Retrieve results from a batch job - -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ * --batch-id TEXT Batch job ID to get results from │ -│ [default: None] │ -│ [required] │ -│ * --output-file TEXT File to save the results to [default: None] │ -│ [required] │ -│ --model TEXT Model in format 'provider/model-name' (e.g., │ -│ 'openai/gpt-4', 'anthropic/claude-3-sonnet') │ -│ [default: openai/gpt-4o-mini] │ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ -``` - -Examples: - -```bash -# Get OpenAI batch results -instructor batch results \ - --batch-id batch_abc123 \ - --output-file openai_results.jsonl \ - --model "openai/gpt-5-nano" - -# Get Anthropic batch results -instructor batch results \ - --batch-id msgbatch_def456 \ - --output-file anthropic_results.jsonl \ - --model "anthropic/claude-3-5-sonnet-20241022" -``` - -### Downloading Raw Files (Legacy) - -For compatibility, the download-file command is still available: - -```bash -$ instructor batch download-file --help - - Usage: instructor batch download-file [OPTIONS] - - Download the file associated with a batch job - -╭─ Options ────────────────────────────────────────────────────────────────────╮ -│ * --batch-id TEXT Batch job ID to download │ -│ [default: None] │ -│ [required] │ -│ * --download-file-path TEXT Path to download file to │ -│ [default: None] │ -│ [required] │ -│ --provider TEXT Provider to use (e.g., 'openai', │ -│ 'anthropic') │ -│ [default: openai] │ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────╯ -``` - -## Provider Support Matrix - -| Operation | OpenAI | Anthropic | -|-----------|--------|-----------| -| **List** | ✅ Enhanced table | ✅ Enhanced table | -| **Create** | ✅ With metadata | ✅ File-based | -| **Cancel** | ✅ Standard API | ✅ Standard API | -| **Delete** | ❌ Not supported | ✅ Standard API | -| **Results** | ✅ Structured parsing | ✅ Structured parsing | - -## Enhanced Features - -- **Rich CLI Tables**: Color-coded status, timestamps, duration calculations -- **Metadata Support**: Add descriptions and custom fields to organize batches -- **Unified Commands**: Same interface works across all providers -- **Provider Detection**: Automatic provider detection from model strings -- **Error Handling**: Clear error messages and helpful notes for unsupported operations -- **Backward Compatibility**: Legacy flags still work with deprecation warnings - -This comprehensive CLI interface provides efficient batch job management across all supported providers with enhanced monitoring and control capabilities. diff --git a/참고/instructor-main/docs/cli/finetune.md b/참고/instructor-main/docs/cli/finetune.md deleted file mode 100644 index fcebe16..0000000 --- a/참고/instructor-main/docs/cli/finetune.md +++ /dev/null @@ -1,153 +0,0 @@ ---- -title: Managing Fine-Tuning Jobs with the Instructor CLI -description: Learn how to create, view, and manage fine-tuning jobs on OpenAI using the Instructor CLI, with essential commands and options. ---- - -# Using the Command Line Interface - -The instructor CLI provides functionalities for managing fine-tuning jobs on OpenAI. - -!!! warning "Incomplete API" -The CLI is still under development and does not yet support all features of the API. If you would like to use a feature that is not yet supported, please consider using the contributing to our library [jxnl/instructor](https://www.github.com/jxnl/instructor) instead. - - !!! note "Low hanging fruit" - - If you want to contribute we're looking for a few things: - - 1. Adding filenames on upload - -## Creating a Fine-Tuning Job - -### View Jobs Options - -```sh -$ instructor jobs --help - - Usage: instructor jobs [OPTIONS] COMMAND [ARGS]... - - Monitor and create fine tuning jobs - -╭─ Options ───────────────────────────────────────────────────────────────────────────────╮ -│ --help Display the help message. │ -╰─────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ cancel Cancel a fine-tuning job. │ -│ create-from-file Create a fine-tuning job from a file. │ -│ create-from-id Create a fine-tuning job from an existing ID. │ -│ list Monitor the status of the most recent fine-tuning jobs. │ -╰─────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ - -``` - -### Create from File - -The create-from-file command uploads and trains a model in a single step. - -```sh -❯ instructor jobs create-from-file --help - -Usage: instructor jobs create-from-file [OPTIONS] FILE - - Create a fine-tuning job from a file. - -╭─ Arguments ───────────────────────────────────────────────────────────────────────────────────────╮ -│ * file TEXT Path to the file for fine-tuning [default: None] [required] │ -╰───────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ─────────────────────────────────────────────────────────────────────────────────────────╮ -│ --model TEXT Model to use for fine-tuning [default: gpt-3.5-turbo] │ -│ --poll INTEGER Polling interval in seconds [default: 2] │ -│ --n-epochs INTEGER Number of epochs for fine-tuning │ -│ --batch-size TEXT Batch size for fine-tuning │ -│ --learning-rate-multiplier TEXT Learning rate multiplier for fine-tuning │ -│ --validation-file TEXT Path to the validation file [default: None] │ -│ --model-suffix TEXT Suffix to identify the model [default: None] │ -│ --help Show this message and exit. │ -╰──────────────────────────────────────────────────────────────────────────────── -``` - -#### Usage - -```sh -$ instructor jobs create-from-file transformed_data.jsonl --validation_file validation_data.jsonl --n_epochs 3 --batch_size 16 --learning_rate_multiplier 0.5 -``` - -### Create from ID - -The create-from-id command uses an uploaded file and trains a model - -```sh -❯ instructor jobs create-from-id --help - - Usage: instructor jobs create-from-id [OPTIONS] ID - - Create a fine-tuning job from an existing ID. - -╭─ Arguments ───────────────────────────────────────────────────────────────────────────╮ -│ * id TEXT ID of the existing fine-tuning job [default: None] [required] │ -╰───────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ─────────────────────────────────────────────────────────────────────────────╮ -│ --model TEXT Model to use for fine-tuning │ -│ [default: gpt-3.5-turbo] │ -│ --n-epochs INTEGER Number of epochs for fine-tuning │ -│ --batch-size TEXT Batch size for fine-tuning │ -│ --learning-rate-multiplier TEXT Learning rate multiplier for fine-tuning │ -│ --validation-file-id TEXT ID of the uploaded validation file │ -│ [default: None] │ -│ --help Show this message and exit. │ -╰───────────────────────────────────────────────────────────────────────────────────────╯ -``` - -#### Usage - -```sh -$ instructor files upload transformed_data.jsonl -$ instructor files upload validation_data.jsonl -$ instructor files list -... -$ instructor jobs create_from_id --validation_file --n_epochs 3 --batch_size 16 --learning_rate_multiplier 0.5 -``` - -### Viewing Files and Jobs - -#### Viewing Jobs - -```sh -$ instructor jobs list - -OpenAI Fine Tuning Job Monitoring -┏━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━━━━━━━━━━━┓ -┃ ┃ ┃ ┃ Completion ┃ ┃ ┃ ┃ ┃ -┃ Job ID ┃ Status ┃ Creation Time ┃ Time ┃ Model Name ┃ File ID ┃ Epochs ┃ Base Model ┃ -┡━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━━━━━━━━━━━┩ -│ ftjob-PWo6uwk... │ 🚫 cancelled │ 2023-08-23 │ N/A │ │ file-F7lJg6Z4... │ 3 │ gpt-3.5-turbo-... │ -│ │ │ 23:10:54 │ │ │ │ │ │ -│ ftjob-1whjva8... │ 🚫 cancelled │ 2023-08-23 │ N/A │ │ file-F7lJg6Z4... │ 3 │ gpt-3.5-turbo-... │ -│ │ │ 22:47:05 │ │ │ │ │ │ -│ ftjob-wGoBDld... │ 🚫 cancelled │ 2023-08-23 │ N/A │ │ file-F7lJg6Z4... │ 3 │ gpt-3.5-turbo-... │ -│ │ │ 22:44:12 │ │ │ │ │ │ -│ ftjob-yd5aRTc... │ ✅ succeeded │ 2023-08-23 │ 2023-08-23 │ ft:gpt-3.5-tur... │ file-IQxAUDqX... │ 3 │ gpt-3.5-turbo-... │ -│ │ │ 14:26:03 │ 15:02:29 │ │ │ │ │ -└────────────────┴──────────────┴────────────────┴────────────────┴─────────────────┴────────────────┴────────┴─────────────────┘ - Automatically refreshes every 5 seconds, press Ctrl+C to exit -``` - -#### Viewing Files - -```sh -$ instructor files list - -OpenAI Files -┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━━━━┓ -┃ File ID ┃ Size (bytes) ┃ Creation Time ┃ Filename ┃ Purpose ┃ -┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━━━━┩ -│ file-0lw2BSNRUlXZXRRu2beCCWjl │ 369523 │ 2023-08-23 23:31:57 │ file │ fine-tune │ -│ file-IHaUXcMEykmFUp1kt2puCDEq │ 369523 │ 2023-08-23 23:09:35 │ file │ fine-tune │ -│ file-ja9vRBf0FydEOTolaa3BMqES │ 369523 │ 2023-08-23 22:42:29 │ file │ fine-tune │ -│ file-F7lJg6Z47CREvmx4kyvyZ6Sn │ 369523 │ 2023-08-23 22:42:03 │ file │ fine-tune │ -│ file-YUxqZPyJRl5GJCUTw3cNmA46 │ 369523 │ 2023-08-23 22:29:10 │ file │ fine-tune │ -└───────────────────────────────┴──────────────┴─────────────────────┴──────────┴───────────┘ -``` - -# Contributions - -We aim to provide a light wrapper around the API rather than offering a complete CLI. Contributions are welcome! Please feel free to make an issue at [jxnl/instructor/issues](https://github.com/jxnl/instructor/issues) or submit a pull request. diff --git a/참고/instructor-main/docs/cli/index.md b/참고/instructor-main/docs/cli/index.md deleted file mode 100644 index 3132ab8..0000000 --- a/참고/instructor-main/docs/cli/index.md +++ /dev/null @@ -1,113 +0,0 @@ ---- -title: Instructor CLI Tools -description: Command-line utilities for monitoring API usage, fine-tuning models, and accessing documentation. ---- - -# Instructor CLI Tools - -
- -- :material-console: **Command Line Utilities** - - Powerful tools to enhance your Instructor workflow - - [:octicons-arrow-right-16: View Commands](#available-commands) - -- :material-chart-line: **Usage Monitoring** - - Track API usage, costs, and token consumption - - [:octicons-arrow-right-16: Usage Guide](usage.md) - -- :material-tune-vertical: **Model Fine-Tuning** - - Create and manage custom model versions - - [:octicons-arrow-right-16: Fine-Tuning Guide](finetune.md) - -- :material-book-open-variant: **Documentation Access** - - Quickly access docs from your terminal - - [:octicons-arrow-right-16: Docs Command](#documentation-command) - -
- -## Getting Started - -### Installation - -The CLI tools are included with the Instructor package: - -```bash -pip install instructor -``` - -### API Setup - -Set your OpenAI API key as an environment variable: - -```bash -export OPENAI_API_KEY="your-api-key-here" -``` - -## Available Commands - -Instructor provides several command-line utilities: - -| Command | Description | Guide | -|---------|-------------|-------| -| `instructor usage` | Track API usage and costs | [Usage Guide](usage.md) | -| `instructor finetune` | Create and manage fine-tuned models | [Fine-Tuning Guide](finetune.md) | -| `instructor docs` | Quick access to documentation | [See below](#documentation-command) | - -## Usage Command - -Monitor your OpenAI API usage directly from the terminal: - -```bash -# View total usage for the current month -instructor usage - -# View usage breakdown by day -instructor usage --by-day - -# Calculate cost for a specific model -instructor usage --model gpt-4 -``` - -For detailed usage statistics and options, see the [Usage Guide](usage.md). - -## Fine-Tuning Command - -Create and manage fine-tuned models with an interactive interface: - -```bash -# Start the fine-tuning interface -instructor finetune -``` - -This launches an interactive application that guides you through the fine-tuning process. Learn more in the [Fine-Tuning Guide](finetune.md). - -## Documentation Command - -Quickly access Instructor documentation from your terminal: - -```bash -# Open main documentation -instructor docs - -# Search for specific topic -instructor docs validation - -# Open specific page -instructor docs concepts/models -``` - -This command opens the Instructor documentation in your default web browser, making it easy to find information when you need it. - -## Support & Contribution - -- **GitHub**: Visit our [GitHub Repository](https://github.com/jxnl/instructor) -- **Issues**: Report bugs or request features on our [Issue Tracker](https://github.com/jxnl/instructor/issues) -- **Discord**: Join our [Discord Community](https://discord.gg/bD9YE9JArw) for support diff --git a/참고/instructor-main/docs/cli/usage.md b/참고/instructor-main/docs/cli/usage.md deleted file mode 100644 index d36037d..0000000 --- a/참고/instructor-main/docs/cli/usage.md +++ /dev/null @@ -1,61 +0,0 @@ ---- -title: OpenAI API Usage CLI Guide -description: Learn how to monitor OpenAI API usage with the CLI tool, including commands for viewing data by model, date, and cost. ---- - -# Using the OpenAI API Usage CLI - -The OpenAI API Usage CLI tool provides functionalities for monitoring your OpenAI API usage, breaking it down by model, date, and cost. - -## Monitoring API Usage - -### View Usage Options - -```sh -$ instructor usage --help - - Usage: instructor usage [OPTIONS] COMMAND [ARGS]... - - Check OpenAI API usage data - -╭─ Options ───────────────────────────────────────────────────────╮ -│ --help Show this message and exit. │ -╰─────────────────────────────────────────────────────────────────╯ -╭─ Commands ──────────────────────────────────────────────────────╮ -│ list Displays OpenAI API usage data for the past N days. │ -╰─────────────────────────────────────────────────────────────────╯ -``` - -### List Usage for Specific Number of Days - -To display API usage for the past 3 days, use the following command: - -```sh -$ instructor usage list --n 3 -``` - -This will output a table similar to: - -```plaintext - Usage Summary by Date, Snapshot, and Cost -┏━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━┓ -┃ Date ┃ Snapshot ID ┃ Total Requests ┃ Total Cost ($) ┃ -┡━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━┩ -│ 2023-09-04 │ gpt-4-0613 │ 44 │ 0.68 │ -│ 2023-09-04 │ gpt-3.5-turbo-16k-0613 │ 195 │ 0.84 │ -│ 2023-09-04 │ text-embedding-ada-002-v2 │ 276 │ 0.00 │ -│ 2023-09-04 │ gpt-4-32k-0613 │ 328 │ 49.45 │ -└────────────┴───────────────────────────┴────────────────┴────────────────┘ -``` - -### List Usage for Today - -To display the API usage for today, simply run: - -```sh -$ instructor usage list -``` - -# Contributions - -We aim to provide a light wrapper around the API rather than offering a complete CLI. Contributions are welcome! Please feel free to make an issue at [jxnl/instructor/issues](https://github.com/jxnl/instructor/issues) or submit a pull request. diff --git a/참고/instructor-main/docs/concepts/alias.md b/참고/instructor-main/docs/concepts/alias.md deleted file mode 100644 index 95fa816..0000000 --- a/참고/instructor-main/docs/concepts/alias.md +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: Pydantic Aliases Overview -description: Explore the concept of aliases in Pydantic. Discover the latest documentation and features for better data validation. ---- - -## See Also - -- [Fields](./fields.md) - Customizing field metadata -- [Response Models](./models.md) - Working with Pydantic models -- [Types](./types.md) - Working with different data types -- [Prompting](./prompting.md) - Prompt engineering techniques - -!!! warning "This page is a work in progress" - - This page is a work in progress. Check out [Pydantic's documentation](https://docs.pydantic.dev/latest/concepts/alias/) diff --git a/참고/instructor-main/docs/concepts/batch.md b/참고/instructor-main/docs/concepts/batch.md deleted file mode 100644 index 0c648dd..0000000 --- a/참고/instructor-main/docs/concepts/batch.md +++ /dev/null @@ -1,223 +0,0 @@ ---- -title: Batch Processing -description: Process multiple LLM requests efficiently using batch processing for 50% cost savings. ---- - -# Batch Processing - -Batch processing lets you send multiple requests in a single operation, saving up to 50% on costs. Instructor supports batch processing across multiple providers. - -## Supported Providers - -| Provider | Models | Cost Savings | -|----------|--------|--------------| -| OpenAI | gpt-4o, gpt-4.1-mini, gpt-4-turbo | 50% | -| Anthropic | claude-3-5-sonnet, claude-3-opus, claude-3-haiku | 50% | -| Google GenAI | gemini-2.5-flash, gemini-2.0-flash, gemini-pro | 50% | - -## Basic Usage - -```python -from instructor.batch import BatchProcessor -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -processor = BatchProcessor("openai/gpt-4.1-mini", User) - -messages_list = [ - [ - {"role": "system", "content": "Extract user information from text."}, - {"role": "user", "content": "Hi, I'm Alice and I'm 28 years old."}, - ], - [ - {"role": "system", "content": "Extract user information from text."}, - {"role": "user", "content": "Hello, I'm Bob, 35 years old."}, - ], -] - -# Create batch file -processor.create_batch_from_messages( - file_path="batch_requests.jsonl", - messages_list=messages_list, - max_tokens=200, - temperature=0.1, -) - -# Submit batch job -batch_id = processor.submit_batch("batch_requests.jsonl") -print(f"Batch job submitted: {batch_id}") - -# Check status and retrieve results -status = processor.get_batch_status(batch_id) -if status['status'] in ['completed', 'ended', 'JOB_STATE_SUCCEEDED']: - from instructor.batch import filter_successful, extract_results - - all_results = processor.retrieve_results(batch_id) - for user in extract_results(all_results): - print(f"Name: {user.name}, Age: {user.age}") -``` - -## In-Memory Processing - -For serverless deployments, use in-memory mode by setting `file_path=None`: - -```python -import time -from instructor.batch import BatchProcessor -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -processor = BatchProcessor("openai/gpt-4.1-mini", User) - -messages_list = [ - [{"role": "user", "content": "Extract: John is 25 years old"}], - [{"role": "user", "content": "Extract: Jane is 30 years old"}], -] - -# Create in-memory buffer (no file_path) -buffer = processor.create_batch_from_messages( - messages_list, - file_path=None, - max_tokens=150, -) - -# Submit and poll for results -batch_id = processor.submit_batch(buffer) - -while True: - status = processor.get_batch_status(batch_id) - if status.get("status") in ["completed", "failed", "cancelled"]: - break - time.sleep(10) - -if status.get("status") == "completed": - results = processor.get_results(batch_id) - for r in results: - if hasattr(r, "result"): - print(f"{r.result.name}, {r.result.age}") -``` - -### When to Use Each Approach - -| Use Case | Approach | -|----------|----------| -| Serverless (Lambda, Cloud Functions) | In-memory | -| Large batch jobs | File-based | -| Security-sensitive environments | In-memory | -| Debugging/audit requirements | File-based | - -## Provider Setup - -### OpenAI - -```bash -export OPENAI_API_KEY="your-openai-key" -``` - -```python -processor = BatchProcessor("openai/gpt-4.1-mini", User) -``` - -### Anthropic - -```bash -export ANTHROPIC_API_KEY="your-anthropic-key" -``` - -```python -processor = BatchProcessor("anthropic/claude-3-5-sonnet-20241022", User) -``` - -### Google GenAI - -```bash -export GOOGLE_CLOUD_PROJECT="your-project-id" -export GCS_BUCKET="your-gcs-bucket-name" -export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json" -``` - -```python -processor = BatchProcessor("google/gemini-2.5-flash", User) -``` - -Required permissions: `roles/aiplatform.user` and `roles/storage.objectUser`. - -## Processing Results - -Results use a Maybe/Result pattern for type-safe handling: - -```python -from instructor.batch import ( - BatchProcessor, - filter_successful, - filter_errors, - extract_results, - get_results_by_custom_id, -) - -all_results = processor.retrieve_results(batch_id) - -# Filter by type -successful = filter_successful(all_results) # List[BatchSuccess[T]] -errors = filter_errors(all_results) # List[BatchError] -objects = extract_results(all_results) # List[T] - -# Access by custom_id -by_id = get_results_by_custom_id(all_results) -if "request-1" in by_id: - result = by_id["request-1"] - if result.success: - print(f"Success: {result.result}") - else: - print(f"Error: {result.error_message}") -``` - -## API Reference - -| Method | Description | -|--------|-------------| -| `create_batch_from_messages(messages_list, file_path=None, ...)` | Create batch file or buffer | -| `submit_batch(file_path_or_buffer, metadata=None)` | Submit batch job, returns job ID | -| `get_batch_status(batch_id)` | Get job status | -| `retrieve_results(batch_id)` | Download and parse results | -| `parse_results(content)` | Parse raw results content | - -## CLI Commands - -```bash -# List batch jobs -instructor batch list --model "openai/gpt-4.1-mini" - -# Create batch from file -instructor batch create-from-file --file-path batch.jsonl --model "openai/gpt-4.1-mini" - -# Get batch results -instructor batch results --batch-id "batch_abc123" --output-file results.jsonl -``` - -## Best Practices - -1. **Batch size**: Include at least 25,000 requests per job for optimal efficiency -2. **Cost optimization**: Use batch processing for non-urgent workloads -3. **Error handling**: Always check both successful and error results -4. **Timeouts**: Batch jobs have execution limits (24 hours for Google) -5. **Storage**: For Google, ensure GCS bucket is in the same region as your batch job - -## Troubleshooting - -| Issue | Solution | -|-------|----------| -| Missing GCS_BUCKET (Google) | Set the `GCS_BUCKET` environment variable | -| Permission Denied (Google) | Add `aiplatform.user` and `storage.objectUser` roles | -| Invalid Model Name | Use format `provider/model-name` | -| Authentication Error | Verify API keys are set correctly | diff --git a/참고/instructor-main/docs/concepts/caching.md b/참고/instructor-main/docs/concepts/caching.md deleted file mode 100644 index 259df3f..0000000 --- a/참고/instructor-main/docs/concepts/caching.md +++ /dev/null @@ -1,430 +0,0 @@ -## See Also - -- [Prompt Caching](./prompt_caching.md) - Cache prompts for cost optimization -- [Performance Optimization](../examples/sqlmodel.md#performance-optimization) - Performance best practices -- [Cost Optimization](../examples/batch_job_oai.md) - Reduce API costs -- [Hooks](./hooks.md) - Monitor cache hits and misses - ---- -title: Caching Strategies with Instructor -description: Learn how to use caching with Instructor to reduce API costs and improve performance. ---- - -For more details on caching concepts, see our [blog](../blog/posts/caching.md). - -## Built-in Caching (v1.9.1 and later) - -Instructor supports caching for every client. Pass a cache adapter when you create the client. The cache parameter flows through to all provider implementations via **kwargs: - -```python -from instructor import from_provider -from instructor.cache import AutoCache, DiskCache - -# Works with any provider - cache flows through **kwargs automatically -client = from_provider("openai/gpt-4.1-mini", cache=AutoCache(maxsize=1000)) -client = from_provider("anthropic/claude-3-haiku", cache=AutoCache(maxsize=1000)) -client = from_provider("google/gemini-2.5-flash", cache=DiskCache(directory=".cache")) - -# Your normal calls are now cached automatically -from pydantic import BaseModel - - -class User(BaseModel): - name: str - - -first = client.create( - messages=[{"role": "user", "content": "Hi."}], response_model=User -) -second = client.create( - messages=[{"role": "user", "content": "Hi."}], response_model=User -) -assert first.name == second.name # second call was served from cache -``` - -### `cache_ttl` per-call override - -Pass `cache_ttl=` alongside `cache=` if you want a result to -expire automatically: - -```python -from instructor import from_provider -from instructor.cache import DiskCache -from pydantic import BaseModel - - -class User(BaseModel): - name: str - - -cache = DiskCache(directory=".cache") -client = from_provider("openai/gpt-4.1-mini") - -client.create( - messages=[{"role": "user", "content": "Hi"}], - response_model=User, - cache=cache, - cache_ttl=3600, # 1 hour -) -``` - -If the underlying cache backend supports TTL (e.g. `DiskCache` does), the -entry will be evicted after the specified duration. For `AutoCache` the -parameter is ignored. - -### Cache-key design - -Under the hood Instructor generates a **deterministic** key for every - call using `instructor.cache.make_cache_key`. - -Components that influence the key: - -| Part | Why it matters | -|-----------------------------|----------------------------------------------| -| `model` | Different model names can yield different answers | -| `messages` / `contents` | The full chat history is hashed | -| `mode` | JSON vs. TOOLS vs. RESPONSES changes formatting | -| `response_model` schema | The entire `model_json_schema()` is included so **any** change in field names, types or *descriptions* busts the cache automatically | - -The function returns a SHA-256 hex digest; its length is constant regardless -of prompt size, so it is safe to use as a Redis key, file path, etc. - -```python -from instructor.cache import make_cache_key -from pydantic import BaseModel - - -class User(BaseModel): - name: str - - -key = make_cache_key( - messages=[{"role": "user", "content": "hello"}], - model="gpt-4.1-mini", - response_model=User, - mode="TOOLS", -) -print(key) # → 9b8f5e2c8c9e… -#> 2e2a9521bd269d62ee9a8559d7deacba0025c1f6da0ec1fc63d472788be096fe -``` - -If you need custom behaviour (e.g. ignoring certain prompt fields) you can -write your own helper and pass a derived key into a bespoke cache adapter. - -### Raw Response Reconstruction - -For raw completion objects (used with `create_with_completion`), we use a `SimpleNamespace` trick to reconstruct the original object structure: - -```python -from pydantic import BaseModel - - -class Completion(BaseModel): - content: str - usage: dict - - -# Example completion object -completion = Completion(content="Hello", usage={"tokens": 10}) - -# When caching: -raw_json = completion.model_dump_json() # Serialize to JSON - -# When restoring from cache: -import json -from types import SimpleNamespace - -restored = json.loads(raw_json, object_hook=lambda d: SimpleNamespace(**d)) -``` - -This approach allows us to restore the original dot-notation access patterns (e.g., `completion.usage.total_tokens`) without requiring the original class definitions. The `SimpleNamespace` objects behave identically to the original completion objects for attribute access while being much simpler to reconstruct from JSON. - -## 1. `functools.cache` for Simple In-Memory Caching - -**When to Use**: Good for functions with immutable arguments, called repeatedly with the same parameters in small to medium-sized applications. Use this when reusing the same data within a single session. - -```python -import time -import functools -import instructor -from pydantic import BaseModel - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class UserDetail(BaseModel): - name: str - age: int - - -@functools.cache -def extract(data) -> UserDetail: - return client.create( - response_model=UserDetail, - messages=[ - {"role": "user", "content": data}, - ], - ) - - -start = time.perf_counter() # (1) -model = extract("Extract jason is 25 years old") -print(f"Time taken: {time.perf_counter() - start}") -#> Time taken: 0.43337099999189377 - -start = time.perf_counter() -model = extract("Extract jason is 25 years old") # (2) -print(f"Time taken: {time.perf_counter() - start}") -#> Time taken: 1.166015863418579e-06 -``` - -1. Using `time.perf_counter()` to measure the time taken to run the function is better than using `time.time()` because it's more accurate and less susceptible to system clock changes. -2. The second time we call `extract`, the result is returned from the cache, and the function is not called. - -!!! warning "Changing the Model does not Invalidate the Cache" - - Note that changing the model does not invalidate the cache. This is because the cache key is based on the function's name and arguments, not the model. This means that if we change the model, the cache will still return the old result. - -Call `extract` multiple times with the same argument, and the result will be cached in memory for faster access. - -**Benefits**: Easy to implement, fast access due to in-memory storage, and requires no additional libraries. - -??? question "What is a decorator?" - - A decorator is a function that takes another function and extends the behavior of the latter function without explicitly modifying it. In Python, decorators are functions that take a function as an argument and return a closure. - - ```python hl_lines="3-5 9" - def decorator(func): - def wrapper(*args, **kwargs): - print("Do something before") # (1) - #> Do something before - result = func(*args, **kwargs) - print("Do something after") # (2) - #> Do something after - return result - - return wrapper - - - @decorator - def say_hello(): - #> Hello! - print("Hello!") - #> Hello! - - - say_hello() - #> "Do something before" - #> "Hello!" - #> "Do something after" - ``` - - 1. The code is executed before the function is called - 2. The code is executed after the function is called - -## 2. `diskcache` for Persistent, Large Data Caching - -??? note "Copy Caching Code" - - The same `instructor_cache` decorator works for both `diskcache` and `redis` caching. Copy the code below and use it for both examples. - - ```python - import functools - import inspect - import diskcache - - cache = diskcache.Cache('./my_cache_directory') # (1) - - - def instructor_cache(func): - """Cache a function that returns a Pydantic model""" - return_type = inspect.signature(func).return_annotation - if not issubclass(return_type, BaseModel): # (2) - raise ValueError("The return type must be a Pydantic model") - - @functools.wraps(func) - def wrapper(*args, **kwargs): - key = f"{func.__name__}-{functools._make_key(args, kwargs, typed=False)}" - # Check if the result is already cached - if (cached := cache.get(key)) is not None: - # Deserialize from JSON based on the return type - return return_type.model_validate_json(cached) - - # Call the function and cache its result - result = func(*args, **kwargs) - serialized_result = result.model_dump_json() - cache.set(key, serialized_result) - - return result - - return wrapper - ``` - - 1. We create a new `diskcache.Cache` instance to store the cached data. This will create a new directory called `my_cache_directory` in the current working directory. - 2. We only want to cache functions that return a Pydantic model to simplify serialization and deserialization logic in this example code - - Remember that you can change this code to support non-Pydantic models, or to use a different caching backend. More over, don't forget that this cache does not invalidate when the model changes, so you might want to encode the `Model.model_json_schema()` as part of the key. - -**When to Use**: Good for applications that need cache persistence between sessions or deal with large datasets. Use this when you want to reuse the same data across multiple sessions or store large amounts of data. - -```python hl_lines="10" -import functools -import inspect -import instructor -import diskcache -from pydantic import BaseModel - -client = instructor.from_provider("openai/gpt-4.1-mini") -cache = diskcache.Cache('./my_cache_directory') - - -def instructor_cache(func): - """Cache a function that returns a Pydantic model""" - return_type = inspect.signature(func).return_annotation # (4) - if not issubclass(return_type, BaseModel): # (1) - raise ValueError("The return type must be a Pydantic model") - - @functools.wraps(func) - def wrapper(*args, **kwargs): - key = ( - f"{func.__name__}-{functools._make_key(args, kwargs, typed=False)}" # (2) - ) - # Check if the result is already cached - if (cached := cache.get(key)) is not None: - # Deserialize from JSON based on the return type (3) - return return_type.model_validate_json(cached) - - # Call the function and cache its result - result = func(*args, **kwargs) - serialized_result = result.model_dump_json() - cache.set(key, serialized_result) - - return result - - return wrapper - - -class UserDetail(BaseModel): - name: str - age: int - - -@instructor_cache -def extract(data) -> UserDetail: - return client.create( - response_model=UserDetail, - messages=[ - {"role": "user", "content": data}, - ], - ) -``` - -1. We only want to cache functions that return a Pydantic model to simplify serialization and deserialization logic -2. We use functool's `_make_key` to generate a unique key based on the function's name and arguments. This is important because we want to cache the result of each function call separately. -3. We use Pydantic's `model_validate_json` to deserialize the cached result into a Pydantic model. -4. We use `inspect.signature` to get the function's return type annotation, which we use to validate the cached result. - -**Benefits**: Reduces computation time for heavy data processing and provides disk-based caching for persistence. - -## 3. Redis Caching Decorator for Distributed Systems - -??? note "Copy Caching Code" - - The same `instructor_cache` decorator works for both `diskcache` and `redis` caching. Copy the code below and use it for both examples. - - ```python - import functools - import inspect - import redis - - cache = redis.Redis("localhost") - - - def instructor_cache(func): - """Cache a function that returns a Pydantic model""" - return_type = inspect.signature(func).return_annotation - if not issubclass(return_type, BaseModel): - raise ValueError("The return type must be a Pydantic model") - - @functools.wraps(func) - def wrapper(*args, **kwargs): - key = f"{func.__name__}-{functools._make_key(args, kwargs, typed=False)}" - # Check if the result is already cached - if (cached := cache.get(key)) is not None: - # Deserialize from JSON based on the return type - return return_type.model_validate_json(cached) - - # Call the function and cache its result - result = func(*args, **kwargs) - serialized_result = result.model_dump_json() - cache.set(key, serialized_result) - - return result - - return wrapper - ``` - - Remember that you can change this code to support non-Pydantic models, or to use a different caching backend. More over, don't forget that this cache does not invalidate when the model changes, so you might want to encode the `Model.model_json_schema()` as part of the key. - -**When to Use**: Good for distributed systems where multiple processes need to access cached data, or for applications that need fast read/write access and handle complex data structures. - -```python -import redis -import functools -import inspect -import instructor - -from pydantic import BaseModel - -client = instructor.from_provider("openai/gpt-4.1-mini") -cache = redis.Redis("localhost") - - -def instructor_cache(func): - """Cache a function that returns a Pydantic model""" - return_type = inspect.signature(func).return_annotation - if not issubclass(return_type, BaseModel): # (1) - raise ValueError("The return type must be a Pydantic model") - - @functools.wraps(func) - def wrapper(*args, **kwargs): - key = f"{func.__name__}-{functools._make_key(args, kwargs, typed=False)}" # (2) - # Check if the result is already cached - if (cached := cache.get(key)) is not None: - # Deserialize from JSON based on the return type - return return_type.model_validate_json(cached) - - # Call the function and cache its result - result = func(*args, **kwargs) - serialized_result = result.model_dump_json() - cache.set(key, serialized_result) - - return result - - return wrapper - - -class UserDetail(BaseModel): - name: str - age: int - - -@instructor_cache -def extract(data) -> UserDetail: - # Assuming client.chat.completions.create returns a UserDetail instance - return client.create( - response_model=UserDetail, - messages=[ - {"role": "user", "content": data}, - ], - ) -``` - -1. We only want to cache functions that return a Pydantic model to simplify serialization and deserialization logic -2. We use functool's `_make_key` to generate a unique key based on the function's name and arguments. This is important because we want to cache the result of each function call separately. - -**Benefits**: Scalable for large-scale systems, supports fast in-memory data storage and retrieval, and works with various data types. - -!!! note "Same Decorator, Different Backend" - - The code above uses the same `instructor_cache` decorator as before. The implementation is the same, but it uses a different caching backend. diff --git a/참고/instructor-main/docs/concepts/citation.md b/참고/instructor-main/docs/concepts/citation.md deleted file mode 100644 index b40d734..0000000 --- a/참고/instructor-main/docs/concepts/citation.md +++ /dev/null @@ -1,190 +0,0 @@ ---- -title: Citation Extraction with CitationMixin -description: Learn how to extract and validate citations from source text using CitationMixin to prevent hallucinations. ---- - -# Citation Extraction with CitationMixin - -CitationMixin is a Pydantic mixin that helps extract and validate citations from source text. It ensures that quotes used in your extracted data actually exist in the source context, preventing hallucinations. - -## What is CitationMixin? - -CitationMixin adds citation validation to your Pydantic models. When you use it, your model gets a `substring_quotes` field that contains quotes from the source text. The mixin automatically validates that these quotes exist in the source and corrects them to match exact spans. - -## Basic Usage - -Inherit from CitationMixin to add citation support to your model: - -```python -from pydantic import BaseModel, Field -from instructor import CitationMixin -import instructor - - -class User(CitationMixin, BaseModel): - name: str = Field(description="The name of the person") - age: int = Field(description="The age of the person") - role: str = Field(description="The role of the person") - - -client = instructor.from_provider("openai/gpt-4o-mini") - -context = "Betty was a student. Jason was a student. Jason is 20 years old" - -user = client.create( - response_model=User, - messages=[ - { - "role": "user", - "content": f"Extract information about Jason from: {context}", - }, - ], - context={"context": context}, -) - -# Verify quotes exist in context -for quote in user.substring_quotes: - assert quote in context - -print(user.model_dump()) -# { -# "name": "Jason", -# "age": 20, -# "role": "student", -# "substring_quotes": [ -# "Jason was a student", -# "Jason is 20 years old", -# ] -# } -``` - -## How It Works - -CitationMixin works in three steps: - -1. **Extraction**: The LLM extracts data and provides quotes in the `substring_quotes` field -2. **Validation**: The mixin checks if each quote exists in the source context using fuzzy matching -3. **Correction**: Quotes are corrected to match exact spans in the source text - -The validation happens automatically when you pass `context={"context": source_text}` to your `create()` call. - -## Using with Validation Context - -CitationMixin uses Pydantic's validation context to access the source text. Pass the source text in the `context` parameter: - -```python -from pydantic import BaseModel, Field -from instructor import CitationMixin -import instructor - - -class Fact(CitationMixin, BaseModel): - statement: str = Field(description="A factual statement") - # substring_quotes is added automatically by CitationMixin - - -client = instructor.from_provider("openai/gpt-4o-mini") - -source_text = """ -The Eiffel Tower was completed in 1889 and stands 330 meters tall. -It was designed by Gustave Eiffel and is located in Paris, France. -""" - -fact = client.create( - response_model=Fact, - messages=[ - { - "role": "user", - "content": f"Extract facts about the Eiffel Tower from: {source_text}", - }, - ], - context={"context": source_text}, -) - -# All quotes are validated and corrected to exact spans -for quote in fact.substring_quotes: - print(f"Quote: {quote}") - assert quote in source_text -``` - -## Fuzzy Matching - -CitationMixin uses fuzzy matching to find quotes even if they don't match exactly. This handles minor differences like: -- Extra whitespace -- Slight wording variations -- Punctuation differences - -The matching allows up to 5 character errors by default, which helps handle cases where the LLM paraphrases slightly. - -## Advanced Example: Question Answering with Citations - -Use CitationMixin to build question-answering systems that cite sources: - -```python -from typing import List -from pydantic import BaseModel, Field -from instructor import CitationMixin -import instructor - - -class Fact(CitationMixin, BaseModel): - statement: str = Field(description="A factual statement") - - -class Answer(CitationMixin, BaseModel): - question: str - facts: List[Fact] = Field(description="List of facts that answer the question") - - -client = instructor.from_provider("openai/gpt-4o-mini") - -source_text = """ -Jason Liu grew up in Toronto, Canada but was born in China. -He went to an arts high school but studied Computational Mathematics and Physics in university. -He worked at Stitchfix and Facebook as part of coop programs. -He started the Data Science club at the University of Waterloo and was president for 2 years. -""" - -answer = client.create( - response_model=Answer, - messages=[ - { - "role": "system", - "content": "Answer questions with exact citations from the source text.", - }, - { - "role": "user", - "content": f"Source: {source_text}\n\nQuestion: What did Jason do during college?", - }, - ], - context={"context": source_text}, -) - -# Verify all citations exist -for fact in answer.facts: - for quote in fact.substring_quotes: - assert quote in source_text - print(f"Verified: {quote}") -``` - -## When to Use CitationMixin - -Use CitationMixin when: - -- You need to verify that extracted information comes from source text -- You're building RAG (Retrieval Augmented Generation) systems -- You want to prevent hallucinations by validating citations -- You need exact quote spans for highlighting or display - -## Limitations - -- Requires passing source text in `context={"context": ...}` -- Uses fuzzy matching which may not catch all paraphrasing -- Only validates quotes, not the accuracy of extracted facts themselves - -## See Also - -- [Validation](./validation.md) - Learn about validation in Instructor -- [Context-Based Validation](./validation.md#context-based-validation) - Using context for validation -- [Citation Examples](../examples/exact_citations.md) - More citation examples -- [RAG Patterns](../blog/posts/rag-and-beyond.md) - Building RAG systems with Instructor diff --git a/참고/instructor-main/docs/concepts/dictionary_operations.md b/참고/instructor-main/docs/concepts/dictionary_operations.md deleted file mode 100644 index 35d270f..0000000 --- a/참고/instructor-main/docs/concepts/dictionary_operations.md +++ /dev/null @@ -1,121 +0,0 @@ -## See Also - -- [Types](./types.md) - Working with different data types -- [Response Models](./models.md) - Working with Pydantic models -- [Fields](./fields.md) - Customizing field metadata -- [Union Types](./unions.md) - Handle multiple possible types - ---- -title: Dictionary Operations Optimization in Instructor -description: Learn about performance optimizations for dictionary operations in Instructor, including message extraction and configuration parameter handling. ---- - -# Dictionary Operations Optimization - -This document explains the dictionary operations optimizations implemented in Instructor. - -## Overview - -Dictionary operations are one of the most common operations in the Instructor codebase, especially when handling message passing between different LLM providers and managing configuration parameters. Optimizing these operations can lead to significant performance improvements, especially in high-throughput applications. - -## Optimized Areas - -### Message Extraction - -The `extract_messages` function was optimized to use direct key lookups instead of nested `get()` calls, which reduces the overhead of function calls and improves performance. - -**Before:** -```python -from typing import Any - - -def extract_messages(kwargs: dict[str, Any]) -> Any: - return kwargs.get( - "messages", kwargs.get("contents", kwargs.get("chat_history", [])) - ) -``` - -**After:** -```python -from typing import Any - - -def extract_messages(kwargs: dict[str, Any]) -> Any: - if "messages" in kwargs: - return kwargs["messages"] - if "contents" in kwargs: - return kwargs["contents"] - if "chat_history" in kwargs: - return kwargs["chat_history"] - return [] -``` - -### Response Processing Functions - -The response processing functions were optimized to: -1. Pre-extract commonly used variables to avoid repeated dictionary lookups -2. Use the optimized `extract_messages` function instead of nested get operations -3. Reduce redundant dictionary operations in error handling - -### Message Handler Selection - -The `handle_reask_kwargs` function was optimized to use direct conditional checks instead of creating a large mapping dictionary, which reduces memory overhead and improves lookup performance. - -**Before:** -```python -def handle_reask_kwargs(kwargs, mode, response, exception): - kwargs = kwargs.copy() - functions = { - Mode.TOOLS: reask_anthropic_tools, - Mode.JSON: reask_anthropic_json, - # ... many more mappings - } - reask_function = functions.get(mode, reask_default) - return reask_function(kwargs=kwargs, response=response, exception=exception) -``` - -**After:** -```python -def handle_reask_kwargs(kwargs, mode, response, exception): - kwargs_copy = kwargs.copy() - - if mode in {Mode.TOOLS, Mode.ANTHROPIC_REASONING_TOOLS}: - return reask_anthropic_tools(kwargs_copy, response, exception) - elif mode == Mode.JSON: - return reask_anthropic_json(kwargs_copy, response, exception) - # ... optimized conditional checks with grouped modes - else: - return reask_default(kwargs_copy, response, exception) -``` - -### System Message Handling - -The `combine_system_messages` function in `utils.py` was optimized to: -1. Cache type checks to avoid repeated calls -2. Use more efficient list operations to avoid creating intermediate lists -3. Optimize type conversion scenarios - -## Benchmarks - -Benchmarks show significant improvements in dictionary operation performance: - -| Operation | Before (ms) | After (ms) | Improvement | -|-----------|-------------|------------|-------------| -| extract_messages | ~0.08 | ~0.03 | ~62% | -| handle_reask_kwargs | ~0.09 | ~0.05 | ~44% | -| combine_system_messages | ~0.12 | ~0.07 | ~42% | - -The exact improvement depends on the specific use case and data patterns. - -## Testing - -Two types of tests were created to ensure the optimizations were safe: - -1. **Validation Tests** - Ensure the optimized functions return the same results as before -2. **Benchmark Tests** - Measure and verify the performance improvements - -These tests help ensure that the optimizations improve performance without changing behavior. - -## Conclusion - -Dictionary operations optimization is a key part of making Instructor more efficient, especially for high-throughput applications. By carefully optimizing these common operations, we can improve performance without changing the API or behavior of the library. \ No newline at end of file diff --git a/참고/instructor-main/docs/concepts/distillation.md b/참고/instructor-main/docs/concepts/distillation.md deleted file mode 100644 index a22290b..0000000 --- a/참고/instructor-main/docs/concepts/distillation.md +++ /dev/null @@ -1,199 +0,0 @@ ---- -title: Seamless Fine-Tuning of Python Functions Using Instructor's Distillation -description: Learn how to fine-tune language models with Python functions using Instructor's `Instructions` for efficient data preparation and logging. ---- - -## See Also - -- [Response Models](./models.md) - Working with Pydantic models -- [Validation](./validation.md) - Ensuring output quality -- [Types](./types.md) - Working with different data types -- [Custom Validators](../learning/validation/custom_validators.md) - Build custom validation logic - -# Distilling python functions into LLM - -`Instructions` from the `Instructor` library offers a seamless way to make language models backward compatible with existing Python functions. By employing Pydantic type hints, it not only ensures compatibility but also facilitates fine-tuning `gpt-4.1-mini` to emulate these functions end-to-end. - -If you want to see the full example checkout [examples/distillation](https://github.com/jxnl/instructor/tree/main/examples/distilations) - -## The Challenges in Function-Level Fine-Tuning - -Replicating the behavior of a Python function in a language model involves intricate data preparation. For instance, teaching a model to execute three-digit multiplication is not as trivial as implementing `def f(a, b): return a * b`. OpenAI's fine-tuning script coupled with their function calling utility provides a structured output, thereby simplifying the data collection process. Additionally, this eliminates the need for passing the schema to the model, thus conserving tokens. - -## The Role of `Instructions` in Simplifying the Fine-Tuning Process - -By using `Instructions`, you can annotate a Python function that returns a Pydantic object, thereby automating the dataset creation for fine-tuning. A handler for logging is all that's needed to build this dataset. - -## How to Implement `Instructions` in Your Code - -## Quick Start: How to Use Instructor's Distillation Feature - -Before we dig into the nitty-gritty, let's look at how easy it is to use Instructor's distillation feature to use function calling finetuning to export the data to a JSONL file. - -```python -import logging -import random -from pydantic import BaseModel - -# Logging setup -logging.basicConfig(level=logging.INFO) - -from instructor import Instructions, FinetuneFormat # pip install instructor - -instructions = Instructions( - name="three_digit_multiply", - finetune_format=FinetuneFormat.MESSAGES, # or FinetuneFormat.RAW - # log handler is used to save the data to a file - # you can imagine saving it to a database or other storage - # based on your needs! - log_handlers=[logging.FileHandler("math_finetunes.jsonl")], -) - - -class Multiply(BaseModel): - a: int - b: int - result: int - - -# Define a function with distillation -# The decorator will automatically generate a dataset for fine-tuning -# They must return a pydantic model to leverage function calling -@instructions.distil -def fn(a: int, b: int) -> Multiply: - resp = a * b - return Multiply(a=a, b=b, result=resp) - - -# Generate some data -for _ in range(10): - random.seed(42) - a = random.randint(100, 999) - b = random.randint(100, 999) - print(fn(a, b)) - #> a=754 b=214 result=161356 - #> a=754 b=214 result=161356 - #> a=754 b=214 result=161356 - #> a=754 b=214 result=161356 - #> a=754 b=214 result=161356 - #> a=754 b=214 result=161356 - #> a=754 b=214 result=161356 - #> a=754 b=214 result=161356 - #> a=754 b=214 result=161356 - #> a=754 b=214 result=161356 -``` - -## The Intricacies of Fine-tuning Language Models - -Fine-tuning isn't just about writing a function like `def f(a, b): return a * b`. It requires detailed data preparation and logging. However, Instructor provides a built-in logging feature and structured outputs to simplify this. - -## Why Instructor and Distillation are Game Changers - -The library offers two main benefits: - -1. **Efficiency**: Streamlines functions, distilling requirements into model weights and a few lines of code. -2. **Integration**: Eases combining classical machine learning and language models by providing a simple interface that wraps existing functions. - -## Role of Instructor in Simplifying Fine-Tuning - -The `from instructor import Instructions` feature is a time saver. It auto-generates a fine-tuning dataset, making it a breeze to imitate a function's behavior. - -## FinetuneFormat Options - -The `finetune_format` parameter controls how the fine-tuning data is structured. There are two options: - -### MESSAGES Format (Default) - -The `MESSAGES` format creates data in OpenAI's chat completion format with messages and function calls. This is the recommended format for most use cases as it matches OpenAI's fine-tuning API format. - -```python -from instructor import Instructions, FinetuneFormat - -instructions = Instructions( - name="my_function", - finetune_format=FinetuneFormat.MESSAGES, - log_handlers=[logging.FileHandler("output.jsonl")], -) -``` - -### RAW Format - -The `RAW` format creates a simpler format with function metadata, arguments, and response. Use this format if you need more control over the data structure or are using a custom fine-tuning pipeline. - -```python -from instructor import Instructions, FinetuneFormat - -instructions = Instructions( - name="my_function", - finetune_format=FinetuneFormat.RAW, - log_handlers=[logging.FileHandler("output.jsonl")], -) -``` - -## Logging Output and Running a Finetune - -Here's how the logging output would look for MESSAGES format: - -```python -{ - "messages": [ - {"role": "system", "content": 'Predict the results of this function: ...'}, - {"role": "user", "content": 'Return fn(133, b=539)'}, - { - "role": "assistant", - "function_call": { - "name": "Multiply", - "arguments": '{"a":133,"b":539,"result":89509}', - }, - }, - ], - "functions": [ - {"name": "Multiply", "description": "Correctly extracted `Multiply`..."} - ], -} -``` - -For RAW format, the output would look like: - -```python -{ - "fn_name": "three_digit_multiply", - "fn_repr": "def fn(a: int, b: int) -> Multiply:\n ...", - "args": [133], - "kwargs": {"b": 539}, - "response": {"a": 133, "b": 539, "result": 89509} -} -``` - -Run a finetune like this: - -```bash -instructor jobs create-from-file math_finetunes.jsonl -``` - -Once a model is trained you can simply change `mode` to `dispatch` and it will use the model to run the function! - -```python -from instructor import Instructions -from pydantic import BaseModel - - -class Multiply(BaseModel): - a: int - b: int - result: int - - -instructions = Instructions( - name="three_digit_multiply", -) - - -@instructions.distil(model='gpt-4.1-mini:finetuned-123', mode="dispatch") -def fn(a: int, b: int) -> Multiply: - # now this code will be short circuited and the model will be used instead. - resp = a + b - return Multiply(a=a, b=b, result=resp) -``` - -With this, you can swap the function implementation, making it backward compatible. You can even imagine using the different models for different tasks or validating and runnign evals by using the original function and comparing it to the distillation. diff --git a/참고/instructor-main/docs/concepts/enums.md b/참고/instructor-main/docs/concepts/enums.md deleted file mode 100644 index b2e3097..0000000 --- a/참고/instructor-main/docs/concepts/enums.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: Using Enums and Literals in Pydantic for Role Management -description: Learn how to implement Enums and Literals in Pydantic to manage standardized user roles with a fallback option. ---- - -To prevent data misalignment, we can use Enums for standardized fields. Always include an "Other" option as a fallback so the model can signal uncertainty. - -```python hl_lines="7 12" -from pydantic import BaseModel, Field -from enum import Enum - - -class Role(Enum): - PRINCIPAL = "PRINCIPAL" - TEACHER = "TEACHER" - STUDENT = "STUDENT" - OTHER = "OTHER" - - -class UserDetail(BaseModel): - age: int - name: str - role: Role = Field( - description="Correctly assign one of the predefined roles to the user." - ) -``` - -If you're having a hard time with `Enum` an alternative is to use `Literal` instead. - -```python hl_lines="4" -from typing import Literal -from pydantic import BaseModel - - -class UserDetail(BaseModel): - age: int - name: str - role: Literal["PRINCIPAL", "TEACHER", "STUDENT", "OTHER"] -``` - -## See Also - -- [Types](./types.md) - Working with different data types including Literal -- [Union Types](./unions.md) - Using unions with enums for multiple choices -- [Response Models](./models.md) - Using enums in Pydantic models -- [Fields](./fields.md) - Customizing enum fields with Field metadata diff --git a/참고/instructor-main/docs/concepts/error_handling.md b/참고/instructor-main/docs/concepts/error_handling.md deleted file mode 100644 index 5c746f7..0000000 --- a/참고/instructor-main/docs/concepts/error_handling.md +++ /dev/null @@ -1,309 +0,0 @@ ---- -title: Error Handling -description: Learn how to handle errors and exceptions when using Instructor for structured outputs. ---- - -# Error Handling - -Instructor provides a comprehensive exception hierarchy to help you handle errors gracefully. All Instructor exceptions inherit from `InstructorError`. - -## Exception Reference - -| Exception | Description | Key Attributes | -|-----------|-------------|----------------| -| `InstructorError` | Base exception for all Instructor errors | - | -| `IncompleteOutputException` | Output truncated due to token limit | `last_completion` | -| `InstructorRetryException` | All retry attempts exhausted | `n_attempts`, `failed_attempts`, `total_usage` | -| `ValidationError` | Response validation failed | - | -| `ResponseParsingError` | Cannot parse LLM response | `mode`, `raw_response` | -| `ProviderError` | Provider-specific error | `provider` | -| `ConfigurationError` | Invalid configuration | - | -| `ModeError` | Invalid mode for provider | `mode`, `provider`, `valid_modes` | -| `ClientError` | Client initialization failed | - | -| `MultimodalError` | Processing image/audio/PDF failed | `content_type`, `file_path` | -| `AsyncValidationError` | Async validation failed | `errors` | - -## Common Exceptions - -### Incomplete Output - -Raised when the LLM output is truncated due to reaching the token limit: - -```python -import instructor -from pydantic import BaseModel -from instructor.core.exceptions import IncompleteOutputException, InstructorRetryException - - -class Report(BaseModel): - content: str - - -client = instructor.from_provider("openai/gpt-4.1-mini", mode=instructor.Mode.JSON) - -try: - response = client.create( - response_model=Report, - messages=[{"role": "user", "content": "Write a long report..."}], - max_tokens=50, - max_retries=0, - ) -except (IncompleteOutputException, InstructorRetryException) as e: - print(f"Output truncated: {e}") - print(f"Last completion: {e.last_completion}") -``` - -### Retry Exhausted - -Raised when all retry attempts fail: - -```python -import instructor -from pydantic import BaseModel -from instructor.core.exceptions import InstructorRetryException - - -class User(BaseModel): - name: str - age: int - - -client = instructor.from_provider("openai/gpt-4.1-mini") - -try: - response = client.create( - response_model=User, - messages=[{"role": "user", "content": "Extract user info..."}], - max_retries=3, - ) -except InstructorRetryException as e: - print(f"Failed after {e.n_attempts} attempts") - for attempt in e.failed_attempts: - print(f" Attempt {attempt.attempt_number}: {attempt.exception}") -``` - -### Validation Error - -Raised when the response fails validation: - -```python -import instructor -from pydantic import BaseModel, field_validator -from instructor.core.exceptions import ValidationError - - -class StrictModel(BaseModel): - value: int - - @field_validator("value") - @classmethod - def validate_value(cls, v: int) -> int: - if v < 0: - raise ValueError("Value must be positive") - return v - - -client = instructor.from_provider("openai/gpt-4.1-mini") - -try: - response = client.create( - response_model=StrictModel, - messages=[{"role": "user", "content": "Extract data..."}], - ) -except ValidationError as e: - print(f"Validation failed: {e}") -``` - -### Provider and Configuration Errors - -Raised for provider-specific issues or invalid configuration: - -```python -import instructor -from instructor.core.exceptions import ConfigurationError, ModeError - -# Invalid provider format -try: - client = instructor.from_provider("invalid-format") -except ConfigurationError as e: - print(f"Configuration error: {e}") - -# Wrong mode for provider -try: - client = instructor.from_provider( - "openai/gpt-4.1-mini", - mode=instructor.Mode.TOOLS, - ) -except ModeError as e: - print(f"Invalid mode. Valid modes: {e.valid_modes}") -``` - -## Best Practices - -### Catch Specific Exceptions - -```python -import logging -import instructor -from pydantic import BaseModel -from instructor.core.exceptions import ( - IncompleteOutputException, - InstructorRetryException, - ValidationError, -) - -logger = logging.getLogger(__name__) - - -class User(BaseModel): - name: str - age: int - - -client = instructor.from_provider("openai/gpt-4.1-mini") - -try: - response = client.create( - response_model=User, - messages=[{"role": "user", "content": "Extract: Sam is 34"}], - ) -except IncompleteOutputException: - logger.warning("Output truncated, retrying with more tokens") - response = client.create( - response_model=User, - messages=[{"role": "user", "content": "Extract: Sam is 34"}], - max_tokens=2000, - ) -except InstructorRetryException as e: - logger.error(f"Failed after {e.n_attempts} attempts") - response = None -except ValidationError as e: - logger.error(f"Validation failed: {e}") - raise -``` - -### Use Base Exception for General Handling - -```python -import instructor -from pydantic import BaseModel -from instructor.core.exceptions import InstructorError - - -class Data(BaseModel): - value: str - - -client = instructor.from_provider("openai/gpt-4.1-mini") - -try: - response = client.create( - response_model=Data, - messages=[{"role": "user", "content": "Extract data"}], - ) -except InstructorError as e: - # Catches any Instructor-specific error - print(f"Instructor error: {type(e).__name__}: {e}") -``` - -### Graceful Degradation - -```python -import instructor -from pydantic import BaseModel, field_validator -from instructor.core.exceptions import ValidationError, InstructorRetryException - - -class StrictData(BaseModel): - value: int - - @field_validator("value") - @classmethod - def validate_value(cls, v: int) -> int: - if v < 0: - raise ValueError("Value must be positive") - return v - - -class RelaxedData(BaseModel): - value: str - - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -def extract_with_fallback(content: str): - try: - return client.create( - response_model=StrictData, - messages=[{"role": "user", "content": content}], - ) - except ValidationError: - # Fall back to less strict model - return client.create( - response_model=RelaxedData, - messages=[{"role": "user", "content": content}], - ) - except InstructorRetryException: - return None -``` - -## Backwards Compatibility - -New exceptions inherit from both `ValueError` and `InstructorError`, so existing code continues to work: - -```python -import instructor -from pydantic import BaseModel -from instructor.core.exceptions import ResponseParsingError - - -class User(BaseModel): - name: str - age: int - - -client = instructor.from_provider("openai/gpt-4.1-mini") - -# Old code still works -try: - response = client.create( - response_model=User, - messages=[{"role": "user", "content": "Extract: Kai is 41"}], - ) -except ValueError as e: - print(f"Error: {e}") - -# New code can access additional context -try: - response = client.create( - response_model=User, - messages=[{"role": "user", "content": "Extract: Kai is 41"}], - ) -except ResponseParsingError as e: - print(f"Mode: {e.mode}, Raw: {e.raw_response}") -``` - -## Integration with Hooks - -Monitor errors using the hooks system: - -```python -import instructor -from instructor.core.exceptions import ValidationError - - -def on_parse_error(error: Exception): - if isinstance(error, ValidationError): - print(f"Validation error: {error}") - - -client = instructor.from_provider("openai/gpt-4.1-mini") -client.hooks.on("parse:error", on_parse_error) -``` - -## See Also - -- [Retrying](./retrying.md) - Retry strategies with Tenacity -- [Validation](./validation.md) - Validation patterns -- [Hooks](./hooks.md) - Error monitoring with hooks diff --git a/참고/instructor-main/docs/concepts/fastapi.md b/참고/instructor-main/docs/concepts/fastapi.md deleted file mode 100644 index 7612ab4..0000000 --- a/참고/instructor-main/docs/concepts/fastapi.md +++ /dev/null @@ -1,107 +0,0 @@ ---- -title: FastAPI Integration with Instructor - API Development Guide -description: Build production-ready APIs with FastAPI and Instructor. Create type-safe endpoints for structured LLM outputs with automatic validation and documentation. ---- - -# Integrating Pydantic Models with FastAPI - -[FastAPI](https://fastapi.tiangolo.com/) is an enjoyable tool for building web applications in Python. It is well known for its integration with `Pydantic` models, which makes defining and validating data structures straightforward and efficient. In this guide, we explore how simple functions that return `Pydantic` models can seamlessly integrate with `FastAPI`. - -## Why Choose FastAPI and Pydantic? - -- FastAPI is a modern, high-performance web framework for building APIs with Python. -- Supports OpenAPI and JSON Schema for automatic documentation and validation. -- Supports AsyncIO for asynchronous programming leveraging the AsyncOpenAI() client - -## Code Example: Starting a FastAPI App with a POST Request - -The following code snippet demonstrates how to start a `FastAPI` app with a POST endpoint. This endpoint accepts and returns data defined by a `Pydantic` model. - -```python -import instructor - -from fastapi import FastAPI -from pydantic import BaseModel - -# Enables response_model -client = instructor.from_provider( - "openai/gpt-4.1-mini", - async_client=True, -) -app = FastAPI() - - -class UserData(BaseModel): - # This can be the model for the input data - query: str - - -class UserDetail(BaseModel): - name: str - age: int - - -@app.post("/endpoint", response_model=UserDetail) -async def endpoint_function(data: UserData) -> UserDetail: - user_detail = await client.create( - response_model=UserDetail, - messages=[ - {"role": "user", "content": f"Extract: `{data.query}`"}, - ], - ) - return user_detail -``` - -## Streaming Responses with FastAPI - -`FastAPI` supports streaming responses, which is useful for returning large amounts of data. This feature is particularly useful when working with large language models (LLMs) that generate a large amount of data. - -```python hl_lines="6-7" -from fastapi import FastAPI -from fastapi.responses import StreamingResponse -from typing import Iterable -from pydantic import BaseModel - -app = FastAPI() - - -class UserData(BaseModel): - query: str - - -class UserDetail(BaseModel): - name: str - age: int - - -# Route to handle SSE events and return users -@app.post("/extract", response_class=StreamingResponse) -async def extract(data: UserData): - users = await client.create( - response_model=Iterable[UserDetail], - stream=True, - messages=[ - {"role": "user", "content": data.query}, - ], - ) - - async def generate(): - async for user in users: - resp_json = user.model_dump_json() - yield f"data: {resp_json}" - yield "data: [DONE]" - - return StreamingResponse(generate(), media_type="text/event-stream") -``` - -## Automatic Documentation with FastAPI - -FastAPI leverages the OpenAPI specification to automatically generate a dynamic and interactive documentation page, commonly referred to as the `/docs` page. This feature is incredibly useful for developers, as it offers a live environment to test API endpoints directly through the browser. - -To explore the capabilities of your API, follow these steps: - -1. Run the API using the Uvicorn command: `uvicorn main:app --reload`. -2. Open your web browser and navigate to `http://127.0.0.1:8000/docs`. -3. You will find an interactive UI where you can send different requests to your API and see the responses in real-time. - -![Screenshot of FastAPI /docs page](response.png) diff --git a/참고/instructor-main/docs/concepts/fields.md b/참고/instructor-main/docs/concepts/fields.md deleted file mode 100644 index 9dd21be..0000000 --- a/참고/instructor-main/docs/concepts/fields.md +++ /dev/null @@ -1,179 +0,0 @@ ---- -title: Customizing Pydantic Models with Field Metadata -description: Learn how to enhance Pydantic models with metadata using Field, including default values, JSON schema customization, and more. ---- - -The `pydantic.Field` function is used to customize and add metadata to fields of models. To learn more, check out the Pydantic [documentation](https://docs.pydantic.dev/latest/concepts/fields/) as this is a near replica of that documentation that is relevant to prompting. - -## Default values - -The `default` parameter is used to define a default value for a field. - -```py -from pydantic import BaseModel, Field - - -class User(BaseModel): - name: str = Field(default='John Doe') - - -user = User() -print(user) -#> name='John Doe' -``` - -You can also use `default_factory` to define a callable that will be called to generate a default value. - -```py -from uuid import uuid4 - -from pydantic import BaseModel, Field - - -class User(BaseModel): - id: str = Field(default_factory=lambda: uuid4().hex) -``` - -!!! info - - The `default` and `default_factory` parameters are mutually exclusive. - -!!! note - - If you use `typing.Optional`, it doesn't mean that the field has a default value of `None` you must use `default` or `default_factory` to define a default value. Then it will be considered `not required` when sent to the language model. - -## Using `Annotated` - -The `Field` function can also be used together with `Annotated`. - -```py -from uuid import uuid4 -from typing_extensions import Annotated -from pydantic import BaseModel, Field - - -class User(BaseModel): - id: Annotated[str, Field(default_factory=lambda: uuid4().hex)] -``` - -## Exclude - -The `exclude` parameter can be used to control which fields should be excluded from the -model when exporting the model. This is helpful when you want to exclude fields that are not relevant to the model -generation like `scratch_pad` or `chain_of_thought` - -See the following example: - -```py -from pydantic import BaseModel, Field -from datetime import date - - -class DateRange(BaseModel): - chain_of_thought: str = Field( - description="Reasoning behind the date range.", exclude=True - ) - start_date: date - end_date: date - - -date_range = DateRange( - chain_of_thought=""" - I want to find the date range for the last 30 days. - Today is 2021-01-30 therefore the start date - should be 2021-01-01 and the end date is 2021-01-30""", - start_date=date(2021, 1, 1), - end_date=date(2021, 1, 30), -) -print(date_range.model_dump_json()) -#> {"start_date":"2021-01-01","end_date":"2021-01-30"} -``` - -## Omitting fields from schema sent to the language model - -In some cases, you may wish to have the language model ignore certain fields in your model. You can do this by using Pydantic's `SkipJsonSchema` annotation. This omits a field from the JSON schema emitted by Pydantic (which `instructor` uses for constructing its prompts and tool definitions). For example: - -```py -from pydantic import BaseModel -from pydantic.json_schema import SkipJsonSchema -from typing import Union - - -class Response(BaseModel): - question: str - answer: str - private_field: SkipJsonSchema[Union[str, None]] = None - - -assert "private_field" not in Response.model_json_schema()["properties"] -``` - -Note that because the language model will never return a value for `private_field`, you'll need a default value (this can be a generator via a declared Pydantic `Field`). - -## Customizing JSON Schema - -There are some fields that are exclusively used to customise the generated JSON Schema: - -- `title`: The title of the field. -- `description`: The description of the field. -- `examples`: The examples of the field. -- `json_schema_extra`: Extra JSON Schema properties to be added to the field. - -These all work as great opportunities to add more information to the JSON schema as part of your prompt engineering. - -Here's an example: - -```py -from pydantic import BaseModel, Field, SecretStr - - -class User(BaseModel): - age: int = Field(description='Age of the user') - name: str = Field(title='Username') - password: SecretStr = Field( - json_schema_extra={ - 'title': 'Password', - 'description': 'Password of the user', - 'examples': ['123456'], - } - ) - - -print(User.model_json_schema()) -""" -{ - 'properties': { - 'age': {'description': 'Age of the user', 'title': 'Age', 'type': 'integer'}, - 'name': {'title': 'Username', 'type': 'string'}, - 'password': { - 'description': 'Password of the user', - 'examples': ['123456'], - 'format': 'password', - 'title': 'Password', - 'type': 'string', - 'writeOnly': True, - }, - }, - 'required': ['age', 'name', 'password'], - 'title': 'User', - 'type': 'object', -} -""" -``` - -## See Also - -- [Response Models](./models.md) - Using Pydantic models with Instructor -- [Fields Tutorial](../learning/patterns/field_validation.md) - Field-level validation patterns -- [Types](./types.md) - Working with different field types -- [Pydantic Fields Documentation](https://docs.pydantic.dev/latest/concepts/fields/) - Complete Field reference - -# General notes on JSON schema generation - -- The JSON schema for Optional fields indicates that the value null is allowed. -- The Decimal type is exposed in JSON schema (and serialized) as a string. -- The JSON schema does not preserve namedtuples as namedtuples. -- When they differ, you can specify whether you want the JSON schema to represent the inputs to validation or the outputs from serialization. -- Sub-models used are added to the `$defs` JSON attribute and referenced, as per the spec. -- Sub-models with modifications (via the Field class) like a custom title, description, or default value, are recursively included instead of referenced. -- The description for models is taken from either the docstring of the class or the argument description to the Field class. diff --git a/참고/instructor-main/docs/concepts/from_provider.md b/참고/instructor-main/docs/concepts/from_provider.md deleted file mode 100644 index fcbfae7..0000000 --- a/참고/instructor-main/docs/concepts/from_provider.md +++ /dev/null @@ -1,386 +0,0 @@ ---- -title: Using from_provider for Unified Client Creation -description: Learn how to use from_provider to create Instructor clients for any LLM provider. ---- - -# Using from_provider - -The `from_provider` function creates Instructor clients for any LLM provider. It uses the same interface across all providers, making it easy to switch between models. - -!!! note "V2 Preview" - - `from_provider` routes to the v2 implementation by default for supported providers. Legacy provider-specific modes are deprecated, emit warnings, and map to generic modes (`Mode.TOOLS`, `Mode.JSON`, `Mode.JSON_SCHEMA`, `Mode.MD_JSON`). - -## Why Use from_provider? - -`from_provider` provides: - -- Simple syntax: One function works for all providers -- Automatic setup: Handles provider-specific configuration automatically -- Consistent interface: Same code works across different providers -- Type safety: Full IDE support with proper type inference -- Easy switching: Change providers with a single string change - -## Basic Usage - -The basic syntax is simple: `instructor.from_provider("provider/model-name")` - -```python -import instructor -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -# Create a client for any provider -client = instructor.from_provider("openai/gpt-4o-mini") -# Or: instructor.from_provider("anthropic/claude-3-5-sonnet") -# Or: instructor.from_provider("google/gemini-2.5-flash") - -# Use the client as usual -user = client.create( - response_model=User, - messages=[{"role": "user", "content": "Extract: John is 30 years old"}], -) -``` - -## Supported Providers - -`from_provider` supports all major LLM providers: - -### Cloud Providers - -- OpenAI: `"openai/gpt-4o"`, `"openai/gpt-4o-mini"`, `"openai/gpt-4-turbo"` -- Anthropic: `"anthropic/claude-3-5-sonnet"`, `"anthropic/claude-3-opus"` -- Google: `"google/gemini-2.5-flash"`, `"google/gemini-pro"` -- Azure OpenAI: `"azure_openai/gpt-4o"` -- AWS Bedrock: `"bedrock/claude-3-5-sonnet"` -- Vertex AI: `"vertexai/gemini-pro"` (or use `"google/gemini-pro"` with `vertexai=True`) - -### Fast Inference Providers - -- Groq: `"groq/llama-3.1-70b"` -- Fireworks: `"fireworks/mixtral-8x7b"` -- Together: `"together/meta-llama/Llama-3-70b"` -- Anyscale: `"anyscale/meta-llama/Llama-3-70b"` - -### Other Providers - -- Mistral: `"mistral/mistral-large"` -- Cohere: `"cohere/command-r-plus"` -- Perplexity: `"perplexity/llama-3.1-sonar"` -- DeepSeek: `"deepseek/deepseek-chat"` -- xAI: `"xai/grok-beta"` -- OpenRouter: `"openrouter/meta-llama/llama-3.1-70b"` -- Ollama: `"ollama/llama3"` (local models) -- LiteLLM: `"litellm/gpt-4o"` (meta-provider) - -See the [Integrations](../integrations/index.md) section for complete provider documentation. - -## Provider String Format - -The provider string follows the format: `"provider/model-name"` - -```python -# Correct formats -"openai/gpt-4o" -"anthropic/claude-3-5-sonnet-20241022" -"google/gemini-2.5-flash" - -# Incorrect formats (will raise errors) -"gpt-4o" # Missing provider prefix -"openai" # Missing model name -"openai/gpt-4o/mini" # Too many slashes -``` - -## Async Clients - -Create async clients by setting `async_client=True`: - -```python -import asyncio -import instructor -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -async def main() -> None: - # Create async client - async_client = instructor.from_provider("openai/gpt-4o-mini", async_client=True) - - # Use with await - await async_client.create( - response_model=User, - messages=[{"role": "user", "content": "Extract: Alice is 25"}], - ) - - -asyncio.run(main()) -``` - -## Advanced Configuration - -### Custom API Keys - -Pass API keys directly or use environment variables: - -```python -import instructor - -# Pass API key directly -client = instructor.from_provider("openai/gpt-4o-mini", api_key="sk-your-key-here") - -# Or use environment variables (recommended) -# export OPENAI_API_KEY=sk-your-key-here -client = instructor.from_provider("openai/gpt-4o-mini") -``` - -### Mode Overrides - -Override the default mode for a provider: - -```python -import instructor - -# OpenAI defaults to TOOLS mode, but you can override -client = instructor.from_provider( - "openai/gpt-4o-mini", mode=instructor.Mode.JSON # Use JSON mode instead -) -``` - -### Caching - -Enable response caching: - -```python -from instructor.cache import AutoCache -import instructor - -cache = AutoCache(maxsize=1000) - -client = instructor.from_provider("openai/gpt-4o-mini", cache=cache) -``` - -### Provider-Specific Options - -Pass provider-specific options through `**kwargs`: - -```python -import os -import instructor - -# For OpenAI -client = instructor.from_provider( - "openai/gpt-4o-mini", organization="org-your-org-id", timeout=30.0 -) - -# For Anthropic -client = instructor.from_provider("anthropic/claude-3-5-sonnet", max_tokens=4096) - -# For Google with Vertex AI -google_api_key = os.environ.pop("GOOGLE_API_KEY", None) - -client = instructor.from_provider( - "google/gemini-pro", - vertexai=True, - project="your-project-id", - location="us-central1", -) - -if google_api_key is not None: - os.environ["GOOGLE_API_KEY"] = google_api_key -``` - -## Default Modes - -Each provider uses a recommended default mode: - -- OpenAI: `Mode.TOOLS` -- Anthropic: `Mode.TOOLS` -- Google: `Mode.TOOLS` or `Mode.JSON` based on the model -- Ollama: `Mode.TOOLS` (if supported) or `Mode.JSON` -- Others: `Mode.TOOLS` or `Mode.MD_JSON` depending on capability - -Legacy provider-specific modes still work but are deprecated. See the [Mode Migration Guide](./mode-migration.md) for details. - -Override these defaults with the `mode` parameter. - -## Error Handling - -`from_provider` raises clear errors for common issues: - -```python -import instructor -from instructor.core.exceptions import ConfigurationError - -try: - # Invalid provider format - client = instructor.from_provider("invalid-format") -except ConfigurationError as e: - print(f"Configuration error: {e}") - """ - Configuration error: Model string must be in format "provider/model-name" (e.g. "openai/gpt-4" or "anthropic/claude-3-sonnet") - """ - -try: - # Unsupported provider - client = instructor.from_provider("unsupported/provider") -except ConfigurationError as e: - print(f"Unsupported provider: {e}") - """ - Unsupported provider: Unsupported provider: unsupported. Supported providers are: ['openai', 'azure_openai', 'databricks', 'anthropic', 'google', 'generative-ai', 'vertexai', 'mistral', 'cohere', 'perplexity', 'groq', 'writer', 'bedrock', 'cerebras', 'deepseek', 'fireworks', 'ollama', 'openrouter', 'xai', 'litellm'] - """ - -try: - # Missing required package - client = instructor.from_provider("anthropic/claude-3") -except ImportError as e: - print(f"Missing package: {e}") - # Install with: pip install anthropic -``` - -## Environment Variables - -Most providers support environment variables for configuration: - -```bash -# OpenAI -export OPENAI_API_KEY=sk-your-key - -# Anthropic -export ANTHROPIC_API_KEY=sk-ant-your-key - -# Google -export GOOGLE_API_KEY=your-key - -# Azure OpenAI -export AZURE_OPENAI_API_KEY=your-key -export AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/ - -# AWS Bedrock -export AWS_DEFAULT_REGION=us-east-1 -export AWS_ACCESS_KEY_ID=your-key -export AWS_SECRET_ACCESS_KEY=your-secret - -# Others -export MISTRAL_API_KEY=your-key -export COHERE_API_KEY=your-key -export GROQ_API_KEY=your-key -export DEEPSEEK_API_KEY=your-key -export OPENROUTER_API_KEY=your-key -``` - -## Switching Between Providers - -One of the biggest advantages of `from_provider` is easy provider switching: - -```python -import instructor -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -# Easy to switch providers -PROVIDER = "openai/gpt-4o-mini" # Change this to switch -# PROVIDER = "anthropic/claude-3-5-sonnet" -# PROVIDER = "google/gemini-2.5-flash" - -client = instructor.from_provider(PROVIDER) - -# Same code works for all providers -user = client.create( - response_model=User, - messages=[{"role": "user", "content": "Extract: Bob is 40"}], -) -``` - -## Best Practices - -1. Use environment variables: Store API keys in environment variables, not in code -2. Use type hints: Let your IDE help with autocomplete and type checking -3. Handle errors: Wrap provider creation in try-except blocks -4. Cache when appropriate: Use caching for repeated requests -5. Choose the right mode: Let defaults work, but override when needed - -## Comparison with Other Methods - -### from_provider vs. Manual Patching - -```python -# Old way (still works, but more verbose) -import openai -import instructor - -openai_client = openai.OpenAI() -client = instructor.patch(openai_client) - -# New way (recommended) -client = instructor.from_provider("openai/gpt-4o-mini") -``` - -### from_provider vs. Provider-Specific Functions - -Provider-specific helpers were removed. Use `from_provider` for all clients: - -```python -import instructor - -openai_client = instructor.from_provider("openai/gpt-4o-mini") -anthropic_client = instructor.from_provider("anthropic/claude-3-5-sonnet") -``` - -## Troubleshooting - -### Provider Not Found - -If you get an error about an unsupported provider: - -1. Check the provider name spelling -2. Verify the provider is in the supported list -3. Check if you need to install an extra package: `uv pip install "instructor[provider-name]"` - -### Import Errors - -If you get import errors: - -```bash -# Install the required package -# For Anthropic -uv pip install anthropic - -# For Google -uv pip install google-genai - -# For others, see integration docs -``` - -### Invalid Model String - -The model string must be in format `"provider/model-name"`: - -```python -# Correct -"openai/gpt-4o" - -# Incorrect -"gpt-4o" # Missing provider -"openai" # Missing model -``` - -## Related Documentation - -- [Getting Started](../getting-started.md) - Quick start guide -- [Patching](./patching.md) - How Instructor enhances clients -- [Integrations](../integrations/index.md) - Provider-specific documentation -- [Migration Guide](./migration.md) - Migrating from old patterns diff --git a/참고/instructor-main/docs/concepts/hooks.md b/참고/instructor-main/docs/concepts/hooks.md deleted file mode 100644 index f9fb3d3..0000000 --- a/참고/instructor-main/docs/concepts/hooks.md +++ /dev/null @@ -1,283 +0,0 @@ ---- -title: Hooks -description: Learn how to use hooks for event handling, logging, and error handling in Instructor. ---- - -# Hooks - -Hooks let you intercept and handle events during the completion and parsing process. Use them to add logging, monitoring, or error handling at different stages of API interactions. - -## Hook Events - -| Event | Description | Handler Signature | -|-------|-------------|-------------------| -| `completion:kwargs` | Arguments passed to completion | `def handler(*args, **kwargs)` | -| `completion:response` | Raw API response received | `def handler(response)` | -| `completion:error` | Error during a retry attempt | `def handler(error, *, attempt_number, max_attempts, is_last_attempt)` | -| `parse:error` | Pydantic validation failed | `def handler(error)` | -| `completion:last_attempt` | Final retry attempt exhausted | `def handler(error, *, attempt_number, max_attempts, is_last_attempt)` | - -`completion:error` and `completion:last_attempt` handlers receive optional retry metadata as keyword arguments. Old-style handlers that only accept `error` continue to work — the metadata is silently dropped for backward compatibility. - -## Registering and Removing Hooks - -```python -import instructor - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -def log_kwargs(*args, **kwargs): - print(f"Model: {kwargs.get('model')}") - - -def log_response(response): - print(f"Response received: {response.id}") - - -# Register hooks -client.on("completion:kwargs", log_kwargs) -client.on("completion:response", log_response) - -# Make a request -resp = client.create( - messages=[{"role": "user", "content": "Hello, world!"}], - response_model=str, -) - -# Remove a specific hook -client.off("completion:kwargs", log_kwargs) - -# Clear all hooks for an event -client.clear("completion:kwargs") - -# Clear all hooks -client.clear() -``` - -You can use enum values or strings for hook names: - -```python -from instructor.hooks import HookName - -client.on(HookName.COMPLETION_KWARGS, log_kwargs) # Using enum -client.on("completion:kwargs", log_kwargs) # Using string -``` - -## Retry Metadata - -`completion:error` and `completion:last_attempt` handlers can receive attempt metadata: - -```python -import instructor - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -def on_error(error: Exception, *, attempt_number: int, max_attempts: int | None, is_last_attempt: bool): - print(f"Attempt {attempt_number}/{max_attempts or '?'} failed: {error}") - if is_last_attempt: - print("No more retries.") - - -client.on("completion:error", on_error) -``` - -Old-style handlers that only accept `error` continue to work unchanged — the metadata is silently dropped. - -## Practical Example: Logging - -```python -import instructor -from pydantic import BaseModel - - -class ErrorCounter: - def __init__(self): - self.count = 0 - - def handle_error(self, error: Exception): - self.count += 1 - print(f"Error #{self.count}: {type(error).__name__}: {error}") - - -client = instructor.from_provider("openai/gpt-4.1-mini") -counter = ErrorCounter() - -client.on("completion:error", counter.handle_error) -client.on("parse:error", counter.handle_error) - - -class User(BaseModel): - name: str - age: int - - -try: - user = client.create( - messages=[{"role": "user", "content": "Extract: John is twenty"}], - response_model=User, - ) - print(f"Extracted: {user}") -except Exception as e: - print(f"Final error: {e}") - -print(f"Total errors: {counter.count}") -``` - -## Error Handling - -Monitor errors by type using Instructor's exception hierarchy: - -```python -import logging -import instructor -from instructor.core.exceptions import ( - IncompleteOutputException, - InstructorRetryException, - ValidationError, - ProviderError, -) - -logger = logging.getLogger(__name__) - - -def handle_error(error: Exception): - if isinstance(error, IncompleteOutputException): - logger.warning(f"Incomplete output: {error}") - elif isinstance(error, ValidationError): - logger.error(f"Validation failed: {error}") - elif isinstance(error, ProviderError): - logger.error(f"Provider error ({error.provider}): {error}") - elif isinstance(error, InstructorRetryException): - logger.critical(f"Retries exhausted after {error.n_attempts} attempts") - else: - logger.error(f"Unexpected error: {error}") - - -client = instructor.from_provider("openai/gpt-4.1-mini") -client.on("completion:error", handle_error) -client.on("parse:error", handle_error) -``` - -## Hook Combination - -Combine different hook sets using the `+` operator: - -```python -import instructor -from instructor.core.hooks import Hooks - -# Create specialized hook sets -logging_hooks = Hooks() -logging_hooks.on("completion:kwargs", lambda **kw: print("Logging kwargs")) - -metrics_hooks = Hooks() -metrics_hooks.on("completion:response", lambda resp: print("Recording metrics")) - -# Combine hooks -combined = logging_hooks + metrics_hooks - -# Or combine multiple at once -all_hooks = Hooks.combine(logging_hooks, metrics_hooks) - -client = instructor.from_provider("openai/gpt-4.1-mini", hooks=combined) -``` - -## Per-Call Hooks - -Specify hooks for individual API calls: - -```python -import instructor -from instructor.core.hooks import Hooks -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -# Client with standard hooks -client_hooks = Hooks() -client_hooks.on("completion:kwargs", lambda **kw: print("Standard logging")) - -client = instructor.from_provider("openai/gpt-4.1-mini", hooks=client_hooks) - -# Debug hooks for specific calls -debug_hooks = Hooks() -debug_hooks.on("parse:error", lambda err: print(f"Debug: {err}")) - -# Per-call hooks combine with client hooks -user = client.create( - messages=[{"role": "user", "content": "Extract: Alice is 25"}], - response_model=User, - hooks=debug_hooks, # Both client and debug hooks run -) -``` - -## Testing with Hooks - -Use hooks to inspect requests and responses in tests: - -```python -import unittest -from unittest.mock import Mock -import instructor - - -class TestMyApp(unittest.TestCase): - def test_completion(self): - client = instructor.from_provider("openai/gpt-4.1-mini") - mock_handler = Mock() - - client.on("completion:response", mock_handler) - - result = client.create( - messages=[{"role": "user", "content": "Hello"}], - response_model=str, - ) - - mock_handler.assert_called_once() - response = mock_handler.call_args[0][0] - self.assertEqual(response.model, "gpt-4.1-mini") -``` - -## Custom Hooks - -Create custom hook systems by extending the base pattern: - -```python -from enum import Enum -from instructor.hooks import HookName - - -class CustomHookName(str, Enum): - CUSTOM_EVENT = "custom:event" - # Include base hooks for compatibility - COMPLETION_KWARGS = HookName.COMPLETION_KWARGS.value - - -class CustomHooks: - def __init__(self): - self._handlers: dict[str, list] = {} - - def on(self, hook_name: CustomHookName, handler): - self._handlers.setdefault(hook_name.value, []).append(handler) - - def emit(self, hook_name: CustomHookName, payload): - for handler in self._handlers.get(hook_name.value, []): - handler(payload) - - -hooks = CustomHooks() -hooks.on(CustomHookName.CUSTOM_EVENT, lambda data: print(f"Custom: {data}")) -hooks.emit(CustomHookName.CUSTOM_EVENT, {"key": "value"}) -``` - -## See Also - -- [Debugging](../debugging.md) - Practical debugging techniques -- [Retrying](./retrying.md) - Monitor retry attempts -- [Error Handling](./error_handling.md) - Exception handling patterns diff --git a/참고/instructor-main/docs/concepts/index.md b/참고/instructor-main/docs/concepts/index.md deleted file mode 100644 index 44422e4..0000000 --- a/참고/instructor-main/docs/concepts/index.md +++ /dev/null @@ -1,122 +0,0 @@ ---- -title: Instructor Concepts - Core Features and Patterns -description: Explore core concepts and features of the Instructor library. Learn about structured outputs, validation, streaming, and advanced patterns. ---- - -# Instructor Concepts - -This section explains the core concepts and features of the Instructor library, organized by category to help you find what you need. - -## Core Concepts - -These are the fundamental concepts you need to understand to use Instructor effectively: - -- [Models](./models.md) - Using Pydantic models to define output structures -- [Patching](./patching.md) - How Instructor patches LLM clients -- [from_provider](./from_provider.md) - Unified interface for creating clients across all providers -- [Migration Guide](./migration.md) - Migrating from older patterns to from_provider -- [Types](./types.md) - Working with different data types in your models -- [Validation](./validation.md) - Validating LLM outputs against your models -- [Prompting](./prompting.md) - Creating effective prompts for structured output extraction -- [Multimodal](./multimodal.md) - Working with Audio Files, Images and PDFs - -## Data Handling and Structures - -These concepts relate to defining and working with different data structures: - -- [Fields](./fields.md) - Working with Pydantic fields and attributes -- [Lists and Arrays](./lists.md) - Handling lists and arrays in your models -- [TypedDicts](./typeddicts.md) - Using TypedDict for flexible typing -- [Union Types](./unions.md) - Working with union types -- [Enums](./enums.md) - Using enumerated types in your models -- [Missing](./maybe.md) - Handling missing or optional values -- [Alias](./alias.md) - Create field aliases -- [Citation](./citation.md) - Extract and validate citations from source text - -## Streaming Features - -These features help you work with streaming responses: - -- [Stream Partial](./partial.md) - Stream partially completed responses -- [Stream Iterable](./iterable.md) - Stream collections of completed objects -- [Raw Response](./raw_response.md) - Access the raw LLM response - -## Error Handling and Validation - -These features help you ensure data quality: - -- [Retrying](./retrying.md) - Configure automatic retry behavior -- [Validators](./reask_validation.md) - Define custom validation logic -- [Hooks](./hooks.md) - Add callbacks for monitoring and debugging - -## Performance Optimization - -These features help you optimize performance: - -- [Caching](./caching.md) - Cache responses to improve performance -- [Prompt Caching](./prompt_caching.md) - Cache prompts to reduce token usage -- [Usage Tokens](./usage.md) - Track token usage -- [Parallel Tools](./parallel.md) - Run multiple tools in parallel -- [Dictionary Operations](./dictionary_operations.md) - Performance optimizations for dictionary operations - -## Integration Features - -These features help you integrate with other technologies: - -- [FastAPI](./fastapi.md) - Integrate with FastAPI -- [Type Adapter](./typeadapter.md) - Use TypeAdapter with Instructor -- [Templating](./templating.md) - Use templates for dynamic prompts -- [Distillation](./distillation.md) - Optimize models for production - -## Philosophy - -- [Philosophy](./philosophy.md) - The guiding principles behind Instructor - -## How These Concepts Work Together - -Instructor is built around a few key ideas that work together: - -1. **Define Structure with Pydantic**: Use Pydantic models to define exactly what data you want. -2. **Create Clients with from_provider**: Use the unified interface to create clients for any provider. -3. **Validate and Retry**: Automatically validate responses and retry if necessary. -4. **Process Streams**: Handle streaming responses for real-time updates. - -### Typical Workflow - -```mermaid -sequenceDiagram - participant User as Your Code - participant Instructor - participant LLM as LLM Provider - - User->>Instructor: Define Pydantic model - User->>Instructor: Create client with from_provider - User->>Instructor: Call create() with response_model - Instructor->>LLM: Send structured request - LLM->>Instructor: Return LLM response - Instructor->>Instructor: Validate against model - - alt Validation Success - Instructor->>User: Return validated Pydantic object - else Validation Failure - Instructor->>LLM: Retry with error context - LLM->>Instructor: Return new response - Instructor->>Instructor: Validate again - Instructor->>User: Return validated object or error - end -``` - -## What to Read Next - -- If you're new to Instructor, start with [Models](./models.md) and [from_provider](./from_provider.md) -- If you're migrating from older patterns, see the [Migration Guide](./migration.md) -- If you're having validation issues, check out [Validators](./reask_validation.md) and [Retrying](./retrying.md) -- For streaming applications, read [Stream Partial](./partial.md) and [Stream Iterable](./iterable.md) -- To optimize your application, look at [Caching](./caching.md) and [Usage Tokens](./usage.md) - -For practical examples of these concepts, visit the [Cookbook](../examples/index.md) section. - -!!! see-also "See Also" - - [Getting Started Guide](../getting-started.md) - Begin your journey with Instructor - - [Examples](../examples/index.md) - Practical implementations of these concepts - - [Integrations](../integrations/index.md) - Connect with different LLM providers diff --git a/참고/instructor-main/docs/concepts/iterable.md b/참고/instructor-main/docs/concepts/iterable.md deleted file mode 100644 index c3848af..0000000 --- a/참고/instructor-main/docs/concepts/iterable.md +++ /dev/null @@ -1,268 +0,0 @@ ---- -title: Iterable Extraction with Instructor - Stream Multiple Objects -description: Use Iterable types to extract and stream multiple structured objects from LLM responses. Perfect for entity extraction and multi-task outputs. ---- - -# Multi-Task and Streaming - -Using an `Iterable` lets you extract multiple structured objects from a single LLM call, streaming them as they arrive. This is useful for entity extraction, multi-task outputs, and more. - -**We recommend using the `create_iterable` method for most use cases.** It's simpler and less error-prone than manually specifying `Iterable[...]` and `stream=True`. - -Here's a simple example showing how to extract multiple users from a single sentence. You can use either the recommended `create_iterable` method or the `create` method with `Iterable[User]`: - -=== "Using `create_iterable` (recommended)" - ```python - import instructor - from pydantic import BaseModel - - client = instructor.from_provider("openai/gpt-4.1-mini") - - - class User(BaseModel): - name: str - age: int - - - resp = client.create_iterable( - messages=[ - { - "role": "user", - "content": "Ivan is 28, lives in Moscow and his friends are Alex, John and Mary who are 25, 30 and 27 respectively", - } - ], - response_model=User, - ) - - for user in resp: - print(user) - #> name='Ivan' age=28 - #> name='Alex' age=25 - #> name='John' age=30 - #> name='Mary' age=27 - ``` - _Recommended for most use cases. Handles streaming and iteration for you._ - -=== "Using `create` with `Iterable[User]`" - ```python - import instructor - from pydantic import BaseModel - from typing import Iterable - - client = instructor.from_provider("openai/gpt-4.1-mini") - - - class User(BaseModel): - name: str - age: int - - - resp = client.create( - messages=[ - { - "role": "user", - "content": "Ivan is 28, lives in Moscow and his friends are Alex, John and Mary who are 25, 30 and 27 respectively", - } - ], - response_model=Iterable[User], - ) - - for user in resp: - print(user) - #> name='Ivan' age=28 - #> name='Alex' age=25 - #> name='John' age=30 - #> name='Mary' age=27 - ``` - _Use this if you need more manual control or compatibility with legacy code._ - ---- - - -We also support more complex extraction patterns such as Unions as you'll see below out of the box. - -???+ warning - - Unions don't work with Gemini because the AnyOf is not supported in the current response schema. - -## Synchronous Usage - -=== "Using `create`" - - ```python - import instructor - from typing import Iterable, Union, Literal - from pydantic import BaseModel - - - class Weather(BaseModel): - location: str - units: Literal["imperial", "metric"] - - - class GoogleSearch(BaseModel): - query: str - - - client = instructor.from_provider("openai/gpt-4.1-mini", mode=instructor.Mode.TOOLS) - - results = client.create( - messages=[ - {"role": "system", "content": "You must always use tools"}, - { - "role": "user", - "content": "What is the weather in toronto and dallas and who won the super bowl?", - }, - ], - response_model=Iterable[Union[Weather, GoogleSearch]], - stream=True, - ) - - for item in results: - print(item) - #> location='Toronto' units='metric' - #> location='Dallas' units='imperial' - #> query='Super Bowl winner' - ``` - -=== "Using `create_iterable` (recommended)" - - ```python - import instructor - from typing import Union, Literal - from pydantic import BaseModel - - - class Weather(BaseModel): - location: str - units: Literal["imperial", "metric"] - - - class GoogleSearch(BaseModel): - query: str - - - client = instructor.from_provider("openai/gpt-4.1-mini", mode=instructor.Mode.TOOLS) - - results = client.create_iterable( - messages=[ - {"role": "system", "content": "You must always use tools"}, - { - "role": "user", - "content": "What is the weather in toronto and dallas and who won the super bowl?", - }, - ], - response_model=Union[Weather, GoogleSearch], - ) - - for item in results: - print(item) - #> location='Toronto' units='metric' - #> location='Dallas' units='imperial' - #> query='Super Bowl winner' - ``` - ---- - -## See Also - -- [Streaming Lists](./lists.md) - Similar functionality with different API -- [Streaming Partial](./partial.md) - Stream partially completed objects -- [List Extraction Tutorial](../learning/patterns/list_extraction.md) - Step-by-step guide -- [Streaming Basics](../learning/streaming/basics.md) - Introduction to streaming - -## Asynchronous Usage - -=== "Using `create`" - - ```python - import instructor - from typing import Iterable, Union, Literal - from pydantic import BaseModel - import asyncio - - - class Weather(BaseModel): - location: str - units: Literal["imperial", "metric"] - - - class GoogleSearch(BaseModel): - query: str - - - aclient = instructor.from_provider( - "openai/gpt-4.1-mini", async_client=True, mode=instructor.Mode.TOOLS - ) - - - async def main(): - results = await aclient.create( - messages=[ - {"role": "system", "content": "You must always use tools"}, - { - "role": "user", - "content": "What is the weather in toronto and dallas and who won the super bowl?", - }, - ], - response_model=Iterable[Union[Weather, GoogleSearch]], - stream=True, - ) - async for item in results: - print(item) - #> location='Toronto' units='metric' - #> location='Dallas' units='imperial' - #> query='Super Bowl winner' - - - asyncio.run(main()) - ``` - -=== "Using `create_iterable` (recommended)" - - ```python - import asyncio - from typing import Literal, Union - - import instructor - from pydantic import BaseModel - - - class Weather(BaseModel): - location: str - units: Literal["imperial", "metric"] - - - class GoogleSearch(BaseModel): - query: str - - - aclient = instructor.from_provider( - "openai/gpt-4.1-mini", async_client=True, mode=instructor.Mode.TOOLS - ) - - - async def iter_results(): - async for item in aclient.create_iterable( - messages=[ - {"role": "system", "content": "You must always use tools"}, - { - "role": "user", - "content": "What is the weather in toronto and dallas and who won the super bowl?", - }, - ], - response_model=Union[Weather, GoogleSearch], - ): - yield item - - - async def main(): - async for item in iter_results(): - print(item) - #> location='Toronto' units='metric' - #> location='Dallas' units='imperial' - #> query='Super Bowl winner' - - - asyncio.run(main()) - ``` diff --git a/참고/instructor-main/docs/concepts/lists.md b/참고/instructor-main/docs/concepts/lists.md deleted file mode 100644 index 1dc3ede..0000000 --- a/참고/instructor-main/docs/concepts/lists.md +++ /dev/null @@ -1,179 +0,0 @@ ---- -title: Streaming Lists with Instructor - Extract Multiple Objects -description: Learn how to extract multiple structured objects from a single LLM call using streaming lists. Stream collections of Pydantic models as they're generated. ---- - -# Multi-task and Streaming - -A common use case of structured extraction is defining a single schema class and then making another schema to create a list to do multiple extraction - -```python -from typing import List -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -class Users(BaseModel): - users: List[User] - - -print(Users.model_json_schema()) -""" -{ - '$defs': { - 'User': { - 'properties': { - 'name': {'title': 'Name', 'type': 'string'}, - 'age': {'title': 'Age', 'type': 'integer'}, - }, - 'required': ['name', 'age'], - 'title': 'User', - 'type': 'object', - } - }, - 'properties': { - 'users': {'items': {'$ref': '#/$defs/User'}, 'title': 'Users', 'type': 'array'} - }, - 'required': ['users'], - 'title': 'Users', - 'type': 'object', -} -""" -``` - -Defining a task and creating a list of classes is a common enough pattern that we make this convenient by making use of `Iterable[T]`. This lets us dynamically create a new class that: - -1. Has dynamic docstrings and class name based on the task -2. Support streaming by collecting tokens until a task is received back out. - -## Extracting Tasks using Iterable - -By using `Iterable` you get a very convenient class with prompts and names automatically defined: - -```python -import instructor -from typing import Iterable -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -client = instructor.from_provider( - "openai/gpt-4.1-mini-1106", - mode=instructor.Mode.JSON, -) - -users = client.create( - temperature=0.1, - response_model=Iterable[User], - stream=False, - messages=[ - { - "role": "user", - "content": ( - "Consider this data: Jason is 10 and John is 30. " - "Correctly segment it into entities. " - "Make sure the JSON is correct." - ), - }, - ], -) -for user in users: - print(user) - #> name='Jason' age=10 - #> name='John' age=30 -``` - -## Streaming Tasks - -We can also generate tasks as the tokens are streamed in by defining an `Iterable[T]` type. - -Lets look at an example in action with the same class - -```python hl_lines="6 26" -import instructor -from typing import Iterable -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -client = instructor.from_provider( - "openai/gpt-4.1-mini", - mode=instructor.Mode.TOOLS, -) - -users = client.create( - temperature=0.1, - stream=True, - response_model=Iterable[User], - messages=[ - {"role": "system", "content": "You are a perfect entity extraction system"}, - {"role": "user", "content": "Extract `Jason is 10 and John is 10`"}, - ], - max_tokens=1000, -) - -for user in users: - print(user) - #> name='Jason' age=10 - #> name='John' age=10 -``` - -## Asynchronous Streaming - -I also just want to call out in this example that `instructor` also supports asynchronous streaming. This is useful when you want to stream a response model and process the results as they come in, but you'll need to use the `async for` syntax to iterate over the results. - -```python -import instructor -from typing import Iterable -from pydantic import BaseModel - - -class UserExtract(BaseModel): - name: str - age: int - - -async def print_iterable_results(): - client = instructor.from_provider( - "openai/gpt-4.1-mini", - async_client=True, - mode=instructor.Mode.TOOLS, - ) - - model = await client.create( - response_model=Iterable[UserExtract], - max_retries=2, - stream=True, - messages=[ - {"role": "user", "content": "Make two up people"}, - ], - ) - async for m in model: - print(m) - #> name='Alice' age=30 - #> name='Bob' age=25 - - -import asyncio - -asyncio.run(print_iterable_results()) -``` - -## See Also - -- [Streaming Partial](./partial.md) - Stream partially completed objects -- [Streaming Lists Tutorial](../learning/streaming/lists.md) - Step-by-step list streaming guide -- [Iterable Patterns](../learning/patterns/list_extraction.md) - List extraction patterns -- [Raw Response](./raw_response.md) - Access original LLM responses diff --git a/참고/instructor-main/docs/concepts/logging.md b/참고/instructor-main/docs/concepts/logging.md deleted file mode 100644 index 5a6767a..0000000 --- a/참고/instructor-main/docs/concepts/logging.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: Logging and Monitoring with Instructor - Debug Guide -description: Implement comprehensive logging for Instructor LLM calls. Track API usage, debug issues, and monitor performance with DEBUG level logging. ---- - -In order to see the requests made to OpenAI and the responses, you can set logging to DEBUG. This will show the requests and responses made to OpenAI. This can be useful for debugging and understanding the requests and responses made to OpenAI. I would love some contributions that make this a lot cleaner, but for now this is the fastest way to see the prompts. - -```python -import instructor -import logging - -from pydantic import BaseModel - - -# Set logging to DEBUG -logging.basicConfig(level=logging.DEBUG) - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class UserDetail(BaseModel): - name: str - age: int - - -user = client.create( - response_model=UserDetail, - messages=[ - {"role": "user", "content": "Extract Jason is 25 years old"}, - ], -) # type: ignore - -""" -... -DEBUG:instructor:Patching `client.chat.completions.create` with mode= -DEBUG:instructor:Instructor Request: mode.value='tool_call', response_model=, new_kwargs={'model': 'gpt-4.1-mini', 'messages': [{'role': 'user', 'content': 'Extract Jason is 25 years old'}], 'tools': [{'type': 'function', 'function': {'name': 'UserDetail', 'description': 'Correctly extracted `UserDetail` with all the required parameters with correct types', 'parameters': {'properties': {'name': {'title': 'Name', 'type': 'string'}, 'age': {'title': 'Age', 'type': 'integer'}}, 'required': ['age', 'name'], 'type': 'object'}}}], 'tool_choice': {'type': 'function', 'function': {'name': 'UserDetail'}}} -DEBUG:instructor:max_retries: 1 -... -DEBUG:instructor:Instructor Pre-Response: ChatCompletion(id='chatcmpl-8zBxMxsOqm5Sj6yeEI38PnU2r6ncC', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content=None, role='assistant', function_call=None, tool_calls=[ChatCompletionMessageToolCall(id='call_E1cftF5U0zEjzIbWt3q0ZLbN', function=Function(arguments='{"name":"Jason","age":25}', name='UserDetail'), type='function')]))], created=1709594660, model='gpt-4.1-mini-0125', object='chat.completion', system_fingerprint='fp_2b778c6b35', usage=CompletionUsage(completion_tokens=9, prompt_tokens=81, total_tokens=90)) -DEBUG:httpcore.connection:close.started -DEBUG:httpcore.connection:close.complete -""" -``` - -## Provider initialization logs - -`from_provider()` now emits structured logs at the `INFO` level when a provider -is initialized. Enable logging to see which provider and model are being used. - -```python -import logging -import instructor - -logging.basicConfig(level=logging.INFO) - -instructor.from_provider("openai/gpt-4.1-mini") -``` - -Example output: - -``` -INFO:instructor.auto_client:Initializing openai provider with model gpt-4.1-mini -INFO:instructor.auto_client:Client initialized -``` diff --git a/참고/instructor-main/docs/concepts/maybe.md b/참고/instructor-main/docs/concepts/maybe.md deleted file mode 100644 index 4676e73..0000000 --- a/참고/instructor-main/docs/concepts/maybe.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -title: Maybe Types and Optional Handling in Instructor -description: Handle optional and nullable data with Maybe types in Instructor. Learn to work with potentially missing fields and optional responses from LLMs. ---- - -# Handling Missing Data - -The `Maybe` pattern is a concept in functional programming used for error handling. Instead of raising exceptions or returning `None`, you can use a `Maybe` type to encapsulate both the result and potential errors. - -This pattern is particularly useful when making LLM calls, as providing language models with an escape hatch can effectively reduce hallucinations. - -## Defining the Model - -Using Pydantic, we'll first define the `UserDetail` and `MaybeUser` classes. - -```python -from pydantic import BaseModel, Field -from typing import Optional - - -class UserDetail(BaseModel): - age: int - name: str - role: Optional[str] = Field(default=None) - - -class MaybeUser(BaseModel): - result: Optional[UserDetail] = Field(default=None) - error: bool = Field(default=False) - message: Optional[str] = Field(default=None) - - def __bool__(self): - return self.result is not None -``` - -Notice that `MaybeUser` has a `result` field that is an optional `UserDetail` instance where the extracted data will be stored. The `error` field is a boolean that indicates whether an error occurred, and the `message` field is an optional string that contains the error message. - -## Defining the function - -Once we have the model defined, we can create a function that uses the `Maybe` pattern to extract the data. - -```python -import instructor -from pydantic import BaseModel, Field -from typing import Optional - -# This enables the `response_model` keyword -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class UserDetail(BaseModel): - age: int - name: str - role: Optional[str] = Field(default=None) - - -class MaybeUser(BaseModel): - result: Optional[UserDetail] = Field(default=None) - error: bool = Field(default=False) - message: Optional[str] = Field(default=None) - - def __bool__(self): - return self.result is not None - - -def extract(content: str) -> MaybeUser: - return client.create( - response_model=MaybeUser, - messages=[ - {"role": "user", "content": f"Extract `{content}`"}, - ], - ) - - -user1 = extract("Jason is a 25-year-old scientist") -print(user1.model_dump_json(indent=2)) -""" -{ - "result": { - "age": 25, - "name": "Jason", - "role": "scientist" - }, - "error": false, - "message": null -} -""" - -user2 = extract("Unknown user") -print(user2.model_dump_json(indent=2)) -""" -{ - "result": null, - "error": false, - "message": null -} -""" -``` - -As you can see, when the data is extracted successfully, the `result` field contains the `UserDetail` instance. When an error occurs, the `error` field is set to `True`, and the `message` field contains the error message. - -If you want to learn more about pattern matching, check out Pydantic's docs on [Structural Pattern Matching](https://docs.pydantic.dev/latest/concepts/models/#structural-pattern-matching) diff --git a/참고/instructor-main/docs/concepts/migration.md b/참고/instructor-main/docs/concepts/migration.md deleted file mode 100644 index 2e8b680..0000000 --- a/참고/instructor-main/docs/concepts/migration.md +++ /dev/null @@ -1,202 +0,0 @@ ---- -title: Migration Guide -description: Migrate from older Instructor patterns to the modern from_provider approach. ---- - -# Migration Guide - -This guide helps you migrate from older Instructor patterns to `from_provider`, the recommended approach for all providers. - -## Why Migrate? - -- **Simpler code**: Less boilerplate, easier to read -- **Consistent interface**: Same pattern works for all providers -- **Better type safety**: Improved IDE support -- **Future-proof**: Recommended pattern going forward - -## Quick Reference - -| Old Pattern | New Pattern | -|-------------|-------------| -| `instructor.patch(openai.OpenAI())` | `instructor.from_provider("openai/model")` | -| `instructor.apatch(openai.AsyncOpenAI())` | `instructor.from_provider("openai/model", async_client=True)` | -| `from_openai(client)` | `instructor.from_provider("openai/model")` | -| `from_anthropic(client)` | `instructor.from_provider("anthropic/model")` | -| `from_genai(client)` | `instructor.from_provider("google/model")` | -| `client.chat.completions.create(...)` | `client.create(...)` | -| `client.messages.create(...)` | `client.create(...)` | - -## Basic Migration - -**Before:** - -```python -import openai -import instructor -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -openai_client = openai.OpenAI() -client = instructor.patch(openai_client) - -user = client.chat.completions.create( - model="gpt-4o-mini", - response_model=User, - messages=[{"role": "user", "content": "Extract: John is 30"}], -) -``` - -**After:** - -```python -import instructor -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -client = instructor.from_provider("openai/gpt-4o-mini") - -user = client.create( - response_model=User, - messages=[{"role": "user", "content": "Extract: John is 30"}], -) -``` - -## Async Migration - -**Before:** - -```python -import openai -import instructor - -openai_client = openai.AsyncOpenAI() -client = instructor.apatch(openai_client) - -user = await client.chat.completions.create(...) -``` - -**After:** - -```python -import instructor - -client = instructor.from_provider("openai/gpt-4o-mini", async_client=True) - -user = await client.create(...) -``` - -## Provider-Specific Migrations - -### Anthropic - -```python -# Before (removed) -import anthropic -from instructor import from_anthropic - -client = from_anthropic(anthropic.Anthropic()) -user = client.messages.create(model="claude-3-5-sonnet", ...) - -# After -client = instructor.from_provider("anthropic/claude-3-5-sonnet") -user = client.create(...) -``` - -### Google/Gemini - -```python -# Before (removed) -import google.genai as genai -from instructor import from_genai - -client = from_genai(genai.Client(), model="gemini-pro") -user = client.generate_content(...) - -# After -client = instructor.from_provider("google/gemini-pro") -user = client.create(messages=[...]) -``` - -## Configuration Options - -Pass configuration directly to `from_provider`: - -```python -import instructor - -# Mode configuration -client = instructor.from_provider("openai/gpt-4o-mini", mode=instructor.Mode.JSON) - -# Custom API settings -client = instructor.from_provider( - "openai/gpt-4o-mini", - api_key="custom-key", - organization="org-id", - timeout=30.0, -) -``` - -## Multiple Providers - -**Before:** - -```python -import openai -import anthropic -import instructor -from instructor import from_anthropic - -openai_client = instructor.patch(openai.OpenAI()) -anthropic_client = from_anthropic(anthropic.Anthropic()) -``` - -**After:** - -```python -import instructor - -openai_client = instructor.from_provider("openai/gpt-4o-mini") -anthropic_client = instructor.from_provider("anthropic/claude-3-5-sonnet") -``` - -## Migration Checklist - -1. **Identify your current pattern**: `patch()`, `apatch()`, or `from_*()` functions -2. **Find your model name**: e.g., `gpt-4o-mini`, `claude-3-5-sonnet` -3. **Replace client creation**: Use `from_provider("provider/model")` -4. **Update method calls**: Change to `client.create(...)` -5. **Use standard message format**: `[{"role": "user", "content": "..."}]` -6. **Test your code** - -## Troubleshooting - -| Error | Cause | Solution | -|-------|-------|----------| -| `'Instructor' object has no attribute 'chat'` | Using old method call | Use `client.create()` instead of `client.chat.completions.create()` | -| Invalid model string | Wrong format | Use `"provider/model-name"` format | -| Message format error | Provider-specific format | Use standard `messages` list format | - -## Backward Compatibility - -Legacy helpers have been removed: - -- `instructor.patch()` → Use `from_provider` instead -- `instructor.apatch()` → Use `from_provider` with `async_client=True` -- `from_openai()`, `from_anthropic()`, etc. → Use `from_provider` - -Update all call sites before upgrading. - -## See Also - -- [from_provider Guide](./from_provider.md) - Complete guide to using from_provider -- [Patching](./patching.md) - How Instructor enhances clients diff --git a/참고/instructor-main/docs/concepts/mode-migration.md b/참고/instructor-main/docs/concepts/mode-migration.md deleted file mode 100644 index 4dce84c..0000000 --- a/참고/instructor-main/docs/concepts/mode-migration.md +++ /dev/null @@ -1,114 +0,0 @@ ---- -title: Mode Migration Guide -description: Migrate from provider-specific modes to the core modes in Instructor. ---- - -# Mode Migration Guide - -This guide helps you move from provider-specific modes to the core modes. -Core modes work across providers and are the recommended choice for new code. - -!!! note "V2 Preview" - - Provider-specific modes are deprecated in v2. They still work, emit warnings, and map to core modes. - -## Core Modes - -These are the core modes you should use: - -- `TOOLS`: Tool or function calling -- `JSON_SCHEMA`: Native schema support when a provider has it -- `MD_JSON`: JSON extracted from text or code blocks -- `PARALLEL_TOOLS`: Multiple tool calls in one response -- `RESPONSES_TOOLS`: OpenAI Responses API tools - -## Quick Mapping - -Use this table to replace legacy modes: - -| Legacy Mode | Core Mode | -|------------|-----------| -| `FUNCTIONS` | `TOOLS` | -| `TOOLS_STRICT` | `TOOLS` | -| `ANTHROPIC_TOOLS` | `TOOLS` | -| `ANTHROPIC_JSON` | `MD_JSON` | -| `COHERE_TOOLS` | `TOOLS` | -| `COHERE_JSON_SCHEMA` | `JSON_SCHEMA` | -| `XAI_TOOLS` | `TOOLS` | -| `XAI_JSON` | `MD_JSON` | -| `MISTRAL_TOOLS` | `TOOLS` | -| `MISTRAL_STRUCTURED_OUTPUTS` | `JSON_SCHEMA` | -| `FIREWORKS_TOOLS` | `TOOLS` | -| `FIREWORKS_JSON` | `MD_JSON` | -| `CEREBRAS_TOOLS` | `TOOLS` | -| `CEREBRAS_JSON` | `MD_JSON` | -| `WRITER_TOOLS` | `TOOLS` | -| `WRITER_JSON` | `MD_JSON` | -| `BEDROCK_TOOLS` | `TOOLS` | -| `BEDROCK_JSON` | `MD_JSON` | -| `PERPLEXITY_JSON` | `MD_JSON` | -| `VERTEXAI_TOOLS` | `TOOLS` | -| `VERTEXAI_JSON` | `MD_JSON` | -| `VERTEXAI_PARALLEL_TOOLS` | `PARALLEL_TOOLS` | - -## Example: Anthropic - -**Before:** - -```python -import instructor -from instructor import Mode - -client = instructor.from_provider( - "anthropic/claude-3-5-haiku-latest", - mode=Mode.ANTHROPIC_TOOLS, -) -``` - -**After:** - -```python -import instructor -from instructor import Mode - -client = instructor.from_provider( - "anthropic/claude-3-5-haiku-latest", - mode=Mode.TOOLS, -) -``` - -## Example: Bedrock - -**Before:** - -```python -import instructor -from instructor import Mode - -client = instructor.from_provider( - "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", - mode=Mode.BEDROCK_TOOLS, -) -``` - -**After:** - -```python -import instructor -from instructor import Mode - -client = instructor.from_provider( - "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", - mode=Mode.BEDROCK_TOOLS, -) -``` - -## Notes - -- Legacy modes still work but show a deprecation warning. -- Use core modes for new code and docs. -- Core tests are parameterized by provider and mode for consistent coverage. -- Streaming extraction is now handled by provider handlers instead of the DSL. -- Legacy `ResponseSchema.parse_*` helpers are deprecated. Use `process_response` or - `ResponseSchema.from_response` with core modes so the v2 registry handles parsing. -- See [Mode Comparison](../modes-comparison.md) for details. diff --git a/참고/instructor-main/docs/concepts/models.md b/참고/instructor-main/docs/concepts/models.md deleted file mode 100644 index 415b405..0000000 --- a/참고/instructor-main/docs/concepts/models.md +++ /dev/null @@ -1,209 +0,0 @@ ---- -title: Using Pydantic Models for Structured Outputs -description: Learn how to define LLM output schemas with Pydantic models. ---- - -# Response Model - -Define LLM output schemas using `pydantic.BaseModel`. For more details, see the [Pydantic documentation](https://docs.pydantic.dev/latest/concepts/models/). - -After defining a Pydantic model, use it as the `response_model` in your client `create` calls. The `response_model` parameter: - -- Defines the schema and prompts for the language model -- Validates the response from the API -- Returns a Pydantic model instance - -## Prompting - -Use docstrings and field annotations to define the prompt for generating responses. - -```python -from pydantic import BaseModel, Field -import instructor - - -class User(BaseModel): - """ - This is the prompt that will be used to generate the response. - Any instructions here will be passed to the language model. - """ - - name: str = Field(description="The name of the user.") - age: int = Field(description="The age of the user.") - - -client = instructor.from_provider("openai/gpt-4o-mini") - -user = client.create( - response_model=User, - messages=[{"role": "user", "content": "Extract: John is 30 years old"}], -) -``` - -Docstrings, types, and field annotations are used to generate the prompt. The `create` method uses this prompt to generate the response. - -## Optional Values - -Use `Optional` and `default` to make fields optional when sent to the language model. - -```python -from pydantic import BaseModel, Field -from typing import Optional -import instructor - - -class User(BaseModel): - name: str = Field(description="The name of the user.") - age: int = Field(description="The age of the user.") - email: Optional[str] = Field(description="The email of the user.", default=None) - - -client = instructor.from_provider("openai/gpt-4o-mini") - -user = client.create( - response_model=User, - messages=[{"role": "user", "content": "Extract: John is 30 years old"}], -) -``` - -Fields can also be omitted from the schema sent to the language model using Pydantic's `SkipJsonSchema` annotation. See [Fields](fields.md#omitting-fields-from-schema-sent-to-the-language-model) for details. - -## Dynamic Model Creation - -Create models at runtime using Pydantic's `create_model` function: - -```python -from pydantic import BaseModel, create_model - - -class FooModel(BaseModel): - foo: str - bar: int = 123 - - -BarModel = create_model( - 'BarModel', - apple=(str, 'russet'), - banana=(str, 'yellow'), - __base__=FooModel, -) -print(BarModel) -#> -print(BarModel.model_fields.keys()) -#> dict_keys(['foo', 'bar', 'apple', 'banana']) -``` - -??? notes "When would I use this?" - - Consider a situation where the model is dynamically defined, based on some configuration or database. For example, we could have a database table that stores the properties of a model for - some model name or id. We could then query the database for the properties of the model and use that to create the model. - - ```sql - SELECT property_name, property_type, description - FROM prompt - WHERE model_name = {model_name} - ``` - - We can then use this information to create the model. - - ```python - from pydantic import BaseModel, create_model, Field - from typing import List - - types = { - 'string': str, - 'integer': int, - 'boolean': bool, - 'number': float, - 'List[str]': List[str], - } - - # Mocked cursor.fetchall() - cursor = [ - ('name', 'string', 'The name of the user.'), - ('age', 'integer', 'The age of the user.'), - ('email', 'string', 'The email of the user.'), - ] - - BarModel = create_model( - 'User', - **{ - property_name: (types[property_type], Field(description=description)) - for property_name, property_type, description in cursor - }, - __base__=BaseModel, - ) - - print(BarModel.model_json_schema()) - """ - { - 'properties': { - 'name': { - 'description': 'The name of the user.', - 'title': 'Name', - 'type': 'string', - }, - 'age': { - 'description': 'The age of the user.', - 'title': 'Age', - 'type': 'integer', - }, - 'email': { - 'description': 'The email of the user.', - 'title': 'Email', - 'type': 'string', - }, - }, - 'required': ['name', 'age', 'email'], - 'title': 'User', - 'type': 'object', - } - """ - ``` - - This would be useful when different users have different descriptions for the same model. We can use the same model but have different prompts for each user. - -## Adding Behavior - -Add methods to Pydantic models like any Python class. This lets you add custom logic to your models. - -```python -from pydantic import BaseModel -from typing import Literal - -import instructor - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class SearchQuery(BaseModel): - query: str - query_type: Literal["web", "image", "video"] - - def execute(self): - print(f"Searching for {self.query} of type {self.query_type}") - #> Searching for cat of type image - return "Results for cat" - - -query = client.create( - model="gpt-4.1-mini", - messages=[{"role": "user", "content": "Search for a picture of a cat"}], - response_model=SearchQuery, -) - -results = query.execute() -print(results) -#> Results for cat -``` - -Now we can call `execute` on our model instance after extracting it from a language model. If you want to see more examples of this checkout our post on [RAG is more than embeddings](../blog/posts/rag-and-beyond.md) - -## See Also - -- [Response Models Tutorial](../learning/getting_started/response_models.md) - Step-by-step guide to creating response models -- [Simple Object Extraction](../learning/patterns/simple_object.md) - Basic extraction patterns -- [Nested Structures](../learning/patterns/nested_structure.md) - Complex hierarchical models -- [Optional Fields](../learning/patterns/optional_fields.md) - Working with optional data -- [Types](./types.md) - Working with different data types -- [Fields](./fields.md) - Advanced field configuration diff --git a/참고/instructor-main/docs/concepts/multimodal.md b/참고/instructor-main/docs/concepts/multimodal.md deleted file mode 100644 index ef4d0e3..0000000 --- a/참고/instructor-main/docs/concepts/multimodal.md +++ /dev/null @@ -1,546 +0,0 @@ ---- -title: Seamless Multimodal Interactions with Instructor -description: Learn how the Image, PDF and Audio class in Instructor enables seamless handling of multimodal content across different AI models. ---- - ---- -title: Multimodal Processing with Instructor - Vision and Audio -description: Process images, audio, and video with Instructor for multimodal structured outputs. Extract data from visual content using GPT-4 Vision and Gemini models. ---- - -# Multimodal - -> We've provided a few different sample files for you to use to test out these new features. All examples below use these files. -> -> - (Image) : An image of some blueberry plants [image.jpg](https://raw.githubusercontent.com/instructor-ai/instructor/main/tests/assets/image.jpg) -> - (Audio) : A Recording of the Original Gettysburg Address : [gettysburg.wav](https://raw.githubusercontent.com/instructor-ai/instructor/main/tests/assets/gettysburg.wav) -> - (PDF) : A sample PDF file which contains a fake invoice [invoice.pdf](https://raw.githubusercontent.com/instructor-ai/instructor/main/tests/assets/invoice.pdf) -> Instructor provides a unified, provider-agnostic interface for working with multimodal inputs like images and PDFs. - -Instructor provides a unified, provider-agnostic interface for working with multimodal inputs like images, PDFs, and audio files. - -With Instructor's multimodal objects, you can easily load media from URLs, Google Cloud Storage URLs, local files, or base64 strings using a consistent API that works across different AI providers (OpenAI, Anthropic, Mistral, etc.). - -Instructor handles all the provider-specific formatting requirements behind the scenes, ensuring your code remains clean and future-proof as provider APIs evolve. Let's see how to use the Image, Audio and PDF classes. - -## `Image` - -This class represents an image that can be loaded from a URL or file path. It provides a set of methods to create `Image` instances from different sources (Eg. URLs, paths and base64 strings). The following shows which methods are supported for the individual providers. - -| Method | OpenAI | Anthropic | Google GenAI | -| ----------------- | ------ | --------- | ------------ | -| `from_url()` | ✅ | ✅ | ✅ | -| `from_gs_url()` | ✅ | ✅ | ✅ | -| `from_path()` | ✅ | ✅ | ✅ | -| `from_base64()` | ✅ | ✅ | ✅ | -| `autodetect()` | ✅ | ✅ | ✅ | - -We also support Anthropic Prompt Caching for images with the `ImageWith - -### Usage - -By using the `Image` class, we can abstract away the differences between the different formats, allowing you to work with a unified interface. - -You can create an `Image` instance from a URL, Google Cloud Storage (GCS) URL, or file path using the `from_url`, `from_gs_url`, or `from_path` methods. The `Image` class will automatically convert the image to a base64-encoded string and include it in the API request. - -```python -import instructor -from instructor.processing.multimodal import Image -from pydantic import BaseModel - - -class ImageDescription(BaseModel): - description: str - items: list[str] - - -# Use our sample image provided above. -url = "https://raw.githubusercontent.com/instructor-ai/instructor/main/tests/assets/image.jpg" - -client = instructor.from_provider("openai/gpt-4.1-mini") - -response = client.create( - response_model=ImageDescription, - messages=[ - { - "role": "user", - "content": [ - "What is in this image?", - Image.from_url(url), - ], - } - ], -) - -print(response) -""" -description='Blueberry bushes with clusters of ripe and unripe blueberries. The berries are blue to purplish in color, and the leaves are green. The sky in the background is cloudy.' items=['blueberry bushes', 'ripe blueberries', 'unripe blueberries', 'green leaves', 'cloudy sky'] -""" -``` - -### Google Cloud Storage Support - -Instructor now supports loading images directly from Google Cloud Storage URLs. This is particularly useful when working with images stored in GCS buckets. - -```python -import instructor -from instructor.processing.multimodal import Image -from pydantic import BaseModel - - -class ImageDescription(BaseModel): - description: str - items: list[str] - - -# Load image from GCS URL (must be publicly accessible) -gs_url = "gs://my-bucket/path/to/image.jpg" - -client = instructor.from_provider("openai/gpt-4.1-mini") - -response = client.create( - response_model=ImageDescription, - messages=[ - { - "role": "user", - "content": [ - "What is in this image?", - Image.from_gs_url(gs_url), - ], - } - ], -) - -print(response) -""" -description='A sample image loaded from Google Cloud Storage.' items=['sample image'] -""" -``` - -> **Note**: GCS URLs must point to publicly accessible objects. The `from_gs_url` method converts `gs://` URLs to `https://storage.googleapis.com/` URLs for access. - -We also provide an `autodetect_images` keyword argument that allows you to provide URLs, GCS URLs, or file paths as normal strings when you set it to true. The system will automatically detect and handle different media types including images, audio, and PDFs. - -You can see an example below. - -```python -import instructor -from pydantic import BaseModel - - -class ImageDescription(BaseModel): - description: str - items: list[str] - - -# Download a sample image for demonstration -url = "https://raw.githubusercontent.com/instructor-ai/instructor/main/tests/assets/image.jpg" - -client = instructor.from_provider("openai/gpt-4.1-mini") - -response = client.create( - response_model=ImageDescription, - autodetect_images=True, # Set this to True - messages=[ - { - "role": "user", - "content": ["What is in this image?", url], - } - ], -) - -print(response) -""" -description='The image shows a close-up of a blueberry bush with ripe blueberries and green leaves. The background includes more blueberry bushes and a cloudy sky.' items=['Blueberry bush', 'Ripe blueberries', 'Green leaves', 'Cloudy sky'] -""" -``` - -If you'll like to support Anthropic prompt caching with images, we provide the `ImageWithCacheControl` Object to do so. Simply use the `from_image_params` method and you'll be able to leverage Anthropic's prompt caching. - -```python -import instructor -from instructor.processing.multimodal import ImageWithCacheControl -from pydantic import BaseModel - - -class ImageDescription(BaseModel): - description: str - items: list[str] - - -# Download a sample image for demonstration -url = "https://raw.githubusercontent.com/instructor-ai/instructor/main/tests/assets/image.jpg" - -client = instructor.from_provider("anthropic/claude-3-5-sonnet-20240620") - -response, completion = client.create_with_completion( - response_model=ImageDescription, - autodetect_images=True, # Set this to True - messages=[ - { - "role": "user", - "content": [ - "What is in this image?", - ImageWithCacheControl.from_image_params( - { - "source": url, - "cache_control": { - "type": "ephemeral", - }, - } - ), - ], - } - ], - max_tokens=1000, -) - -print(response) -""" -description='A bush with numerous clusters of blueberries surrounded by green leaves, under a cloudy sky.' items=['blueberries', 'green leaves', 'cloudy sky'] -""" - -print(completion.usage.cache_creation_input_tokens) -#> 1820 -``` - -By leveraging Instructor's multimodal capabilities, you can focus on building your application logic without worrying about the intricacies of each provider's image handling format. This not only saves development time but also makes your code more maintainable and adaptable to future changes in AI provider APIs. - -## `Audio` - -> Note : Only OpenAI and Gemini support audio files at the moment. For Gemini, we're passing in the raw bytes as bytes for this feature. If you'd like to use the `Files` API instead, we also support it, [read more at](../integrations/genai.md) to see how to do so. - -Similar to the Image class, we provide methods to create `Audio` instances. - -| Method | OpenAI | Google GenAI | -| --------------- | ------ | ------------ | -| `from_url()` | ✅ | ✅ | -| `from_gs_url()` | ✅ | ✅ | -| `from_path()` | ✅ | ✅ | -| `from_base64()` | ✅ | ✅ | -| `autodetect()` | ✅ | ✅ | - -The `Audio` class represents an audio file that can be loaded from a URL, Google Cloud Storage URL, or file path. It provides methods to create `Audio` instances using the `from_path`, `from_url`, `from_gs_url`, `from_base64`, and `autodetect` methods. - -The `Audio` class will automatically convert it to the right format and include it in the API request. - -```python -from pydantic import BaseModel -import instructor -from instructor.processing.multimodal import Audio - -# Initialize the client -client = instructor.from_provider("openai/gpt-4o-audio-preview") - - -# Define our response model -class AudioDescription(BaseModel): - summary: str - transcript: str - - -url = "https://raw.githubusercontent.com/instructor-ai/instructor/main/tests/assets/gettysburg.wav" - -# Make the API call with the audio file -resp = client.create( - response_model=AudioDescription, - modalities=["text"], - audio={"voice": "alloy", "format": "wav"}, - messages=[ - { - "role": "user", - "content": [ - "Extract the following information from the audio:", - Audio.from_url(url), - ], - }, - ], -) - -print(resp) -""" -summary='This excerpt is from a famous historical speech discussing the founding principles of equality and liberty, and the ongoing civil war testing the endurance of those principles.' transcript='Four score and seven years ago our fathers brought forth on this continent a new nation, conceived in Liberty, and dedicated to the proposition that all men are created equal. Now we are engaged in a great civil war, testing whether that nation, or any nation so conceived and so dedicated, can long endure.' -""" -``` - -### Google Cloud Storage Support - -You can also load audio files directly from Google Cloud Storage: - -```python -from pydantic import BaseModel -import instructor -from instructor.processing.multimodal import Audio - -# Initialize the client -client = instructor.from_provider("openai/gpt-4o-audio-preview") - - -# Define our response model -class AudioDescription(BaseModel): - summary: str - transcript: str - - -# Load audio from GCS URL (must be publicly accessible) -gs_url = "gs://my-bucket/path/to/audio.wav" - -# Make the API call with the GCS audio file -resp = client.create( - response_model=AudioDescription, - modalities=["text"], - audio={"voice": "alloy", "format": "wav"}, - messages=[ - { - "role": "user", - "content": [ - "Extract the following information from the audio:", - Audio.from_gs_url(gs_url), - ], - }, - ], -) - -print(resp) -""" -summary='A short historical speech about equality and liberty.' transcript='Four score and seven years ago our fathers brought forth...' -""" -``` - -## `PDF` - -The `PDF` class represents a PDF file that can be loaded from a URL or file path. - -It provides methods to create `PDF` instances and is currently supported for OpenAI, Mistral, GenAI, Anthropic, and Bedrock client integrations. - -| Method | OpenAI | Anthropic | Google GenAI | Mistral | Bedrock | -| ----------------- | ------ | --------- | ------------ | ------- | ------- | -| `from_url()` | ✅ | ✅ | ✅ | ✅ | ✅ | -| `from_gs_url()` | ✅ | ✅ | ✅ | ✅ | ✅ | -| `from_path()` | ✅ | ✅ | ✅ | ❎ | ✅ | -| `from_base64()` | ✅ | ✅ | ✅ | ❎ | ✅ | -| `autodetect()` | ✅ | ✅ | ✅ | ✅ | ✅ | - -For Gemini, we also provide two additional methods that make working with the google-genai files package easy which you can access in the `PDFWithGenaiFile` object. - -For Anthropic, you can enable caching with the `PDFWithCacheControl` object. Note that this has caching configured by default for easy usage. - -We provide examples of how to use all three object classes below. - -For Bedrock, you can convert a `PDF` into the Bedrock-native document format with `PDF.to_bedrock()` and include the result in the message content list. - -### Usage - -```python -import instructor -from pydantic import BaseModel -from instructor.processing.multimodal import PDF - -# Set up the client -url = "https://raw.githubusercontent.com/instructor-ai/instructor/main/tests/assets/invoice.pdf" -client = instructor.from_provider("openai/gpt-4.1-mini") - - -# Create a model for analyzing PDFs -class Invoice(BaseModel): - total: float - items: list[str] - - -# Load and analyze a PDF -response = client.create( - response_model=Invoice, - messages=[ - { - "role": "user", - "content": [ - "Analyze this document", - PDF.from_url(url), - ], - } - ], -) - -print(response) -""" -total=220.0 items=['English Tea - 2 units at $100 each', 'Tofu - 10 units at $2 each'] -""" -``` - -### Google Cloud Storage Support - -You can load PDF files directly from Google Cloud Storage URLs: - -```python -import instructor -from pydantic import BaseModel -from instructor.processing.multimodal import PDF - -# Set up the client -gs_url = "gs://my-bucket/path/to/document.pdf" -client = instructor.from_provider("openai/gpt-4.1-mini") - - -# Create a model for analyzing PDFs -class Invoice(BaseModel): - total: float - items: list[str] - - -# Load and analyze a PDF from GCS (must be publicly accessible) -response = client.create( - response_model=Invoice, - messages=[ - { - "role": "user", - "content": [ - "Analyze this document", - PDF.from_gs_url(gs_url), - ], - } - ], -) - -print(f"Total = {response.total:.0f}, items = {response.items}") -#> Total = 220, items = ['English Tea', 'Tofu'] -``` - -### Caching - -If you'd like to cache the PDF for Anthropic, we provide the `PDFWithCacheControl` class which has caching configured by default. - -```python -import instructor -from pydantic import BaseModel -from instructor.processing.multimodal import PDFWithCacheControl - -# Set up the client -url = "https://raw.githubusercontent.com/instructor-ai/instructor/main/tests/assets/invoice.pdf" -client = instructor.from_provider("anthropic/claude-3-5-sonnet-20240620") - - -# Create a model for analyzing PDFs -class Invoice(BaseModel): - total: float - items: list[str] - - -# Load and analyze a PDF -response, completion = client.create_with_completion( - response_model=Invoice, - messages=[ - { - "role": "user", - "content": [ - "Analyze this document", - PDFWithCacheControl.from_url(url), - ], - } - ], - max_tokens=1000, -) - -print(f"Total = {response.total:.0f}, items = {response.items}") -#> Total = 220, items = ['English Tea', 'Tofu'] - -print(completion.usage.cache_creation_input_tokens) -#> 2091 -``` - -### Using Files - -We also provide a convinient wrapper around the Files API - allowing you to use both uploaded files and to block the main thread while your file is uploading. - -In this example below, we download the sample PDF and then upload it using the `Files` api provided by the `google.genai` sdk. - -```python -import instructor -from pydantic import BaseModel -from instructor.processing.multimodal import PDFWithGenaiFile -import requests - -# Set up the client -url = "https://raw.githubusercontent.com/instructor-ai/instructor/main/tests/assets/invoice.pdf" -client = instructor.from_provider("google/gemini-2.5-flash") - - -# Create a model for analyzing PDFs -class Invoice(BaseModel): - total: float - items: list[str] - - -# Load and analyze a PDF -with requests.get(url) as download_response: - pdf_data = download_response.content - with open("./invoice.pdf", "wb") as f: - f.write(pdf_data) - -response = client.create( - response_model=Invoice, - messages=[ - { - "role": "user", - "content": [ - "Analyze this document", - PDFWithGenaiFile.from_new_genai_file( - file_path="./invoice.pdf", - retry_delay=10, - max_retries=20, - ), - ], - } - ], -) - -print(response) -#> total=220.0 items=['English Tea', 'Tofu'] -``` - -If you've already uploaded your file ahead of time, we also support it. Just provide us with the file name as seen below - -```python -import instructor -from pydantic import BaseModel -from instructor.processing.multimodal import PDFWithGenaiFile -import requests - -# Set up the client -url = "https://raw.githubusercontent.com/instructor-ai/instructor/main/tests/assets/invoice.pdf" -client = instructor.from_provider("google/gemini-2.5-flash") - - -# Create a model for analyzing PDFs -class Invoice(BaseModel): - total: float - items: list[str] - - -# Load and analyze a PDF -with requests.get(url) as download_response: - pdf_data = download_response.content - with open("./invoice.pdf", "wb") as f: - f.write(pdf_data) - -file = client.files.upload( - file="invoice.pdf", -) - -response = client.create( - response_model=Invoice, - messages=[ - { - "role": "user", - "content": [ - "Analyze this document", - PDFWithGenaiFile.from_existing_genai_file(file_name=file.name), - ], - } - ], -) - -print(response) -#> total=220.0 items=['English Tea', 'Tofu'] -``` - -This way you have more granular control over how the file is uploaded, potentially also processing multiple file uploads at once too. diff --git a/참고/instructor-main/docs/concepts/parallel.md b/참고/instructor-main/docs/concepts/parallel.md deleted file mode 100644 index f50d408..0000000 --- a/참고/instructor-main/docs/concepts/parallel.md +++ /dev/null @@ -1,156 +0,0 @@ ---- -title: Parallel Tools -description: Learn about parallel tools in OpenAI, Google, and Anthropic. ---- - -## See Also - -- [from_provider Guide](./from_provider.md#async-clients) - Async client setup -- [Batch Processing](../examples/batch_job_oai.md) - Process multiple requests efficiently -- [Iterable](./iterable.md) - Extract multiple objects -- [Lists](./lists.md) - Working with collections - -# Parallel Tools - -Parallel Tool Calling is a feature that allows you to call multiple functions in a single request. - -!!! warning "Experimental Feature" - - Parallel Tool Calling is supported by Google, OpenAI, and Anthropic. Make sure to use the equivalent parallel tool `mode` for your client. - -## Understanding Parallel Tool Calling - -Parallel Function Calling helps you to significantly reduce the latency of your application without having to build a parent schema as a wrapper around these tool calls. - -=== "OpenAI" - - ```python hl_lines="20 32" - from __future__ import annotations - - import instructor - - from typing import Iterable, Literal - from pydantic import BaseModel - - - class Weather(BaseModel): - location: str - units: Literal["imperial", "metric"] - - - class GoogleSearch(BaseModel): - query: str - - - client = instructor.from_provider( - "openai/gpt-4.1-mini", - mode=instructor.Mode.PARALLEL_TOOLS, - ) - function_calls = client.create( - messages=[ - {"role": "system", "content": "You must always use tools"}, - { - "role": "user", - "content": "What is the weather in toronto and dallas and who won the super bowl?", - }, - ], - response_model=Iterable[Weather | GoogleSearch], - ) - - for fc in function_calls: - print(fc) - #> location='Toronto' units='metric' - #> location='Dallas' units='metric' - #> query='who won the super bowl 2023' - ``` - -=== "Vertex AI" - - ```python - from typing import Iterable, Literal - - import instructor - from pydantic import BaseModel - - try: - import vertexai - import vertexai.generative_models as gm - from instructor import from_vertexai - except ImportError: - vertexai = None - gm = None - from_vertexai = None - - - class Weather(BaseModel): - location: str - units: Literal["imperial", "metric"] - - - class GoogleSearch(BaseModel): - query: str - - - if from_vertexai is not None and vertexai is not None and gm is not None: - vertexai.init(project="your-project-id", location="us-central1") - client = from_vertexai( - gm.GenerativeModel("gemini-2.5-flash"), - mode=instructor.Mode.PARALLEL_TOOLS, - ) - function_calls = client.create( - messages=[ - { - "role": "user", - "content": "What is the weather in toronto and dallas and who won the super bowl?", - }, - ], - response_model=Iterable[Weather | GoogleSearch], - ) - - for fc in function_calls: - print(fc) - #> location='Toronto' units='metric' - #> location='Dallas' units='imperial' - #> query='who won the super bowl' - ``` - -=== "Anthropic" - - ```python hl_lines="20 32" - import instructor - from typing import Iterable, Literal - from pydantic import BaseModel - - - class Weather(BaseModel): - location: str - units: Literal["imperial", "metric"] - - - class GoogleSearch(BaseModel): - query: str - - - client = instructor.from_provider( - "anthropic/claude-3-7-sonnet-latest", - mode=instructor.Mode.PARALLEL_TOOLS, - ) - function_calls = client.create( - messages=[ - {"role": "system", "content": "You must always use tools"}, - { - "role": "user", - "content": "What is the weather in toronto and dallas and who won the super bowl?", - }, - ], - response_model=Iterable[Weather | GoogleSearch], - ) - - for fc in function_calls: - print(fc) - #> location='Toronto' units='metric' - ``` - -We need to set the response model to `Iterable[Weather | GoogleSearch]` to indicate that the response will be a list of `Weather` and `GoogleSearch` objects. - -This is necessary because the response will be a list of objects, and we need to specify the types of the objects in the list. This returns an iterable which you can then iterate over diff --git a/참고/instructor-main/docs/concepts/partial.md b/참고/instructor-main/docs/concepts/partial.md deleted file mode 100644 index f24dc0f..0000000 --- a/참고/instructor-main/docs/concepts/partial.md +++ /dev/null @@ -1,210 +0,0 @@ ---- -title: Streaming Partial Responses with Instructor and OpenAI -description: Learn to utilize field-level streaming with Instructor and OpenAI for incremental responses in Python. ---- - -# Streaming Partial Responses - -!!! info "Literal" - - If the data structure you're using has literal values, you need to make sure to import the `PartialLiteralMixin` mixin. - - ```python - from typing import Literal - from pydantic import BaseModel - from instructor.dsl.partial import PartialLiteralMixin - - - class User(BaseModel, PartialLiteralMixin): - name: str - age: int - category: Literal["admin", "user", "guest"] - - - # The rest of your code below - ``` - - This is because `jiter` throws an error otherwise if it encounters a incomplete Literal value while it's being streamed in - -Field level streaming provides incremental snapshots of the current state of the response model that are immediately useable. This approach is particularly relevant in contexts like rendering UI components. - -Instructor supports this pattern by making use of `create_partial`. This lets us dynamically create a new class that treats all of the original model's fields as `Optional`. - -## Understanding Partial Responses - -Consider what happens whene we define a response model: - -```python -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int -``` - -If we streamed json out from OpenAI, we would only be able to parse when the object is completed returned! - -``` -{"name": "Jo -{"name": "John", "ag -{"name": "John", "age: -{"name": "John", "age": 25} # Completed -``` - -When specifying a `create_partial` and setting `stream=True`, the response from `instructor` becomes a `Generator[T]`. As the generator yields results, you can iterate over these incremental updates. The last value yielded by the generator represents the completed extraction! - -``` -{"name": "Jo => User(name="Jo", age=None) -{"name": "John", "ag => User(name="John", age=None) -{"name": "John", "age: => User(name="John", age=None) -{"name": "John", "age": 25} => User(name="John", age=25) -``` - -!!! warning "Limited Validator Support" - - Due to the streaming nature of the response model, we do not support validators since they would not be able to be applied to the streaming response. - -Let's look at an example of streaming an extraction of conference information, that would be used to stream in an react component. - -```python -import instructor -from pydantic import BaseModel -from typing import List -from rich.console import Console - -client = instructor.from_provider("openai/gpt-4.1-mini") - -text_block = """ -In our recent online meeting, participants from various backgrounds joined to discuss the upcoming tech conference. The names and contact details of the participants were as follows: - -- Name: John Doe, Email: johndoe@email.com, Twitter: @TechGuru44 -- Name: Jane Smith, Email: janesmith@email.com, Twitter: @DigitalDiva88 -- Name: Alex Johnson, Email: alexj@email.com, Twitter: @CodeMaster2023 - -During the meeting, we agreed on several key points. The conference will be held on March 15th, 2024, at the Grand Tech Arena located at 4521 Innovation Drive. Dr. Emily Johnson, a renowned AI researcher, will be our keynote speaker. - -The budget for the event is set at $50,000, covering venue costs, speaker fees, and promotional activities. Each participant is expected to contribute an article to the conference blog by February 20th. - -A follow-up meetingis scheduled for January 25th at 3 PM GMT to finalize the agenda and confirm the list of speakers. -""" - - -class User(BaseModel): - name: str - email: str - twitter: str - - -class MeetingInfo(BaseModel): - users: List[User] - date: str - location: str - budget: int - deadline: str - - -extraction_stream = client.create_partial( - response_model=MeetingInfo, - messages=[ - { - "role": "user", - "content": f"Get the information about the meeting and the users {text_block}", - }, - ], - stream=True, -) - - -console = Console() - -for extraction in extraction_stream: - obj = extraction.model_dump() - console.clear() - console.print(obj) - -print(extraction.model_dump_json(indent=2)) -""" -{ - "users": [ - { - "name": "John Doe", - "email": "johndoe@email.com", - "twitter": "@TechGuru44" - }, - { - "name": "Jane Smith", - "email": "janesmith@email.com", - "twitter": "@DigitalDiva88" - }, - { - "name": "Alex Johnson", - "email": "alexj@email.com", - "twitter": "@CodeMaster2023" - } - ], - "date": "March 15th, 2024", - "location": "Grand Tech Arena, 4521 Innovation Drive", - "budget": 50000, - "deadline": "February 20th" -} -""" -``` - -This will output the following: - -![Partial Streaming Gif](../img/partial.gif) - -## Asynchronous Streaming - -I also just want to call out in this example that `instructor` also supports asynchronous streaming. This is useful when you want to stream a response model and process the results as they come in, but you'll need to use the `async for` syntax to iterate over the results. - -```python -import instructor -from pydantic import BaseModel - -client = instructor.from_provider( - "openai/gpt-5-nano", - async_client=True, -) - - -class User(BaseModel): - name: str - age: int - - -async def print_partial_results(): - user = client.create_partial( - response_model=User, - max_retries=2, - stream=True, - messages=[ - {"role": "user", "content": "Jason is 12 years old"}, - ], - ) - async for m in user: - print(m) - #> name=None age=None - #> name=None age=None - #> name=None age=None - #> name='' age=None - #> name='Jason' age=None - #> name='Jason' age=None - #> name='Jason' age=None - #> name='Jason' age=None - #> name='Jason' age=12 - #> name='Jason' age=12 - - -import asyncio - -asyncio.run(print_partial_results()) -``` - -## See Also - -- [Streaming Lists](./lists.md) - Stream collections of completed objects -- [Streaming Basics](../learning/streaming/basics.md) - Introduction to streaming concepts -- [Iterable Streaming](./iterable.md) - Stream multiple objects -- [Raw Response](./raw_response.md) - Access original LLM responses diff --git a/참고/instructor-main/docs/concepts/patching.md b/참고/instructor-main/docs/concepts/patching.md deleted file mode 100644 index 4fd3dd8..0000000 --- a/참고/instructor-main/docs/concepts/patching.md +++ /dev/null @@ -1,169 +0,0 @@ ---- -title: How Instructor Patches LLM Clients -description: Learn how Instructor adds structured output capabilities to LLM clients through patching. ---- - -# Patching - -Patching adds structured output features to LLM client libraries. This page explains how it works. For most users, [`from_provider`](./from_provider.md) is simpler than manual patching. - -!!! tip "Recommended Approach" - Use [`from_provider`](./from_provider.md) instead of manual patching. It works the same way across all providers. See the [Migration Guide](./migration.md) if you're using older patching patterns. - -## What is Patching? - -Patching adds new features to LLM client objects without changing their original code. When Instructor patches a client, it adds: - -- New parameters: `response_model`, `max_retries`, and `context` to completion methods -- Validation: Checks responses against Pydantic models -- Retry logic: Retries when validation fails -- Compatibility: The patched client still works with all original methods - -## How Patching Works - -When Instructor patches a client, it: - -1. Wraps the completion method: Intercepts calls to `create()` or `chat.completions.create()` -2. Converts schemas: Changes Pydantic models into provider-specific formats (JSON schema, tool definitions, etc.) -3. Validates responses: Checks LLM outputs against your Pydantic model -4. Handles retries: Retries with validation feedback if needed -5. Returns typed objects: Converts validated JSON into Pydantic model instances - -## Patching Modes - -Different providers support different modes for structured extraction. Instructor automatically selects the best mode for each provider, but you can override it: - -### Tool Calling (TOOLS) - -Uses the provider's function/tool calling API. This is the default for OpenAI. - -Supported by: OpenAI, Anthropic (ANTHROPIC_TOOLS), Google (GENAI_TOOLS), Ollama (for supported models) - -### JSON Mode - -Instructs the model to return JSON directly. Works with most providers. - -Supported by: OpenAI, Anthropic, Google, Ollama, and most providers - -### Markdown JSON (MD_JSON) - -Asks for JSON wrapped in markdown. Only use for specific providers like Databricks. - -Supported by: Databricks, some vision models - -## Default Modes by Provider - -Each provider uses a recommended default mode: - -- **OpenAI**: `Mode.TOOLS` (function calling) -- **Anthropic**: `Mode.TOOLS` (tool use) -- **Google**: `Mode.TOOLS` (function calling) -- **Ollama**: `Mode.TOOLS` (if model supports it) or `Mode.JSON` -- **Others**: Provider-specific defaults - -When using `from_provider`, these defaults are applied automatically. You can override them with the `mode` parameter. - -## Manual Patching (Advanced) - -If you need to patch a client manually (not recommended for most users): - -```python -import openai -import instructor -from pydantic import BaseModel - - -class YourModel(BaseModel): - message: str - - -# Create the base client -openai_client = openai.OpenAI() - -# Patch it manually -client = instructor.patch(openai_client, mode=instructor.Mode.TOOLS) - -# Now use it -response = client.chat.completions.create( - response_model=YourModel, - messages=[{"role": "user", "content": "Say hello"}], -) -``` - -However, using `from_provider` is simpler and recommended: - -```python -import instructor -from pydantic import BaseModel - - -# Simpler approach -class YourModel(BaseModel): - message: str - - -client = instructor.from_provider("openai/gpt-4o-mini") -_response = client.create( - response_model=YourModel, - messages=[{"role": "user", "content": "Say hello"}], -) -``` - -## What Gets Patched? - -Instructor adds these features to patched clients: - -### New Parameters - -- `response_model`: A Pydantic model or type that defines the expected output structure -- `max_retries`: Number of retry attempts if validation fails (default: 0) -- `context`: Additional context for validation hooks - -### Enhanced Methods - -The patched client's `create()` method: -- Accepts `response_model` parameter -- Validates responses automatically -- Retries on validation failures -- Returns typed Pydantic objects instead of raw responses - -## Provider-Specific Considerations - -### OpenAI - -- Default mode: `TOOLS` (function calling) -- Supports streaming with structured outputs - -### Anthropic - -- Default mode: `ANTHROPIC_TOOLS` (tool use) -- Uses Claude's native tool calling API - -### Google Gemini - -- Default mode: `GENAI_TOOLS` (function calling) -- Requires `jsonref` package for tool calling -- Some limitations with strict validation and enums - -### Ollama (Local Models) - -- Default mode: `TOOLS` (if model supports it) or `JSON` -- Models like llama3.1, llama3.2, mistral-nemo support tools -- Older models fall back to JSON mode - -## When to Use Manual Patching - -Manual patching is rarely needed. Use it only if: - -1. You need fine-grained control over the patching process -2. You're working with a custom client implementation -3. You're debugging patching behavior - -For 99% of use cases, `from_provider` is the better choice. - -## Related Documentation - -- [from_provider Guide](./from_provider.md) - Recommended way to create patched clients -- [Migration Guide](./migration.md) - Migrating from manual patching to from_provider -- [Modes Comparison](../modes-comparison.md) - Detailed comparison of different modes -- [Integrations](../integrations/index.md) - Provider-specific documentation diff --git a/참고/instructor-main/docs/concepts/philosophy.md b/참고/instructor-main/docs/concepts/philosophy.md deleted file mode 100644 index 274b459..0000000 --- a/참고/instructor-main/docs/concepts/philosophy.md +++ /dev/null @@ -1,354 +0,0 @@ ---- -title: Philosophy -description: The principles behind Instructor - why simple beats complex every time. ---- - -# Philosophy - -Great tools make hard things easy without making easy things hard. That's Instructor. - -## Start with what developers know - -Most AI frameworks invent their own abstractions. We don't. - -```python -import instructor -from pydantic import BaseModel - - -# What you already know (Pydantic) -class User(BaseModel): - name: str - age: int - - -# What Instructor adds -client = instructor.from_provider("openai/gpt-4.1-mini") -_user = client.create( - response_model=User, - messages=[{"role": "user", "content": "Jane is 33"}], -) # That's it -``` - -If you know Pydantic, you know Instructor. No new concepts, no new syntax, no 200-page manual. - -## Your escape hatch is always there - -The worst frameworks are roach motels - easy to get in, impossible to get out. Instructor is different: - -```python -import instructor -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -# With Instructor -client = instructor.from_provider("openai/gpt-4.1-mini") -_result = client.create( - response_model=User, - messages=[{"role": "user", "content": "Jane is 33"}], -) - -# Want to go back to raw API? Just remove response_model: -client = instructor.from_provider("openai/gpt-4.1-mini") -_result = client.create(messages=[{"role": "user", "content": "Say hello"}]) - -# Or use the provider directly: -from openai import OpenAI - -_raw_client = OpenAI() # Back to vanilla -``` - -We patch, we don't wrap. Your code, your control. - -## Show, don't hide - -Bad frameworks hide complexity. Good tools help you understand it. - -```python -import instructor -from pydantic import BaseModel - - -class User(BaseModel): - name: str - age: int - - -# See exactly what Instructor sends -instructor.logfire.configure() # Full observability - -client = instructor.from_provider("openai/gpt-4.1-mini") -result = client.create( - response_model=User, - messages=[{"role": "user", "content": "Jane is 33"}], -) - -# Access raw responses -_raw_response = result._raw_response # See what the LLM actually returned -``` - -When something goes wrong (and it will), you can see exactly what happened. - -## Composition beats configuration - -No YAML files. No decorators. No magic. Just functions. - -```python -import instructor -from pydantic import BaseModel - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class User(BaseModel): - name: str - age: int - - -class Company(BaseModel): - name: str - industry: str - - -class Analysis(BaseModel): - user: User - company: Company - - -# Build complex systems with simple functions -def extract_user(text: str) -> User: - return client.create( - response_model=User, messages=[{"role": "user", "content": text}] - ) - - -def extract_company(text: str) -> Company: - return client.create( - response_model=Company, messages=[{"role": "user", "content": text}] - ) - - -def analyze_email(email: str) -> Analysis: - user = extract_user(email) - company = extract_company(email) - return Analysis(user=user, company=company) - - -# Compose however makes sense for YOUR application -_analysis = analyze_email("Please introduce Jane from Acme.") -``` - -## Start simple, grow naturally - -The best code is code that grows with your needs: - -```python -import instructor -from instructor import Partial -from pydantic import BaseModel, field_validator - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class User(BaseModel): - name: str - age: int - - -# Day 1: Just get it working -_user = client.create( - response_model=User, - messages=[{"role": "user", "content": "Jane is 33"}], -) - - -# Day 7: Add validation -class User(BaseModel): - name: str - age: int - - @field_validator("age") - def check_age(cls, value: int) -> int: - if value < 0 or value > 150: - raise ValueError("Invalid age") - return value - - -# Day 14: Add retries for production -_user = client.create( - response_model=User, - messages=[{"role": "user", "content": "Jane is 33"}], - max_retries=3, -) - - -# Day 30: Add streaming for better UX -def update_ui(_partial: Partial[User]) -> None: - pass - - -for partial in client.create( - response_model=Partial[User], - messages=[{"role": "user", "content": "Jane is 33"}], - stream=True, -): - update_ui(partial) -``` - -Each addition is one line. No refactoring. No migration guide. - -## What we intentionally DON'T do - -### No prompt engineering - -We don't write prompts for you. You know your domain better than we do. - -```python -# We DON'T do this: -# @instructor.prompt("Extract the user information carefully") -# def extract_user(text: str): -# ... - - -# You write your own prompts: -text = "Jane is 33" -_messages = [ - {"role": "system", "content": "You are a precise data extractor"}, - {"role": "user", "content": f"Extract user from: {text}"}, -] -``` - -### No new abstractions - -We don't invent concepts like "Agents", "Chains", or "Tools". Those are your domain concepts. - -```python -import instructor -from pydantic import BaseModel - -# We DON'T do this: -# class UserExtractionAgent(instructor.Agent): -# tools = [instructor.WebSearch(), instructor.Calculator()] - - -class User(BaseModel): - name: str - age: int - - -def search_web(query: str) -> str: - return f"Results for {query}" - - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -# You build what makes sense: -def extract_user_with_search(query: str) -> User: - # Your logic, your way - search_results = search_web(query) - return client.create( - response_model=User, messages=[{"role": "user", "content": search_results}] - ) - - -_user = extract_user_with_search("Find Jane") -``` - -### No framework lock-in - -Your code should work with or without us: - -```python -import instructor -from pydantic import BaseModel - - -# This is just a Pydantic model -class User(BaseModel): - name: str - age: int - - -# This is just a function -def process_user(user: User) -> dict: - return {"name": user.name.upper(), "adult": user.age >= 18} - - -client = instructor.from_provider("openai/gpt-4.1-mini") - -# Instructor just connects them to LLMs -user = client.create( - response_model=User, - messages=[{"role": "user", "content": "Jane is 33"}], -) - -_result = process_user(user) # Works with or without Instructor -``` - -## The result - -By following these principles, we get: - -- **Tiny API surface**: Learn it in minutes, not days -- **Zero vendor lock-in**: Switch providers or remove Instructor anytime -- **Debuggable**: When things break, you can see why -- **Composable**: Build complex systems from simple parts -- **Pythonic**: If it feels natural in Python, it feels natural in Instructor - -## In practice - -Here's what building with Instructor actually looks like: - -```python -from enum import Enum -from typing import List - -import instructor -from pydantic import BaseModel - - -# Your domain models (not ours) -class Priority(str, Enum): - HIGH = "high" - MEDIUM = "medium" - LOW = "low" - - -class Ticket(BaseModel): - title: str - description: str - priority: Priority - estimated_hours: float - - -# Your business logic (not ours) -def prioritize_tickets(tickets: List[Ticket]) -> List[Ticket]: - return sorted(tickets, key=lambda t: (t.priority.value, -t.estimated_hours)) - - -# Connect to LLM (one line) -client = instructor.from_provider("openai/gpt-4.1-mini") - -# Extract structured data (simple function call) -tickets = client.create( - response_model=List[Ticket], - messages=[{"role": "user", "content": "Parse these support tickets: ..."}], -) - -# Use your business logic -_prioritized = prioritize_tickets(tickets) -``` - -No framework. No abstractions. Just Python. - -## The philosophy in one sentence - -**Make structured LLM outputs as easy as defining a Pydantic model.** - -Everything else follows from that. diff --git a/참고/instructor-main/docs/concepts/prompt_caching.md b/참고/instructor-main/docs/concepts/prompt_caching.md deleted file mode 100644 index 2db9a13..0000000 --- a/참고/instructor-main/docs/concepts/prompt_caching.md +++ /dev/null @@ -1,272 +0,0 @@ ---- -title: Understanding Prompt Caching for API Efficiency -description: Explore how prompt caching optimizes performance for API calls in OpenAI and Anthropic, enhancing efficiency and reducing costs. ---- - -## See Also - -- [Caching](./caching.md) - General caching concepts -- [Cost Optimization](../examples/batch_job_oai.md) - Reduce API costs -- [Performance Optimization](../examples/sqlmodel.md#performance-optimization) - Performance best practices -- [Anthropic Integration](../integrations/anthropic.md) - Anthropic prompt caching support - -# Prompt Caching - -Prompt Caching is a feature that allows you to cache portions of your prompt, optimizing performance for multiple API calls with shared context. This helps to reduce cost and improve response times. - -## Prompt Caching in OpenAI - -OpenAI implements a prompt caching mechanism to optimize performance for API requests with similar prompts. - -> Prompt Caching works automatically on all your API requests (no code changes required) and has no additional fees associated with it. - -This optimization is especially useful for applications making multiple API calls with shared context, minimizing redundant processing and improving overall performance. - -Prompt Caching is enabled for the following models: - -- gpt-4o -- gpt-4.1-mini -- o1-preview -- o1-mini - -Caching is based on prefix matching, so if you're using a system prompt that contains a common set of instructions, you're likely to see a cache hit as long as you move all variable parts of the prompt to the end of the message when possible. - -## Prompt Caching in Anthropic - -Prompt Caching is now generally avaliable for Anthropic. This enables you to cache specific prompt portions, reuse cached content in subsequent calls, and reduce processed data per request. - -??? note "Source Text" - - In the following example, we'll be using a short excerpt from the novel "Pride and Prejudice" by Jane Austen. This text serves as an example of a substantial context that might typically lead to slow response times and high costs when working with language models. You can download it manually [here](https://www.gutenberg.org/cache/epub/1342/pg1342.txt) - - ``` - _Walt Whitman has somewhere a fine and just distinction between "loving - by allowance" and "loving with personal love." This distinction applies - to books as well as to men and women; and in the case of the not very - numerous authors who are the objects of the personal affection, it - brings a curious consequence with it. There is much more difference as - to their best work than in the case of those others who are loved "by - allowance" by convention, and because it is felt to be the right and - proper thing to love them. And in the sect--fairly large and yet - unusually choice--of Austenians or Janites, there would probably be - found partisans of the claim to primacy of almost every one of the - novels. To some the delightful freshness and humour of_ Northanger - Abbey, _its completeness, finish, and_ entrain, _obscure the undoubted - critical facts that its scale is small, and its scheme, after all, that - of burlesque or parody, a kind in which the first rank is reached with - difficulty._ Persuasion, _relatively faint in tone, and not enthralling - in interest, has devotees who exalt above all the others its exquisite - delicacy and keeping. The catastrophe of_ Mansfield Park _is admittedly - theatrical, the hero and heroine are insipid, and the author has almost - wickedly destroyed all romantic interest by expressly admitting that - Edmund only took Fanny because Mary shocked him, and that Fanny might - very likely have taken Crawford if he had been a little more assiduous; - yet the matchless rehearsal-scenes and the characters of Mrs. Norris and - others have secured, I believe, a considerable party for it._ Sense and - Sensibility _has perhaps the fewest out-and-out admirers; but it does - not want them._ - _I suppose, however, that the majority of at least competent votes - would, all things considered, be divided between_ Emma _and the present - book; and perhaps the vulgar verdict (if indeed a fondness for Miss - Austen be not of itself a patent of exemption from any possible charge - of vulgarity) would go for_ Emma. _It is the larger, the more varied, the - more popular; the author had by the time of its composition seen rather - more of the world, and had improved her general, though not her most - peculiar and characteristic dialogue; such figures as Miss Bates, as the - Eltons, cannot but unite the suffrages of everybody. On the other hand, - I, for my part, declare for_ Pride and Prejudice _unhesitatingly. It - seems to me the most perfect, the most characteristic, the most - eminently quintessential of its author's works; and for this contention - in such narrow space as is permitted to me, I propose here to show - cause._ - _In the first place, the book (it may be barely necessary to remind the - reader) was in its first shape written very early, somewhere about 1796, - when Miss Austen was barely twenty-one; though it was revised and - finished at Chawton some fifteen years later, and was not published till - 1813, only four years before her death. I do not know whether, in this - combination of the fresh and vigorous projection of youth, and the - critical revision of middle life, there may be traced the distinct - superiority in point of construction, which, as it seems to me, it - possesses over all the others. The plot, though not elaborate, is almost - regular enough for Fielding; hardly a character, hardly an incident - could be retrenched without loss to the story. The elopement of Lydia - and Wickham is not, like that of Crawford and Mrs. Rushworth, a_ coup de - théâtre; _it connects itself in the strictest way with the course of the - story earlier, and brings about the denouement with complete propriety. - All the minor passages--the loves of Jane and Bingley, the advent of Mr. - Collins, the visit to Hunsford, the Derbyshire tour--fit in after the - same unostentatious, but masterly fashion. There is no attempt at the - hide-and-seek, in-and-out business, which in the transactions between - Frank Churchill and Jane Fairfax contributes no doubt a good deal to the - intrigue of_ Emma, _but contributes it in a fashion which I do not think - the best feature of that otherwise admirable book. Although Miss Austen - always liked something of the misunderstanding kind, which afforded her - opportunities for the display of the peculiar and incomparable talent to - be noticed presently, she has been satisfied here with the perfectly - natural occasions provided by the false account of Darcy's conduct given - by Wickham, and by the awkwardness (arising with equal naturalness) from - the gradual transformation of Elizabeth's own feelings from positive - aversion to actual love. I do not know whether the all-grasping hand of - the playwright has ever been laid upon_ Pride and Prejudice; _and I dare - say that, if it were, the situations would prove not startling or - garish enough for the footlights, the character-scheme too subtle and - delicate for pit and gallery. But if the attempt were made, it would - certainly not be hampered by any of those loosenesses of construction, - which, sometimes disguised by the conveniences of which the novelist can - avail himself, appear at once on the stage._ - _I think, however, though the thought will doubtless seem heretical to - more than one school of critics, that construction is not the highest - merit, the choicest gift, of the novelist. It sets off his other gifts - and graces most advantageously to the critical eye; and the want of it - will sometimes mar those graces--appreciably, though not quite - consciously--to eyes by no means ultra-critical. But a very badly-built - novel which excelled in pathetic or humorous character, or which - displayed consummate command of dialogue--perhaps the rarest of all - faculties--would be an infinitely better thing than a faultless plot - acted and told by puppets with pebbles in their mouths. And despite the - ability which Miss Austen has shown in working out the story, I for one - should put_ Pride and Prejudice _far lower if it did not contain what - seem to me the very masterpieces of Miss Austen's humour and of her - faculty of character-creation--masterpieces who may indeed admit John - Thorpe, the Eltons, Mrs. Norris, and one or two others to their company, - but who, in one instance certainly, and perhaps in others, are still - superior to them._ - _The characteristics of Miss Austen's humour are so subtle and delicate - that they are, perhaps, at all times easier to apprehend than to - express, and at any particular time likely to be differently - apprehended by different persons. To me this humour seems to possess a - greater affinity, on the whole, to that of Addison than to any other of - the numerous species of this great British genus. The differences of - scheme, of time, of subject, of literary convention, are, of course, - obvious enough; the difference of sex does not, perhaps, count for much, - for there was a distinctly feminine element in "Mr. Spectator," and in - Jane Austen's genius there was, though nothing mannish, much that was - masculine. But the likeness of quality consists in a great number of - common subdivisions of quality--demureness, extreme minuteness of touch, - avoidance of loud tones and glaring effects. Also there is in both a - certain not inhuman or unamiable cruelty. It is the custom with those - who judge grossly to contrast the good nature of Addison with the - savagery of Swift, the mildness of Miss Austen with the boisterousness - of Fielding and Smollett, even with the ferocious practical jokes that - her immediate predecessor, Miss Burney, allowed without very much - protest. Yet, both in Mr. Addison and in Miss Austen there is, though a - restrained and well-mannered, an insatiable and ruthless delight in - roasting and cutting up a fool. A man in the early eighteenth century, - of course, could push this taste further than a lady in the early - nineteenth; and no doubt Miss Austen's principles, as well as her heart, - would have shrunk from such things as the letter from the unfortunate - husband in the_ Spectator, _who describes, with all the gusto and all the - innocence in the world, how his wife and his friend induce him to play - at blind-man's-buff. But another_ Spectator _letter--that of the damsel - of fourteen who wishes to marry Mr. Shapely, and assures her selected - Mentor that "he admires your_ Spectators _mightily"--might have been - written by a rather more ladylike and intelligent Lydia Bennet in the - days of Lydia's great-grandmother; while, on the other hand, some (I - think unreasonably) have found "cynicism" in touches of Miss Austen's - own, such as her satire of Mrs. Musgrove's self-deceiving regrets over - her son. But this word "cynical" is one of the most misused in the - English language, especially when, by a glaring and gratuitous - falsification of its original sense, it is applied, not to rough and - snarling invective, but to gentle and oblique satire. If cynicism means - the perception of "the other side," the sense of "the accepted hells - beneath," the consciousness that motives are nearly always mixed, and - that to seem is not identical with to be--if this be cynicism, then - every man and woman who is not a fool, who does not care to live in a - fool's paradise, who has knowledge of nature and the world and life, is - a cynic. And in that sense Miss Austen certainly was one. She may even - have been one in the further sense that, like her own Mr. Bennet, she - took an epicurean delight in dissecting, in displaying, in setting at - work her fools and her mean persons. I think she did take this delight, - and I do not think at all the worse of her for it as a woman, while she - was immensely the better for it as an artist. - ``` - -```python -import instructor -from pydantic import BaseModel - - -class Character(BaseModel): - name: str - description: str - - -# Note: For testing this example locally, create a book.txt file with content like: -# Sample book.txt content: -# "Pride and Prejudice by Jane Austen -# -# It is a truth universally acknowledged, that a single man in possession of a good fortune, must be in want of a wife. -# However little known the feelings or views of such a man may be on his first entering a neighbourhood, this truth is -# so well fixed in the minds of the surrounding families, that he is considered the rightful property of some one or -# other of their daughters..." -book = """ -Pride and Prejudice by Jane Austen - -It is a truth universally acknowledged, that a single man in possession of a good fortune, must be in want of a wife. -However little known the feelings or views of such a man may be on his first entering a neighbourhood, this truth is -so well fixed in the minds of the surrounding families, that he is considered the rightful property of some one or -other of their daughters... -""" - -# Uncomment to read from an actual file instead of using the sample text above -# with open("./book.txt") as f: -# book = f.read() - -client = instructor.from_provider("anthropic/claude-3-5-sonnet-20240620") - -resp, completion = client.create_with_completion( - model="claude-3-5-sonnet-20240620", - messages=[ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "" + book + "", - "cache_control": {"type": "ephemeral"}, # (1)! - }, - { - "type": "text", - "text": "Extract a character from the text given above", - }, - ], - }, - ], - response_model=Character, - max_tokens=1000, - ) - -print(completion) -# Message( -# id='msg_01QcqjktYc1PXL8nk7y5hkMV', -# content=[ -# ToolUseBlock( -# id='toolu_019wABRzQxtSbXeuuRwvJo15', -# input={ -# 'name': 'Jane Austen', -# 'description': 'A renowned English novelist of the early 19th century, known for her wit, humor, and keen observations of human nature. She is the author of -# several classic novels including "Pride and Prejudice," "Emma," "Sense and Sensibility," and "Mansfield Park." Austen\'s writing is characterized by its subtlety, delicate touch, -# and ability to create memorable characters. Her work often involves social commentary and explores themes of love, marriage, and societal expectations in Regency-era England.' -# }, -# name='Character', -# type='tool_use' -# ) -# ], -# model='claude-3-5-sonnet-20240620', -# role='assistant', -# stop_reason='tool_use', -# stop_sequence=None, -# type='message', -# usage=Usage(cache_creation_input_tokens=2777, cache_read_input_tokens=0, input_tokens=30, output_tokens=161) -# ) -``` - -1. Anthropic requires that you explicitly pass in the `cache_control` parameter to indicate that you want to cache the content. - -!!! Warning "Caching Considerations" - - **Minimum cache size**: For Claude Haiku, your cached content needs to be a minimum of 2048 tokens. For Claude Sonnet, the minimum is 1024 tokens. - -**Benefits**: The cost of reading from the cache is 10x lower than if we were to process the same message again and enables us to execute our queries significantly faster. - -We've written a more detailed blog on how to use the `create_with_completion` method [here](../blog/posts/anthropic-prompt-caching.md) to validate you're getting a cache hit with instructor. diff --git a/참고/instructor-main/docs/concepts/prompting.md b/참고/instructor-main/docs/concepts/prompting.md deleted file mode 100644 index dff4df7..0000000 --- a/참고/instructor-main/docs/concepts/prompting.md +++ /dev/null @@ -1,319 +0,0 @@ ---- -title: Prompt Engineering Best Practices -description: Learn prompt engineering tips for using Pydantic and Instructor effectively. ---- - -# General Tips for Prompt Engineering - -When using Instructor and Pydantic, make your models self-descriptive, modular, and flexible while keeping data integrity. - -- Modularity: Design self-contained components for reuse -- Self-description: Use Pydantic's `Field` for clear field descriptions -- Optionality: Use Python's `Optional` type for nullable fields and set defaults -- Standardization: Use enumerations for fields with fixed values; include a fallback option -- Dynamic data: Use key-value pairs for arbitrary properties and limit list lengths -- Entity relationships: Define explicit identifiers and relationship fields -- Contextual logic: Optionally add a "chain of thought" field in reusable components for extra context - -## Modular Chain of Thought {#chain-of-thought} - -Use chain of thought to improve data quality. You can add it to specific components rather than making it global. - -```python hl_lines="4 5" -from pydantic import BaseModel, Field - - -class Role(BaseModel): - chain_of_thought: str = Field( - ..., description="Think step by step to determine the correct title" - ) - title: str - - -class UserDetail(BaseModel): - age: int - name: str - role: Role -``` - -## Utilize Optional Attributes - -Use Python's Optional type and set a default value to prevent undesired defaults like empty strings. - -```python hl_lines="6" -from typing import Optional -from pydantic import BaseModel, Field - - -class UserDetail(BaseModel): - age: int - name: str - role: Optional[str] = Field(default=None) -``` - -## Handling Errors Within Function Calls - -Create a wrapper class to hold either the result of an operation or an error message. This lets you stay within a function call even if an error occurs, improving error handling without breaking the code flow. - -```python -from pydantic import BaseModel, Field -from typing import Optional - - -class UserDetail(BaseModel): - age: int - name: str - role: Optional[str] = Field(default=None) - - -class MaybeUser(BaseModel): - result: Optional[UserDetail] = Field(default=None) - error: bool = Field(default=False) - message: Optional[str] - - def __bool__(self): - return self.result is not None -``` - -With the `MaybeUser` class, you can either receive a `UserDetail` object in result or get an error message in message. - -### Simplification with the Maybe Pattern - -Simplify this using Instructor to create the `Maybe` pattern dynamically from any `BaseModel`. - -```python -import instructor -from pydantic import BaseModel - - -class UserDetail(BaseModel): - age: int - name: str - - -MaybeUser = instructor.Maybe(UserDetail) -``` - -This lets you quickly create a Maybe type for any class. - -## Tips for Enumerations - -Use Enums for standardized fields to prevent data misalignment. Always include an "Other" option as a fallback so the model can signal uncertainty. - -```python hl_lines="7 12" -from enum import Enum, auto -from pydantic import BaseModel, Field - - -class Role(Enum): - PRINCIPAL = auto() - TEACHER = auto() - STUDENT = auto() - OTHER = auto() - - -class UserDetail(BaseModel): - age: int - name: str - role: Role = Field( - description="Correctly assign one of the predefined roles to the user." - ) -``` - -## Literals {#literals} - -If you're having a hard time with `Enum` an alternative is to use `Literal` - -```python hl_lines="4" -from typing import Literal -from pydantic import BaseModel - - -class UserDetail(BaseModel): - age: int - name: str - role: Literal["PRINCIPAL", "TEACHER", "STUDENT", "OTHER"] -``` - -If you'd like to improve performance more you can reiterate the requirements in the field descriptions or in the docstrings. - -## Reiterate Long Instructions - -For complex attributes, repeat the instructions in the field's description. - -```python hl_lines="5 11" -from pydantic import BaseModel, Field - - -class Role(BaseModel): - """ - Extract the role based on the following rules ... - """ - - instructions: str = Field( - ..., - description="Restate the instructions and rules to correctly determine the title.", - ) - title: str - - -class UserDetail(BaseModel): - age: int - name: str - role: Role -``` - -## Handle Arbitrary Properties - -When you need to extract undefined attributes, use a list of key-value pairs. - -```python hl_lines="10" -from typing import List -from pydantic import BaseModel, Field - - -class Property(BaseModel): - key: str - value: str - - -class UserDetail(BaseModel): - age: int - name: str - properties: List[Property] = Field( - ..., description="Extract any other properties that might be relevant." - ) -``` - -## Limiting the Length of Lists - -When dealing with lists of attributes, especially arbitrary properties, manage the length. Use prompting and enumeration to limit the list length and keep a manageable set of properties. - -```python hl_lines="2 9" -from typing import List -from pydantic import BaseModel, Field - - -class Property(BaseModel): - index: str = Field(..., description="Monotonically increasing ID") - key: str - value: str - - -class UserDetail(BaseModel): - age: int - name: str - properties: List[Property] = Field( - ..., - description="Numbered list of arbitrary extracted properties, should be less than 6", - ) -``` - -### Using Tuples for Simple Types - -For simple types, tuples can be a more compact alternative to custom classes, especially when the properties don't require additional descriptions. - -```python hl_lines="4" -from typing import List, Tuple -from pydantic import BaseModel, Field - - -class UserDetail(BaseModel): - age: int - name: str - properties: List[Tuple[int, str]] = Field( - ..., - description="Numbered list of arbitrary extracted properties, should be less than 6", - ) -``` - -## Advanced Arbitrary Properties - -For multiple users, use consistent key names when extracting properties. - -```python -from typing import List -from pydantic import BaseModel - - -class UserDetail(BaseModel): - id: int - age: int - name: str - - -class UserDetails(BaseModel): - """ - Extract information for multiple users. - Use consistent key names for properties across users. - """ - - users: List[UserDetail] -``` - -This refined guide should offer a cleaner and more organized approach to structure engineering in Python. - -## Defining Relationships Between Entities - -When relationships exist between entities, define them explicitly in the model. The following example shows how to define relationships between users by adding an id and a friends field: - -```python hl_lines="2 5 8" -from typing import List -from pydantic import BaseModel, Field - - -class UserDetail(BaseModel): - id: int = Field(..., description="Unique identifier for each user.") - age: int - name: str - friends: List[int] = Field( - ..., - description="Correct and complete list of friend IDs, representing relationships between users.", - ) - - -class UserRelationships(BaseModel): - users: List[UserDetail] = Field( - ..., - description="Collection of users, correctly capturing the relationships among them.", - ) -``` - -## Reusing Components with Different Contexts - -You can reuse the same component for different contexts within a model. In this example, the TimeRange component is used for both work_time and leisure_time. - -```python hl_lines="9 10" -from pydantic import BaseModel, Field - - -class TimeRange(BaseModel): - start_time: int = Field(..., description="The start time in hours.") - end_time: int = Field(..., description="The end time in hours.") - - -class UserDetail(BaseModel): - id: int = Field(..., description="Unique identifier for each user.") - age: int - name: str - work_time: TimeRange = Field( - ..., description="Time range during which the user is working." - ) - leisure_time: TimeRange = Field( - ..., description="Time range reserved for leisure activities." - ) -``` - -Sometimes, a component like TimeRange may need context or additional logic to work well. Adding a "chain of thought" field within the component can help understand or optimize the time range allocations. - -```python hl_lines="2" -from pydantic import BaseModel, Field - - -class TimeRange(BaseModel): - chain_of_thought: str = Field( - ..., description="Step by step reasoning to get the correct time range" - ) - start_time: int = Field(..., description="The start time in hours.") - end_time: int = Field(..., description="The end time in hours.") -``` diff --git a/참고/instructor-main/docs/concepts/raw_response.md b/참고/instructor-main/docs/concepts/raw_response.md deleted file mode 100644 index 76164f7..0000000 --- a/참고/instructor-main/docs/concepts/raw_response.md +++ /dev/null @@ -1,160 +0,0 @@ ---- -title: Creating a Model with OpenAI Completions -description: Learn how to create a custom model using OpenAI's API to extract user data efficiently with Python. ---- - - -# Creating a model with completions - -In instructor>1.0.0 we have a custom client, if you wish to use the raw response you can do the following - -```python -import instructor - -from pydantic import BaseModel - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class UserExtract(BaseModel): - name: str - age: int - - -user, completion = client.create_with_completion( - response_model=UserExtract, - messages=[ - {"role": "user", "content": "Extract jason is 25 years old"}, - ], -) - -print(user) -#> name='jason' age=25 - -print(completion) -""" -ChatCompletion( - id='chatcmpl-D1KqvmcGn5zeYfqRdquwERAH0wIVB', - choices=[ - Choice( - finish_reason='stop', - index=0, - logprobs=None, - message=ChatCompletionMessage( - content=None, - refusal=None, - role='assistant', - annotations=[], - audio=None, - function_call=None, - tool_calls=[ - ChatCompletionMessageFunctionToolCall( - id='call_8VastKJ2gYWNrYEQmBXGWnRv', - function=Function( - arguments='{"name":"jason","age":25}', name='UserExtract' - ), - type='function', - ) - ], - ), - ) - ], - created=1769210857, - model='gpt-4.1-mini-2025-04-14', - object='chat.completion', - service_tier='default', - system_fingerprint='fp_376a7ccef1', - usage=CompletionUsage( - completion_tokens=10, - prompt_tokens=79, - total_tokens=89, - completion_tokens_details=CompletionTokensDetails( - accepted_prediction_tokens=None, - audio_tokens=0, - reasoning_tokens=0, - rejected_prediction_tokens=None, - ), - prompt_tokens_details=PromptTokensDetails(audio_tokens=0, cached_tokens=0), - ), -) -""" -``` - -## Raw response with a list response model - -If your response model is a list (for example, `list[UserExtract]`), you can still use `create_with_completion()`. Instructor wraps the list in a `ResponseList` (also called `ListResponse`) that behaves like a normal list but also preserves the raw response. - -### What is ResponseList? - -`ResponseList` is a special list type that Instructor uses when your `response_model` is a list. It extends Python's built-in `list` type and adds a `_raw_response` attribute to store the provider's raw response object. - -This is necessary because `create_with_completion()` needs to return both the parsed result and the raw response. For single objects, this is straightforward: `(model_instance, raw_response)`. For lists, we need a way to attach the raw response to the list itself, which is what `ResponseList` does. - -### Using ResponseList - -The returned value behaves exactly like a normal Python list, but you can access the raw response using `get_raw_response()`: - -```python -import instructor -from pydantic import BaseModel - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class UserExtract(BaseModel): - name: str - age: int - - -users, completion = client.create_with_completion( - response_model=list[UserExtract], - messages=[ - {"role": "user", "content": "Extract users: Jason is 25, Ivan is 30"}, - ], -) - -# Use it like a normal list -print(users[0]) -#> name='Jason' age=25 -print(len(users)) -#> 2 - -# Access the raw response -raw = users.get_raw_response() -assert raw == completion - -# ResponseList supports all list operations -for user in users: - print(user.name) -#> Jason -#> Ivan -``` - -## See Also - -- [Hooks](./hooks.md) - Monitor LLM interactions without accessing raw responses -- [Debugging](../debugging.md) - Debugging techniques for LLM outputs -- [Response Models](./models.md) - Working with structured response models - -## Anthropic Raw Response - -You can also access the raw response from Anthropic models. This is useful for debugging or when you need to access additional information from the response. - -```python -import instructor - -client = instructor.from_provider("anthropic/claude-3-5-sonnet-latest") - - -user, completion = client.create_with_completion( - response_model=UserExtract, - messages=[ - {"role": "user", "content": "Extract jason is 25 years old"}, - ], -) - -print(user) -#> name='Jason' age=25 - -print(completion) -""" \ No newline at end of file diff --git a/참고/instructor-main/docs/concepts/reask_validation.md b/참고/instructor-main/docs/concepts/reask_validation.md deleted file mode 100644 index 7c15507..0000000 --- a/참고/instructor-main/docs/concepts/reask_validation.md +++ /dev/null @@ -1,327 +0,0 @@ ---- -title: Enhancing AI Validations with Pydantic's Framework -description: Learn how to improve AI outputs using Pydantic for validation and reasking techniques. ---- - -# Validation and Reasking - -Instead of framing "self-critique" or "self-reflection" in AI as new concepts, we can view them as validation errors with clear error messages that the system can use to self-correct. - -## Pydantic - -Pydantic offers a customizable and expressive validation framework for Python. Instructor leverages Pydantic's validation framework to provide a uniform developer experience for both code-based and LLM-based validation, as well as a reasking mechanism for correcting LLM outputs based on validation errors. To learn more check out the [Pydantic docs](https://docs.pydantic.dev/latest/concepts/validators/) on validators. - -!!! note "Good llm validation is just good validation" - - If you want to see some more examples on validators checkout our blog post [Good LLM validation is just good validation](https://python.useinstructor.com/blog/2023/10/23/good-llm-validation-is-just-good-validation/) - -### Code-based Validation Example - -First define a Pydantic model with a validator using the `Annotation` class from `typing_extensions`. - -Enforce a naming rule using Pydantic's built-in validation: - -```python hl_lines="5-8 12" -from pydantic import BaseModel, ValidationError -from typing_extensions import Annotated -from pydantic import AfterValidator - - -def name_must_contain_space(v: str) -> str: - if " " not in v: - raise ValueError("Name must contain a space.") - return v.lower() - - -class UserDetail(BaseModel): - age: int - name: Annotated[str, AfterValidator(name_must_contain_space)] - - -try: - person = UserDetail(age=29, name="Jason") -except ValidationError as e: - print(e) - """ - 1 validation error for UserDetail - name - Value error, Name must contain a space. [type=value_error, input_value='Jason', input_type=str] - For further information visit https://errors.pydantic.dev/2.11/v/value_error - """ -``` - -#### Output for Code-Based Validation - -```plaintext -1 validation error for UserDetail -name - Value error, name must contain a space (type=value_error) -``` - -As we can see, Pydantic raises a validation error when the name attribute does not contain a space. This is a simple example, but it demonstrates how Pydantic can be used to validate attributes of a model. - -### LLM-Based Validation Example - -LLM-based validation can also be plugged into the same Pydantic model. Here, if the answer attribute contains content that violates the rule "don't say objectionable things," Pydantic will raise a validation error. - -```python hl_lines="9 15" -import instructor -from instructor import llm_validator -from pydantic import BaseModel, ValidationError, BeforeValidator -from typing_extensions import Annotated - - -# Apply the patch to the OpenAI client -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class QuestionAnswer(BaseModel): - question: str - answer: Annotated[ - str, - BeforeValidator(llm_validator("don't say objectionable things", client=client)), - ] - - -try: - qa = QuestionAnswer( - question="What is the meaning of life?", - answer="The meaning of life is to be evil and steal", - ) -except ValidationError as e: - print(e) - """ - 1 validation error for QuestionAnswer - answer - Assertion failed, The statement promotes objectionable behavior by encouraging evil and stealing. [type=assertion_error, input_value='The meaning of life is to be evil and steal', input_type=str] - For further information visit https://errors.pydantic.dev/2.11/v/assertion_error - """ -``` - -#### Output for LLM-Based Validation - -It is important to note here that the error message is generated by the LLM, not the code, so it'll be helpful for re-asking the model. - -```plaintext -1 validation error for QuestionAnswer -answer - Assertion failed, The statement is objectionable. (type=assertion_error) -``` - -## Using Reasking Logic to Correct Outputs - -Validators are a great tool for ensuring some property of the outputs. When you use the `patch()` method with the `openai` client, you can use the `max_retries` parameter to set the number of times you can reask the model to correct the output. - -It is a great layer of defense against bad outputs of two forms: - -1. Pydantic Validation Errors (code or llm based) -2. JSON Decoding Errors (when the model returns a bad response) - -### Step 1: Define the Response Model with Validators - -Notice that the field validator wants the name in uppercase, but the user input is lowercase. The validator will raise a `ValueError` if the name is not in uppercase. - -```python hl_lines="12-17" -import instructor -from pydantic import BaseModel, field_validator - -# Apply the patch to the OpenAI client -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class UserDetails(BaseModel): - name: str - age: int - - @field_validator("name") - @classmethod - def validate_name(cls, v): - if v.upper() != v: - raise ValueError("Name must be in uppercase.") - return v -``` - -### Step 2. Using the Client with Retries - -Here, the `UserDetails` model is passed as the `response_model`, and `max_retries` is set to 2. - -```python -import instructor -from pydantic import BaseModel - -client = instructor.from_provider( - "openai/gpt-4.1-mini", - mode=instructor.Mode.TOOLS, -) - - -class UserDetails(BaseModel): - name: str - age: int - - -model = client.create( - response_model=UserDetails, - max_retries=2, - messages=[ - {"role": "user", "content": "Extract jason is 25 years old"}, - ], -) - -print(model.model_dump_json(indent=2)) -""" -{ - "name": "jason", - "age": 25 -} -""" -``` - -### What happens behind the scenes? - -Behind the scenes, the `instructor.from_provider()` method adds a `max_retries` parameter to the `openai.ChatCompletion.create()` method. The `max_retries` parameter will trigger up to 2 reattempts if the `name` attribute fails the uppercase validation in `UserDetails`. - -```python -from pydantic import ValidationError - - -try: - ... -except ValidationError as e: - kwargs["messages"].append(response.choices[0].message) - kwargs["messages"].append( - { - "role": "user", - "content": f"Please correct the function call; errors encountered:\n{e}", - } - ) -``` - -## Advanced Validation Techniques - -### Using Context for Dynamic Validation - -The `context` parameter allows you to pass additional data to your validators, enabling validation against runtime data like source documents, allowed values, or external references. This is accessed in validators via `ValidationInfo`. - -Here's a complete example showing context-based validation: - -```python -import instructor -from pydantic import BaseModel, ValidationInfo, field_validator - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class QuoteExtraction(BaseModel): - """Extract a claim with a supporting quote from source text.""" - - claim: str - supporting_quote: str - - @field_validator('supporting_quote') - @classmethod - def verify_quote_in_source(cls, v: str, info: ValidationInfo): - """Verify the quote exists in the source text.""" - import re - - context = info.context - if context: - source_text = context.get('source_text', '') - # Normalize whitespace for comparison - normalized_source = re.sub(r'\s+', ' ', source_text.strip()) - normalized_quote = re.sub(r'\s+', ' ', v.strip()) - if normalized_quote not in normalized_source: - raise ValueError( - f"The quote must be an exact substring from the source text. " - f"Quote '{v}' was not found in the source." - ) - return v - - -source_text = """ -The Python programming language was created by Guido van Rossum -and first released in 1991. It emphasizes code readability and -simplicity, making it popular for beginners and experts alike. -""" - -extraction = client.create( - response_model=QuoteExtraction, - max_retries=2, - messages=[ - { - "role": "system", - "content": "Extract a claim and find an exact quote from the text that supports it.", - }, - { - "role": "user", - "content": "Source text: {{ source_text }}\n\nExtract a claim about Python.", - }, - ], - context={"source_text": source_text}, -) - -print(f"Claim: {extraction.claim}") -#> Claim: Python emphasizes code readability and simplicity. -print(f"Quote: {extraction.supporting_quote}") -""" -Quote: It emphasizes code readability and simplicity, making it popular for beginners and experts alike. -""" -``` - -In this example: -- The `context` parameter passes the source text to the validator -- `ValidationInfo` provides access to the context in the validator -- If the LLM generates a quote that doesn't exist in the source, validation fails and the model is re-asked - -For more advanced examples including multi-field validation and citation verification, check out our [exact citations example](../examples/exact_citations.md). - -## Optimizing Token usage - -Pydantic automatically includes a URL within the error message itself when an error is thrown so that users can learn more about the specific error that was thrown. Some users might want to remove this URL since it adds extra tokens that otherwise might not add much value to the validation process. - -We've created a small helper function that you can use below which removes this url in the error message - -```python hl_lines="6" -from instructor.utils import disable_pydantic_error_url -from pydantic import BaseModel, ValidationError -from typing_extensions import Annotated -from pydantic import AfterValidator - -disable_pydantic_error_url() # (1)! - - -def name_must_contain_space(v: str) -> str: - if " " not in v: - raise ValueError("Name must contain a space.") - return v.lower() - - -class UserDetail(BaseModel): - age: int - name: Annotated[str, AfterValidator(name_must_contain_space)] - - -try: - person = UserDetail(age=29, name="Jason") -except ValidationError as e: - print(e) - """ - 1 validation error for UserDetail - name - Value error, Name must contain a space. [type=value_error, input_value='Jason', input_type=str] - """ -``` - -1. We disable the error by setting an environment variable `PYDANTIC_ERRORS_INCLUDE_URL` to `0`. This is valid only for the duration that the script is executed for, once the function is not called, the original behaviour is restored. - -## See Also - -- [Validation](./validation.md) - Core validation concepts and strategies -- [Retrying](./retrying.md) - Configure automatic retry behavior with Tenacity -- [Custom Validators](../learning/validation/custom_validators.md) - Build custom validation logic -- [Field Validation](../learning/patterns/field_validation.md) - Field-level validation patterns -- [Retry Mechanisms](../learning/validation/retry_mechanisms.md) - Practical retry configuration guide - -## Takeaways - -By integrating these advanced validation techniques, we not only improve the quality and reliability of LLM-generated content, but also pave the way for more autonomous and effective systems. diff --git a/참고/instructor-main/docs/concepts/response.png b/참고/instructor-main/docs/concepts/response.png deleted file mode 100644 index 4a8a12c..0000000 Binary files a/참고/instructor-main/docs/concepts/response.png and /dev/null differ diff --git a/참고/instructor-main/docs/concepts/retrying.md b/참고/instructor-main/docs/concepts/retrying.md deleted file mode 100644 index 03a0c38..0000000 --- a/참고/instructor-main/docs/concepts/retrying.md +++ /dev/null @@ -1,327 +0,0 @@ ---- -title: "Retry Logic with Tenacity" -description: "Learn how to implement retry logic with Tenacity for LLM applications, including exponential backoff, conditional retries, and error handling." ---- - -# Retry Logic with Tenacity - -Tenacity is a Python library for adding retry logic to your applications. Combined with Instructor, it helps handle API failures, rate limits, and validation errors. - -## Basic Retry with Exponential Backoff - -The most common pattern uses exponential backoff to delay retries: - -```python -import instructor -from pydantic import BaseModel -from tenacity import retry, stop_after_attempt, wait_exponential - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class UserInfo(BaseModel): - name: str - age: int - email: str - - -@retry(stop=stop_after_attempt(3), wait=wait_exponential(multiplier=1, min=4, max=10)) -def extract_user_info(text: str) -> UserInfo: - """Extract user information with retry logic.""" - return client.create( - response_model=UserInfo, - messages=[{"role": "user", "content": f"Extract user info: {text}"}], - ) - - -try: - user = extract_user_info("John is 30 years old with email john@example.com") - print(f"Success: {user.name}, {user.age}, {user.email}") - #> Success: John, 30, john@example.com -except Exception as e: - print(f"Failed after retries: {e}") -``` - -## Error-Specific Retries - -Retry only on specific error types for better control: - -```python -import instructor -from openai import APIError, RateLimitError -from pydantic import BaseModel, ValidationError -from tenacity import ( - retry, - retry_if_exception_type, - stop_after_attempt, - wait_exponential, -) - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class UserInfo(BaseModel): - name: str - age: int - email: str - - -# Retry on API errors with longer delays -@retry( - retry=retry_if_exception_type((RateLimitError, APIError)), - stop=stop_after_attempt(5), - wait=wait_exponential(multiplier=2, min=1, max=60), -) -def handle_api_errors(text: str) -> UserInfo: - return client.create( - response_model=UserInfo, - messages=[{"role": "user", "content": text}], - ) - - -# Retry on validation errors with shorter delays -@retry( - retry=retry_if_exception_type(ValidationError), - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=1, max=10), -) -def handle_validation_errors(text: str) -> UserInfo: - return client.create( - response_model=UserInfo, - messages=[{"role": "user", "content": text}], - ) -``` - -## Custom Retry Conditions - -Retry based on the result content rather than exceptions: - -```python -import instructor -from pydantic import BaseModel -from tenacity import retry, retry_if_result, stop_after_attempt - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class UserInfo(BaseModel): - name: str - age: int - email: str - - -def should_retry(result: UserInfo) -> bool: - """Retry if the result doesn't meet quality criteria.""" - return result.age < 0 or result.age > 150 or not result.email - - -@retry(retry=retry_if_result(should_retry), stop=stop_after_attempt(3)) -def extract_valid_user(text: str) -> UserInfo: - return client.create( - response_model=UserInfo, - messages=[{"role": "user", "content": text}], - ) -``` - -## Context-Based Validation with Retries - -Use the `context` parameter to pass runtime data to validators: - -```python -import instructor -from pydantic import BaseModel, ValidationInfo, field_validator, ValidationError -from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class Citation(BaseModel): - """A claim with a supporting quote from source text.""" - - claim: str - quote: str - - @field_validator('quote') - @classmethod - def verify_quote_exists(cls, v: str, info: ValidationInfo): - context = info.context - if context: - source_text = context.get('source_text', '') - if v not in source_text: - raise ValueError(f"Quote '{v}' not found in source text.") - return v - - -@retry( - retry=retry_if_exception_type(ValidationError), - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=2, max=10), -) -def extract_citation(claim: str, source_text: str) -> Citation: - return client.create( - response_model=Citation, - messages=[ - { - "role": "system", - "content": "Extract the claim and find an exact quote from the source.", - }, - { - "role": "user", - "content": "Source: {{ source_text }}\n\nClaim: {{ claim }}", - }, - ], - context={"source_text": source_text, "claim": claim}, - ) - - -source = "The Eiffel Tower was completed in 1889 and stands 330 meters tall." -citation = extract_citation("The tower is over 300 meters", source) -print(f"Quote: {citation.quote}") -``` - -## Logging and Monitoring - -Add logging to track retry attempts: - -```python -import logging -import instructor -from pydantic import BaseModel -from tenacity import after_log, before_log, retry, stop_after_attempt, wait_exponential - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class UserInfo(BaseModel): - name: str - age: int - email: str - - -logger = logging.getLogger(__name__) -logging.basicConfig(level=logging.INFO) - - -@retry( - stop=stop_after_attempt(3), - wait=wait_exponential(multiplier=1, min=4, max=10), - before=before_log(logger, logging.INFO), - after=after_log(logger, logging.ERROR), -) -def logged_extraction(text: str) -> UserInfo: - return client.create( - response_model=UserInfo, - messages=[{"role": "user", "content": text}], - ) -``` - -## Instructor's Built-in Retries - -Instructor has built-in retry support that works alongside Tenacity: - -```python -import instructor -from instructor import Mode -from pydantic import BaseModel -from tenacity import retry, stop_after_attempt - -client = instructor.from_provider( - "openai/gpt-4.1-mini", - mode=Mode.JSON, - max_retries=3, - retry_delay=1, -) - - -class UserInfo(BaseModel): - name: str - age: int - email: str - - -# Combine Instructor and Tenacity retries for additional resilience -@retry(stop=stop_after_attempt(2)) -def double_retry_extraction(text: str) -> UserInfo: - return client.create( - response_model=UserInfo, - messages=[{"role": "user", "content": text}], - ) -``` - -## Failed Attempts Tracking - -When retries fail, Instructor provides detailed failure history: - -```python -import instructor -from instructor.core.exceptions import InstructorRetryException -from pydantic import BaseModel, field_validator - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class UserInfo(BaseModel): - name: str - age: int - - @field_validator('age') - @classmethod - def validate_age(cls, v): - if v < 0 or v > 150: - raise ValueError(f"Age {v} is invalid") - return v - - -try: - result = client.create( - response_model=UserInfo, - messages=[{"role": "user", "content": "Extract: John is -5 years old"}], - max_retries=3, - ) -except InstructorRetryException as e: - print(f"Failed after {e.n_attempts} attempts") - for attempt in e.failed_attempts: - print(f"Attempt {attempt.attempt_number}: {attempt.exception}") -``` - -Failed attempts are automatically propagated to reask handlers, enabling contextual error messages and progressive corrections. - -## Best Practices - -### Choose Appropriate Strategies - -| Error Type | Attempts | Min Delay | Max Delay | -|------------|----------|-----------|-----------| -| Rate limits | 5 | 1s | 60-120s | -| Validation errors | 2-3 | 1s | 10s | -| Network errors | 4 | 2s | 30s | - -### Always Set Stop Conditions - -```python -from tenacity import retry, stop_after_attempt - -# Good: bounded retries -@retry(stop=stop_after_attempt(3)) -def bounded_retry(): - pass - -# Bad: could retry forever -@retry() # Don't do this! -def unbounded_retry(): - pass -``` - -## Troubleshooting - -**Infinite retries**: Always set `stop_after_attempt()` or `stop_after_delay()`. - -**Too many retries**: Use `retry_if_exception_type()` to retry only on specific errors. - -**Still hitting rate limits**: Increase max delay and use `wait_exponential()` with higher multipliers. - -## Related Resources - -- [Tenacity Documentation](https://tenacity.readthedocs.io/) -- [Error Handling](./error_handling.md) -- [Validation](./validation.md) diff --git a/참고/instructor-main/docs/concepts/semantic_validation.md b/참고/instructor-main/docs/concepts/semantic_validation.md deleted file mode 100644 index 41906af..0000000 --- a/참고/instructor-main/docs/concepts/semantic_validation.md +++ /dev/null @@ -1,457 +0,0 @@ ---- -title: Semantic Validation with LLMs -description: Using LLMs for complex validation that goes beyond rule-based approaches to evaluate content based on natural language criteria. ---- - -## See Also - -- [Validation](./validation.md) - Core validation concepts and strategies -- [Custom Validators](../learning/validation/custom_validators.md) - Build custom validation logic -- [Field Validation](../learning/patterns/field_validation.md) - Field-level validation patterns -- [Reask Validation](./reask_validation.md) - Automatic retry with validation feedback -- [LLM Validator](./validation.md#semantic-validation) - Semantic validation examples - -# Semantic Validation with LLMs - -This guide covers semantic validation in Instructor - using LLMs themselves to validate content against complex, subjective, or contextual criteria that would be difficult to implement with traditional rule-based approaches. - -## Overview - -Semantic validation leverages the language understanding capabilities of LLMs to validate inputs against natural language criteria. While traditional validation uses explicit rules and patterns, semantic validation can understand nuance, context, and subjective qualities in data. - -### When to Use Semantic Validation - -Semantic validation is particularly useful for: - -- **Complex criteria** that are difficult to express with rules -- **Subjective qualities** like tone, style, or appropriateness -- **Contextual validation** that requires understanding relationships between fields -- **Policy enforcement** that involves nuanced understanding of guidelines -- **Content moderation** for detecting harmful or inappropriate content - -### How It Works - -In Instructor, semantic validation is implemented through the `llm_validator` function, which creates a validator that uses an LLM to check if values conform to specified requirements: - -```python -import instructor -from typing import Annotated -from pydantic import BaseModel, BeforeValidator -from instructor import llm_validator - -# Initialize client -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class UserComment(BaseModel): - username: str - comment: Annotated[ - str, - BeforeValidator( - llm_validator( - "Comment must be constructive, respectful, and not contain hate speech or profanity", - client=client, - ) - ), - ] -``` - -The `llm_validator` function takes: - -1. A natural language description of the validation criteria -2. An Instructor client instance to perform the validation -3. Optional parameters for configuration - -During validation, the LLM evaluates whether the input matches the specified criteria and either passes the value or raises a validation error with a detailed explanation. - -## Validation Flow - -The following diagram illustrates how semantic validation works in Instructor: - -```mermaid -flowchart TD - A[Input Data] --> B[Pydantic Validation Process] - B --> C{Field has Semantic\nValidator?} - C -->|No| D[Standard Validation] - C -->|Yes| E[Call LLM with Validation Criteria] - E --> F{LLM Determines\nValue is Valid?} - F -->|Yes| G[Validation Passes] - F -->|No| H[Validation Fails with LLM-Generated Error] - H --> I{Auto-Retry Enabled?} - I -->|Yes| J[Try Again with Error Context] - I -->|No| K[Return Validation Error] - J --> E - - classDef process fill:#e2f0fb,stroke:#b8daff,color:#004085; - classDef decision fill:#fff3cd,stroke:#ffeeba,color:#856404; - classDef success fill:#d4edda,stroke:#c3e6cb,color:#155724; - classDef error fill:#f8d7da,stroke:#f5c6cb,color:#721c24; - - class A,B,E,J process - class C,F,I decision - class G,D success - class H,K error -``` - -## Basic Usage - -Here's a complete example of semantic validation in action: - -```python -# Standard library imports -from typing import Annotated - -# Third-party imports -from pydantic import BaseModel, BeforeValidator -import instructor -from instructor import llm_validator - -# Initialize client -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class ProductDescription(BaseModel): - """Model for validating product descriptions.""" - - name: str - description: Annotated[ - str, - BeforeValidator( - llm_validator( - """The description must be: - 1. Professional and factual - 2. Free of excessive hyperbole or unsubstantiated claims - 3. Between 50-200 words in length - 4. Written in third person (no "you" or "your") - 5. Free of spelling and grammar errors""", - client=client, - ) - ), - ] - - -# Example usage with Jinja templating -try: - product = client.create( - response_model=ProductDescription, - messages=[ - { - "role": "system", - "content": "Generate a product description based on the product name.", - }, - {"role": "user", "content": "Create a description for: {{ product_name }}"}, - ], - context={"product_name": "UltraClean 9000 Washing Machine"}, - ) - print(product.model_dump_json(indent=2)) - """ - { - "name": "UltraClean 9000 Washing Machine", - "description": "The UltraClean 9000 Washing Machine offers reliable and efficient cleaning with multiple wash settings and a high-capacity drum. It features an easy-to-use control panel and a design that suits modern home environments. The machine aims to provide a practical solution for everyday laundry needs with standard noise levels and energy consumption." - } - """ -except Exception as e: - print(f"Validation error: {e}") - """ - Validation error: - - - - 1 validation error for ProductDescription - description - Assertion failed, The description contains excessive hyperbole and unsubstantiated claims. It needs to be more professional and factual. [type=assertion_error, input_value='The UltraClean 9000 Wash...ior laundry experience.', input_type=str] - - - ChatCompletion(id='chatcmpl-D08R5P8Ne4q4TvAbiSa6Kh18wQxQd', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content=None, refusal=None, role='assistant', annotations=[], audio=None, function_call=None, tool_calls=[ChatCompletionMessageFunctionToolCall(id='call_RZlWM3SJheQAv84bS1apYcFJ', function=Function(arguments='{"name":"UltraClean 9000 Washing Machine","description":"The UltraClean 9000 Washing Machine is a state-of-the-art appliance designed to deliver exceptional cleaning performance with maximum efficiency. Featuring advanced cleaning technology, multiple wash cycles, and energy-saving modes, it ensures your clothes come out spotless every time. Its sleek design and user-friendly interface make laundry effortless and convenient, while durable construction guarantees long-lasting use. Ideal for modern households, the UltraClean 9000 combines powerful washing capabilities with quiet operation for a superior laundry experience."}', name='ProductDescription'), type='function')]))], created=1768924799, model='gpt-4.1-mini-2025-04-14', object='chat.completion', service_tier='default', system_fingerprint='fp_376a7ccef1', usage=CompletionUsage(completion_tokens=300, prompt_tokens=2619, total_tokens=2919, completion_tokens_details=CompletionTokensDetails(accepted_prediction_tokens=None, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=None), prompt_tokens_details=PromptTokensDetails(audio_tokens=0, cached_tokens=0))) - - - - - - 1 validation error for ProductDescription - description - Assertion failed, The description contains hyperbolic and exaggerated language, which does not align with the requirement of being professional and factual. It also includes unsubstantiated claims such as 'efficient laundry' and 'reliable performance'. [type=assertion_error, input_value='The UltraClean 9000 Wash...lar home laundry needs.', input_type=str] - - - ChatCompletion(id='chatcmpl-D08R96HSWzEZhcj9nWHCn4th6IIxB', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content=None, refusal=None, role='assistant', annotations=[], audio=None, function_call=None, tool_calls=[ChatCompletionMessageFunctionToolCall(id='call_jsbD8AbEK8MvFWkVPOK0mooT', function=Function(arguments='{"name":"UltraClean 9000 Washing Machine","description":"The UltraClean 9000 Washing Machine is designed for efficient laundry with multiple wash settings to suit different fabric types. It includes energy-saving features to reduce power consumption during operation. The machine has a capacity suitable for medium to large households and operates with reduced noise levels. The user interface is straightforward, offering ease of use. Built with durable materials, the UltraClean 9000 provides reliable performance for regular home laundry needs."}', name='ProductDescription'), type='function')]))], created=1768924803, model='gpt-4.1-mini-2025-04-14', object='chat.completion', service_tier='default', system_fingerprint='fp_376a7ccef1', usage=CompletionUsage(completion_tokens=300, prompt_tokens=2619, total_tokens=2919, completion_tokens_details=CompletionTokensDetails(accepted_prediction_tokens=None, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=None), prompt_tokens_details=PromptTokensDetails(audio_tokens=0, cached_tokens=0))) - - - - - - 1 validation error for ProductDescription - description - Assertion failed, The description contains some marketing language and exaggerated claims, which do not align with a professional and factual tone. It also lacks specific details and technical information about the washing machine. [type=assertion_error, input_value="The UltraClean 9000 Wash...ehold washing machines.", input_type=str] - - - ChatCompletion(id='chatcmpl-D08RCpkeVCnl1jfV4HXHHRxogx46h', choices=[Choice(finish_reason='stop', index=0, logprobs=None, message=ChatCompletionMessage(content=None, refusal=None, role='assistant', annotations=[], audio=None, function_call=None, tool_calls=[ChatCompletionMessageFunctionToolCall(id='call_1MdJh2HvUMYzIxU8qj9BPmCG', function=Function(arguments='{"name":"UltraClean 9000 Washing Machine","description":"The UltraClean 9000 Washing Machine features multiple wash cycles and fabric care settings. It is designed to operate with an energy-saving mode to reduce electricity usage. The machine\'s capacity supports the needs of medium to large households. It includes noise reduction technology for quieter operation and has a user interface with basic controls for ease of operation. The machine is constructed from standard materials commonly used in household washing machines."}', name='ProductDescription'), type='function')]))], created=1768924806, model='gpt-4.1-mini-2025-04-14', object='chat.completion', service_tier='default', system_fingerprint='fp_376a7ccef1', usage=CompletionUsage(completion_tokens=300, prompt_tokens=2619, total_tokens=2919, completion_tokens_details=CompletionTokensDetails(accepted_prediction_tokens=None, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=None), prompt_tokens_details=PromptTokensDetails(audio_tokens=0, cached_tokens=0))) - - - - - - - 1 validation error for ProductDescription - description - Assertion failed, The description contains some marketing language and exaggerated claims, which do not align with a professional and factual tone. It also lacks specific details and technical information about the washing machine. [type=assertion_error, input_value="The UltraClean 9000 Wash...ehold washing machines.", input_type=str] - - """ -``` - -## Advanced Validation Patterns - -### Content Policy Enforcement - -This example validates user-generated content against community guidelines: - -```python -import instructor -from typing import Annotated -from pydantic import BaseModel, BeforeValidator -from instructor import llm_validator - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class Comment(BaseModel): - """Model representing a user comment with content moderation.""" - - user_id: str - content: Annotated[ - str, - BeforeValidator( - llm_validator( - """Content must comply with community guidelines: - - No hate speech, harassment, or discrimination - - No explicit sexual or violent content - - No promotion of illegal activities - - No sharing of personal information - - No spamming or excessive self-promotion""", - client=client, - ) - ), - ] -``` - -### Topic Relevance Validation - -This validator ensures that responses stay on topic: - -```python -import instructor -from typing import Annotated -from pydantic import BaseModel, BeforeValidator -from instructor import llm_validator - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class ForumPost(BaseModel): - topic: str - post: Annotated[ - str, - BeforeValidator( - llm_validator( - "The post must be directly relevant to the specified topic and not drift to unrelated subjects", - client=client, - ) - ), - ] - - # Using Jinja templating for validation against dynamic values - @classmethod - def validate_post(cls, topic_name: str, post_content: str) -> "ForumPost": - return client.create( - response_model=cls, - messages=[ - { - "role": "system", - "content": """Validate that the forum post content stays relevant to the topic. - If it's not relevant, explain why in detail.""", - }, - { - "role": "user", - "content": """ - Topic: {{ topic }} - - Post content: - {{ post }} - - Is this post relevant to the topic? - """, - }, - ], - context={ - "topic": topic_name, - "post": post_content, - }, - ) -``` - -### Fact-Checking Validator - -This complex validator assesses factual accuracy: - -```python -import instructor -from typing import List -from pydantic import BaseModel, Field - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class FactCheckedClaim(BaseModel): - """Model for validating factual accuracy of claims.""" - - claim: str - is_accurate: bool = Field(description="Whether the claim is factually accurate") - supporting_evidence: List[str] = Field( - default_factory=list, - description="Evidence supporting or refuting the claim", - ) - - @classmethod - def validate_claim(cls, text: str) -> "FactCheckedClaim": - return client.create( - response_model=cls, - messages=[ - { - "role": "system", - "content": "You are a fact-checking system. Assess the factual accuracy of the claim.", - }, - { - "role": "user", - "content": "Fact check this claim: {{ claim }}", - }, - ], - context={"claim": text}, - ) -``` - -## Complex Multi-Field Validation - -For validation that needs to compare multiple fields, you can use model validators: - -```python -import instructor -from typing import List -from pydantic import BaseModel, model_validator -from instructor.validation import Validator # For response type - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class Report(BaseModel): - """Model representing a report with related fields that need semantic validation.""" - - title: str - summary: str - key_findings: List[str] - - @model_validator(mode="after") - def validate_consistency(self): - # Semantic validation at the model level using Jinja templating - validation_result = client.create( - response_model=Validator, - messages=[ - { - "role": "system", - "content": "Validate that the summary accurately reflects the key findings.", - }, - { - "role": "user", - "content": """ - Please validate if this summary accurately reflects the key findings: - - Title: {{ title }} - Summary: {{ summary }} - - Key findings: - {% for finding in findings %} - - {{ finding }} - {% endfor %} - - Evaluate for consistency, completeness, and accuracy. - """, - }, - ], - context={ - "title": self.title, - "summary": self.summary, - "findings": self.key_findings, - }, - ) - - if not validation_result.is_valid: - raise ValueError(f"Consistency error: {validation_result.reason}") - - return self -``` - -## Best Practices - -1. **Be Specific in Criteria**: Provide clear, detailed validation criteria in natural language -2. **Use Appropriate Models**: Larger models tend to give better, more nuanced validation -3. **Balance Cost and Latency**: Remember that each validation adds an LLM API call -4. **Provide Examples**: Include examples of both valid and invalid content in your criteria -5. **Handle Retries**: Configure retry logic for edge cases -6. **Use Jinja Templates**: When validating against dynamic values, use Jinja templating -7. **Separate Concerns**: Keep validation criteria focused on specific aspects -8. **Consider Context**: Use model-level validation when comparing multiple fields - -## Advanced Configuration - -The `llm_validator` function supports several configuration options: - -```python -import instructor -from instructor import llm_validator -from pydantic import BaseModel, BeforeValidator -from typing import Annotated - -client = instructor.from_provider("openai/gpt-4.1-mini") - -# Configure the validator with options -validator = llm_validator( - statement="Must be a professional, concise product description", - client=client, # Required Instructor client - allow_override=True, # Allow LLM to fix invalid values - model="gpt-4o", # Specify model to use for validation - temperature=0.2, # Add variability (default is 0) -) - - -class Product(BaseModel): - description: Annotated[str, BeforeValidator(validator)] -``` - -## Performance Considerations - -Semantic validation adds API calls to your workflow, which impacts: - -1. **Latency**: Each validation requires an additional API call -2. **Cost**: More API calls mean higher usage costs -3. **Reliability**: Depends on API availability and response quality - -Consider these trade-offs when implementing semantic validation, especially for high-volume applications. - -## Comparison with Rule-Based Validation - -| Aspect | Rule-Based Validation | Semantic Validation | -|--------|----------------------|---------------------| -| **Implementation** | Regular expressions, constraints | Natural language criteria | -| **Complexity** | Simple rules, explicit patterns | Can handle subjective criteria | -| **Speed** | Fast, no external calls | Slower, requires API calls | -| **Cost** | No additional API costs | Each validation costs tokens | -| **Flexibility** | Limited to programmable rules | Can validate against any natural language criteria | -| **Maintenance** | Rules must be updated manually | Criteria can be more adaptable | - -## Related Resources - -- [Validation in Instructor](./validation.md) - Core validation concepts -- [Custom Validators](../learning/validation/custom_validators.md) - Creating custom validators -- [llm_validator API Reference](../api.md#api-reference) - Full API reference - ---- - -Semantic validation expands what's possible with validation beyond traditional rule-based approaches. By using LLMs to validate content against natural language criteria, you can build more sophisticated validation systems that understand context, nuance, and complex relationships. diff --git a/참고/instructor-main/docs/concepts/templating.md b/참고/instructor-main/docs/concepts/templating.md deleted file mode 100644 index 381e16a..0000000 --- a/참고/instructor-main/docs/concepts/templating.md +++ /dev/null @@ -1,243 +0,0 @@ ---- -title: Prompt Templating with Jinja - Dynamic Prompt Generation -description: Create dynamic prompts using Jinja templating with Instructor. Build reusable, versioned prompts with Pydantic validation and security. ---- - -# Prompt Templating - -With Instructor's Jinja templating, you can: - -- Dynamically adapt prompts to any context -- Easily manage and version your prompts better -- Integrate seamlessly with validation processes -- Handle sensitive information securely - -Our solution offers: - -- Separation of prompt structure and content -- Complex logic implementation within prompts -- Template reusability across scenarios -- Enhanced prompt versioning and logging -- Pydantic integration for validation and type safety - -## Context is available to the templating engine - -The `context` parameter is a dictionary that is passed to the templating engine. It is used to pass in the relevant variables to the templating engine. This single `context` parameter will be passed to jinja to render out the final prompt. - -```python hl_lines="14-15 19-22" -import instructor -from pydantic import BaseModel - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class User(BaseModel): - name: str - age: int - - -resp = client.create( - messages=[ - { - "role": "user", - "content": """Extract the information from the - following text: `{{ data }}`""", # (1)! - }, - ], - response_model=User, - context={"data": "John Doe is thirty years old"}, # (2)! -) - -print(resp) -#> name='John Doe' age=30 -``` - -1. Declare jinja style template variables inside the prompt itself (e.g. `{{ name }}`) -2. Pass in the variables to be used in the `context` parameter - -### Context is available to Pydantic validators - -In this example, we demonstrate how to leverage the `context` parameter with Pydantic validators to enhance our validation and data processing capabilities. By passing the `context` to the validators, we can implement dynamic validation rules and data transformations based on the input context. This approach allows for flexible and context-aware validation, such as checking for banned words or applying redaction patterns to sensitive information. - -```python hl_lines="15-16 26-30" -import instructor -from pydantic import BaseModel, ValidationInfo, field_validator -import re - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class Response(BaseModel): - text: str - - @field_validator('text') - @classmethod - def redact_regex(cls, v: str, info: ValidationInfo): - context = info.context - if context: - redact_patterns = context.get('redact_patterns', []) - for pattern in redact_patterns: - v = re.sub(pattern, '****', v) - return v - - -response = client.create( - response_model=Response, - messages=[ - { - "role": "user", - "content": """ - Write about a {{ topic }} - - {% if banned_words %} - You must not use the following banned words: - - - {% for word in banned_words %} - * {{ word }} - {% endfor %} - - {% endif %} - """, - }, - ], - context={ - "topic": "jason and now his phone number is 123-456-7890", - "redact_patterns": [ - r"\b\d{3}[-.]?\d{3}[-.]?\d{4}\b", # Phone number pattern - r"\b\d{3}-\d{2}-\d{4}\b", # SSN pattern - ], - }, - max_retries=3, -) - -print(response.text) -""" -Jason is a young man who loves technology and enjoys staying connected with his friends and family. He is known for his friendly demeanor and his passion for learning new things. Recently, he got a new phone, and his contact number is ****. Jason uses his phone not only to communicate but also to explore various apps, stay organized, and capture moments through photography. -""" -``` - -1. Access the variables passed into the `context` variable inside your Pydantic validator - -2. Pass in the variables to be used for validation and/or rendering into the `context` parameter - -### Jinja Syntax - -Jinja is used to render the prompts, allowing the use of familiar Jinja syntax. This enables rendering of lists, conditionals, and more. It also allows calling functions and methods within Jinja. - -This makes formatting of prompts and rendering logic extremely easy. - -```python hl_lines="29-34 37-43" -import instructor -from pydantic import BaseModel - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class Citation(BaseModel): - source_ids: list[int] - text: str - - -class Response(BaseModel): - answer: list[Citation] - - -resp = client.create( - messages=[ - { - "role": "user", - "content": """ - You are a {{ role }} tasks with the following question - - - {{ question }} - - - Use the following context to answer the question, make sure to return [id] for every citation: - - - {% for chunk in context %} - - {{ chunk.id }} - {{ chunk.text }} - - {% endfor %} - - - {% if rules %} - Make sure to follow these rules: - - {% for rule in rules %} - * {{ rule }} - {% endfor %} - {% endif %} - """, - }, - ], - response_model=Response, - context={ - "role": "professional educator", - "question": "What is the capital of France?", - "context": [ - {"id": 1, "text": "Paris is the capital of France."}, - {"id": 2, "text": "France is a country in Europe."}, - ], - "rules": ["Use markdown."], - }, -) - -print(resp) -#> answer=[Citation(source_ids=[1], text='The capital of France is Paris.')] -# answer=[Citation(source_ids=[1], text='The capital of France is Paris.')] -``` - -### Working with Secrets - -Your prompts might need to include sensitive user information when they're sent to your model provider. This is probably something you don't want to hard code into your prompt or captured in your logs. An easy way to get around this is to use the `SecretStr` type from `Pydantic` in your model definitions. - -```python -from pydantic import BaseModel, SecretStr -import instructor - - -class UserContext(BaseModel): - name: str - address: SecretStr - - -class Address(BaseModel): - street: SecretStr - city: str - state: str - zipcode: str - - -client = instructor.from_provider("openai/gpt-4.1-mini") -context = UserContext(name="scolvin", address="secret address") - -address = client.create( - messages=[ - { - "role": "user", - "content": "{{ user.name }} is `{{ user.address.get_secret_value() }}`, normalize it to an address object", - }, - ], - context={"user": context}, - response_model=Address, -) -print(context) -#> name='scolvin' address=SecretStr('**********') -print(address) -""" -street=SecretStr('**********') city='secret address' state='secret address' zipcode='secret address' -""" -``` - -This allows you to preserve your sensitive information while still using it in your prompts. - -## Security - -We use the `jinja2.sandbox.SandboxedEnvironment` to prevent security issues with the templating engine. This means that you can't use arbitrary python code in your prompts. But this doesn't mean that you should pass untrusted input to the templating engine, as this could still be abused for things like Denial of Service attacks. - -You should [always sanitize](https://jinja.palletsprojects.com/en/stable/sandbox/#security-considerations) any input that you pass to the templating engine. diff --git a/참고/instructor-main/docs/concepts/typeadapter.md b/참고/instructor-main/docs/concepts/typeadapter.md deleted file mode 100644 index efd41dd..0000000 --- a/참고/instructor-main/docs/concepts/typeadapter.md +++ /dev/null @@ -1,8 +0,0 @@ ---- -title: TypeAdapter in Instructor - Custom Type Handling -description: Use Pydantic TypeAdapter for custom type validation and serialization with Instructor. Handle complex types and custom validation logic in structured outputs. ---- - -!!! warning "This page is a work in progress" - - This page is a work in progress. Check out [Pydantic's documentation](https://docs.pydantic.dev/latest/concepts/type_adapter/) diff --git a/참고/instructor-main/docs/concepts/typeddicts.md b/참고/instructor-main/docs/concepts/typeddicts.md deleted file mode 100644 index a639932..0000000 --- a/참고/instructor-main/docs/concepts/typeddicts.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: Using TypedDicts with OpenAI API -description: Learn how to utilize TypedDicts in Python with the OpenAI API for structured data responses. ---- - ---- -title: TypedDict Support in Instructor - Dictionary Validation -description: Use Python TypedDict for type-safe dictionary structures with Instructor. Validate dictionary schemas without Pydantic models for lightweight structured outputs. ---- - -# TypedDicts - -We also support typed dicts. - -```python -from typing_extensions import TypedDict -import instructor - - -class User(TypedDict): - name: str - age: int - - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -response = client.create( - response_model=User, - messages=[ - { - "role": "user", - "content": "Timothy is a man from New York who is turning 32 this year", - } - ], -) -``` \ No newline at end of file diff --git a/참고/instructor-main/docs/concepts/types.md b/참고/instructor-main/docs/concepts/types.md deleted file mode 100644 index 769f72e..0000000 --- a/참고/instructor-main/docs/concepts/types.md +++ /dev/null @@ -1,330 +0,0 @@ ---- -title: Working with Types in Instructor -description: Learn how to use different data types with Instructor, from simple primitives to complex types. ---- - -# Working with Types in Instructor - -Instructor supports a wide range of types for your structured outputs, from simple primitives to complex nested structures. - -## Simple Types - -In addition to `pydantic.BaseModel` (the recommended approach), Instructor also supports: - -- Primitive types: `str`, `int`, `float`, `bool` -- Collection types: `List`, `Dict` -- Type composition: `Union`, `Literal`, `Optional` -- Specialized outputs: [Iterable](lists.md), [Partial](partial.md) - -You can use these types directly in your `response_model` parameter without wrapping them in a Pydantic model. - -For better documentation and control, use `typing.Annotated` to add more context to your types. - -## What happens behind the scenes? - -We will actually wrap the response model with a `pydantic.BaseModel` of the following form: - -```python -from typing import Annotated -from pydantic import create_model, Field, BaseModel - -typehint = Annotated[bool, Field(description="Sample Description")] - -model = create_model("Response", content=(typehint, ...), __base__=BaseModel) - -print(model.model_json_schema()) -""" -{ - 'properties': { - 'content': { - 'description': 'Sample Description', - 'title': 'Content', - 'type': 'boolean', - } - }, - 'required': ['content'], - 'title': 'Response', - 'type': 'object', -} -""" -``` - -## Primitive Types (str, int, float, bool) - -```python -import instructor - -client = instructor.from_provider("openai/gpt-4.1-mini") - -# Response model with simple types like str, int, float, bool -resp = client.create( - response_model=bool, - messages=[ - { - "role": "user", - "content": "Is it true that Paris is the capital of France?", - }, - ], -) -assert resp is True, "Paris is the capital of France" -print(resp) -#> True -``` - -## Annotated - -Annotations can be used to add more information about the type. This can be useful for adding descriptions to the type, along with more complex information like field names, and more. - -```python -import instructor -from typing import Annotated -from pydantic import Field - -client = instructor.from_provider("openai/gpt-4.1-mini") - -UpperCaseStr = Annotated[str, Field(description="string must be upper case")] - -# Response model with simple types like str, int, float, bool -resp = client.create( - response_model=UpperCaseStr, - messages=[ - { - "role": "user", - "content": "What is the capital of france?", - }, - ], -) -assert resp == "PARIS", "Paris is the capital of France" -print(resp) -#> PARIS -``` - -## Literal - -When doing simple classification Literals go quite well, they support literal of string, int, bool. - -```python -import instructor -from typing import Literal - -client = instructor.from_provider("openai/gpt-4.1-mini") - -resp = client.create( - response_model=Literal["BILLING", "SHIPPING"], - messages=[ - { - "role": "user", - "content": "Classify the following messages: 'I am having trouble with my billing'", - }, - ], -) -assert resp == "BILLING" -print(resp) -#> BILLING -``` - -## Enum - -Enums are harder to get right without some addition promping but are useful if these are values that are shared across the application. - -```python -import instructor -from enum import Enum - - -class Label(str, Enum): - BILLING = "BILLING" - SHIPPING = "SHIPPING" - - -client = instructor.from_provider("openai/gpt-4.1-mini") - -resp = client.create( - response_model=Label, - messages=[ - { - "role": "user", - "content": "Classify the following messages: 'I am having trouble with my billing'", - }, - ], -) -assert resp == Label.BILLING -print(resp) -#> BILLING -``` - -## List - -```python -import instructor -from typing import List - -client = instructor.from_provider("openai/gpt-4.1-mini") - -resp = client.create( - response_model=List[int], - messages=[ - { - "role": "user", - "content": "Give me the first 5 prime numbers", - }, - ], -) - -assert resp == [2, 3, 5, 7, 11] -print(resp) -#> [2, 3, 5, 7, 11] -``` - -## Union - -Union is a great way to handle multiple types of responses, similar to multiple function calls but not limited to the function calling api, like in JSON_SCHEMA modes. - -```python -import instructor -from pydantic import BaseModel -from typing import Union - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class Add(BaseModel): - a: int - b: int - - -class Weather(BaseModel): - location: str - - -resp = client.create( - response_model=Union[Add, Weather], - messages=[ - { - "role": "user", - "content": "What is 5 + 5?", - }, - ], -) - -assert resp == Add(a=5, b=5) -print(resp) -#> a=5 b=5 -``` - -## See Also - -- [Response Models](./models.md) - Using Pydantic models for structured outputs -- [Enums](./enums.md) - Working with enumerated types -- [Union Types](./unions.md) - Handling multiple possible types -- [Lists](./lists.md) - Working with collections -- [Optional Fields](../learning/patterns/optional_fields.md) - Handling missing data - -## Complex Types - -### Pandas DataFrame - -This is a more complex example, where we use a custom type to convert markdown to a pandas DataFrame. - -```python -from io import StringIO -from typing import Annotated, Any -from pydantic import BeforeValidator, PlainSerializer, InstanceOf, WithJsonSchema -import pandas as pd -import instructor - - -def md_to_df(data: Any) -> Any: - # Convert markdown to DataFrame - if isinstance(data, str): - return ( - pd.read_csv( - StringIO(data), # Process data - sep="|", - index_col=1, - ) - .dropna(axis=1, how="all") - .iloc[1:] - .applymap(lambda x: x.strip()) - ) - return data - - -MarkdownDataFrame = Annotated[ - # Validates final type - InstanceOf[pd.DataFrame], - # Converts markdown to DataFrame - BeforeValidator(md_to_df), - # Converts DataFrame to markdown on model_dump_json - PlainSerializer(lambda df: df.to_markdown()), - # Adds a description to the type - WithJsonSchema( - { - "type": "string", - "description": """ - The markdown representation of the table, - each one should be tidy, do not try to join - tables that should be seperate""", - } - ), -] - - -client = instructor.from_provider("openai/gpt-4.1-mini") - -resp = client.create( - response_model=MarkdownDataFrame, - messages=[ - { - "role": "user", - "content": "Jason is 20, Sarah is 30, and John is 40", - }, - ], -) - -assert isinstance(resp, pd.DataFrame) -print(resp) -""" - Age - Name -Jason 20 -Sarah 30 -John 40 -""" -``` - -### Lists of Unions - -Just like Unions we can use List of Unions to represent multiple types of responses. This will feel similar to the parallel function calls but not limited to the function calling api, like in JSON_SCHEMA modes. - -```python -import instructor -from pydantic import BaseModel -from typing import Union, List - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class Weather(BaseModel, frozen=True): - location: str - - -class Add(BaseModel, frozen=True): - a: int - b: int - - -resp = client.create( - response_model=List[Union[Add, Weather]], - messages=[ - { - "role": "user", - "content": "Add 5 and 5, and also whats the weather in Toronto?", - }, - ], -) - -assert resp == [Add(a=5, b=5), Weather(location="Toronto")] -print(resp) -#> [Add(a=5, b=5), Weather(location='Toronto')] -``` diff --git a/참고/instructor-main/docs/concepts/union.md b/참고/instructor-main/docs/concepts/union.md deleted file mode 100644 index 555225a..0000000 --- a/참고/instructor-main/docs/concepts/union.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -title: Using Union Types in Pydantic Models -description: Learn how to implement Union types in Pydantic models to handle multiple action types in Python. ---- - -!!! note "Redirect Notice" - This page has been consolidated into the comprehensive [Union Types](./unions.md) guide. - Please visit that page for complete information about working with union types in Instructor. - - - diff --git a/참고/instructor-main/docs/concepts/unions.md b/참고/instructor-main/docs/concepts/unions.md deleted file mode 100644 index df8c4ae..0000000 --- a/참고/instructor-main/docs/concepts/unions.md +++ /dev/null @@ -1,456 +0,0 @@ ---- -title: Union Types in Instructor -description: Learn how to use Union types to handle multiple possible response types in Instructor ---- - -# Working with Union Types in Instructor - -This guide explains how to work with union types in Instructor, allowing you to handle multiple possible response types from language models. Union types are particularly useful when you need the LLM to choose between different response formats or action types. - -!!! note "Union vs. union" - The content from the original `union.md` page has been consolidated into this more comprehensive guide. That page showed a basic example of using Union types for multiple action types. - -## Basic Union Types - -Union types let you specify that a field can be one of several types: - -```python -from typing import Union -from pydantic import BaseModel - - -class Response(BaseModel): - value: Union[str, int] # Can be either string or integer -``` - -## Discriminated Unions - -Use discriminated unions to handle different response types: - -```python -from typing import Literal, Union -from pydantic import BaseModel -import instructor - - -class UserQuery(BaseModel): - type: Literal["user"] - username: str - - -class SystemQuery(BaseModel): - type: Literal["system"] - command: str - - -Query = Union[UserQuery, SystemQuery] - -# Usage with Instructor -client = instructor.from_provider("openai/gpt-4.1-mini") - -response = client.create( - response_model=Query, - messages=[{"role": "user", "content": "Parse: user lookup jsmith"}], -) -``` - -## Optional Fields - -Combine Union with Optional for nullable fields: - -```python -from typing import Optional -from pydantic import BaseModel - - -class User(BaseModel): - name: str - email: Optional[str] = None # Same as Union[str, None] -``` - -## Best Practices - -1. **Type Hints**: Use proper type hints for clarity and better IDE support -2. **Discriminators**: Add discriminator fields (like `type`) for complex unions to help the LLM choose correctly -3. **Validation**: Add validators for union fields to ensure the data is valid -4. **Documentation**: Document expected types clearly in your models with docstrings -5. **Field Names**: Use descriptive field names to guide the model's output -6. **Examples**: Include examples in your Pydantic models to help the LLM understand the expected format - -## Common Patterns - -### Multiple Response Types -```python -from typing import Union, Literal -from pydantic import BaseModel - - -class SuccessResponse(BaseModel): - status: Literal["success"] - data: dict - - -class ErrorResponse(BaseModel): - status: Literal["error"] - message: str - - -Response = Union[SuccessResponse, ErrorResponse] -``` - -### Nested Unions -```python -from typing import Literal, Union, List -from pydantic import BaseModel - - -class TextContent(BaseModel): - type: Literal["text"] - text: str - - -class ImageContent(BaseModel): - type: Literal["image"] - url: str - - -class Message(BaseModel): - content: List[Union[TextContent, ImageContent]] -``` - -## Dynamic Action Selection with Unions - -You can use Union types to write "agents" that dynamically choose actions by selecting an output class. For example, in a search and lookup function: - -```python -from pydantic import BaseModel -from typing import Union - - -class Search(BaseModel): - query: str - - def execute(self): - # Implementation for search - return f"Searching for: {self.query}" - - -class Lookup(BaseModel): - key: str - - def execute(self): - # Implementation for lookup - return f"Looking up key: {self.key}" - - -class Action(BaseModel): - action: Union[Search, Lookup] - - def execute(self): - return self.action.execute() -``` - -With this pattern, the LLM can decide whether to perform a search or a lookup based on the user's input: - -```python -import instructor -from pydantic import BaseModel -from typing import Union - - -class Search(BaseModel): - query: str - - def execute(self): - # Implementation for search - return f"Searching for: {self.query}" - - -class Lookup(BaseModel): - key: str - - def execute(self): - # Implementation for lookup - return f"Looking up key: {self.key}" - - -class Action(BaseModel): - action: Union[Search, Lookup] - - def execute(self): - return self.action.execute() - - -client = instructor.from_provider("openai/gpt-4.1-mini") - -# Let the LLM decide what action to take -result = client.create( - response_model=Action, - messages=[ - { - "role": "system", - "content": "You're an assistant that helps search or lookup information.", - }, - {"role": "user", "content": "Find information about climate change"}, - ], -) - -# Execute the chosen action -print(result.execute()) # Likely outputs: "Searching for: climate change" -#> Searching for: climate change -``` - -## Integration with Instructor - -### import instructor -from typing import Union, Literal -from pydantic import BaseModel - - -class SuccessResponse(BaseModel): - status: Literal["success"] - data: dict - - -class ErrorResponse(BaseModel): - status: Literal["error"] - message: str - - -Response = Union[SuccessResponse, ErrorResponse] - -client = instructor.from_provider("openai/gpt-4.1-mini") - -result = client.create( - response_model=Response, - messages=[ - { - "role": "system", - "content": "You are a helpful assistant that processes requests and returns either a success or error response.", - }, - { - "role": "user", - "content": "Process this request: Get user information for id 123", - }, - ], -) - -# Check the result type -if isinstance(result, ErrorResponse): - print(f"Error: {result.message}") - #> Error: Request not supported: Get user information for id 123 -else: - print(f"Success: {result.data}") -: User information for id 123 is not available. -else: - print(f"Success: {result.data}") -``` - -### Streaming with Unions -```python -def stream_content(): - response = client.create( - response_model=Message, - stream=True, - messages=[{"role": "user", "content": "Generate mixed content"}], - ) - for partial in response: - if partial.content: - for item in partial.content: - if isinstance(item, TextContent): - print(f"Text: {item.text}") - elif isinstance(item, ImageContent): - print(ffrom pydantic import ValidationError, BaseModel -from typing import Union, Literal - - -class SuccessResponse(BaseModel): - status: Literal["success"] - data: dict - - -class ErrorResponse(BaseModel): - status: Literal["error"] - message: str - - -Response = Union[SuccessResponse, ErrorResponse] - -try: - # This will fail because "invalid" is not a valid status - response = SuccessResponse(status="invalid", data={"key": "value"}) -except ValidationError as e: - print(f"Validation error: {e}") - """ - Validation error: 1 validation error for SuccessResponse - status - Input should be 'success' [type=literal_error, input_value='invalid', input_type=str] - """ -id", data={"key": "value"}) -except ValidationError as e: - print(f"Validation error: {e}") - """ - Validation error: 1 validation error for SuccessResponse - status - Input should be 'success' [type=literal_error, input_value='invalid', input_type=str] - """ -``` - -## Type Checking - -Use isinstance() for runtime type checking: - -```python -from typing import Union, Literal -from pydantic import BaseModel - - -class SuccessResponse(BaseModel): - status: Literal["success"] - data: dict - - -class ErrorResponse(BaseModel): - status: Literal["error"] - message: str - - -Response = Union[SuccessResponse, ErrorResponse] - - -def process_response(response: Response): - if isinstance(response, SuccessResponse): - # Handle success case - print(f"Success: {response.data}") - elif isinstance(response, ErrorResponse): - # Handle error case - print(f"Error: {response.message}") -``` - -For more information about union types, check out the [Pydantic documentation on unions](https://docs.pydantic.dev/latest/concepts/types/#unions). - -```from typing import Literal, Union -from pydantic import BaseModel -import instructor -from openai import OpenAI - - -class Action(BaseModel): - """Base action class.""" - - type: str - - -class SendMessage(BaseModel): - type: Literal["send_message"] - message: str - recipient: str - - -class MakePayment(BaseModel): - type: Literal["make_payment"] - amount: float - recipient: str - - -Action = Union[SendMessage, MakePayment] - -# Usage with Instructor -client = instructor.from_provider("openai/gpt-4o") -response = client.create( - response_model=Action, - messages=[{"role": "user", "content": "Send a payment of $50 to John."}], -) - ], -) -``` - -```from typing import Literal, Union -from pydantic import BaseModel -import instructor -from openai import OpenAI - - -class SearchAction(BaseModel): - type: Literal["search"] - query: str - - -class EmailAction(BaseModel): - type: Literal["email"] - to: str - subject: str - body: str - - -Action = Union[SearchAction, EmailAction] - -# The model can choose which action to take -client = instructor.from_provider("openai/gpt-4o") -response = client.create( - response_model=Action, - messages=[{"role": "user", "content": "Find me information about climate change."}], -) - ], -) -``` - -```from typing import Literal, Union -from pydantic import BaseModel -import instructor -from openai import OpenAI - - -class TextResponse(BaseModel): - type: Literal["text"] - content: str - - -class ImageResponse(BaseModel): - type: Literal["image"] - url: str - caption: str - - -Response = Union[TextResponse, ImageResponse] - -# Patched client -``` - -## See Also - -- [Types](./types.md) - Working with different data types in Instructor -- [Enums](./enums.md) - Using enumerated types for structured choices -- [Optional Fields](../learning/patterns/optional_fields.md) - Handling optional data -- [Validation](./validation.md) - Validating union type responses -- [Union Examples](../examples/index.md) - Practical union type examples -client = instructor.from_provider("openai/gpt-4o") -response = client.create( - response_model=Response, - messages=[{"role": "user", "content": "Tell me a joke about programming."}], -) - ], -) -``` - -```from typing import Union -from pydantic import BaseModel - - -class Response(BaseModel): - """A more complex example showing nested Union fields.""" - - result: Union[str, int, float, bool] - bool] -``` - -```from typing import Dict, List, Union, Any -from pydantic import BaseModel - - -class Response(BaseModel): - """A more complex example showing nested Union fields.""" - - data: Dict[str, Union[str, int, List[Any]]] -Any]]] -``` diff --git a/참고/instructor-main/docs/concepts/usage.md b/참고/instructor-main/docs/concepts/usage.md deleted file mode 100644 index 7783196..0000000 --- a/참고/instructor-main/docs/concepts/usage.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -title: Handling Non-Streaming Requests in OpenAI with Usage Tracking -description: Learn how to manage non-streaming requests in OpenAI, track token usage, and handle exceptions with Python. ---- - -## See Also - -- [Getting Started](../getting-started.md) - Quick start guide -- [from_provider Guide](./from_provider.md) - Detailed client configuration -- [Response Models](./models.md) - Working with Pydantic models -- [Raw Response](./raw_response.md) - Access original LLM responses - -The easiest way to get usage for non streaming requests is to access the raw response. - -```python -import instructor - -from pydantic import BaseModel - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class UserExtract(BaseModel): - name: str - age: int - - -user, completion = client.create_with_completion( - response_model=UserExtract, - messages=[ - {"role": "user", "content": "Extract jason is 25 years old"}, - ], -) - -print(completion.usage) -""" -CompletionUsage( - completion_tokens=10, - prompt_tokens=79, - total_tokens=89, - completion_tokens_details=CompletionTokensDetails( - accepted_prediction_tokens=None, - audio_tokens=0, - reasoning_tokens=0, - rejected_prediction_tokens=None, - ), - prompt_tokens_details=PromptTokensDetails(audio_tokens=0, cached_tokens=0), -) -""" -``` - -You can catch an IncompleteOutputException whenever the context length is exceeded and react accordingly, such as by trimming your prompt by the number of exceeding tokens. - -```python -from instructor.core.exceptions import IncompleteOutputException -import instructor -from pydantic import BaseModel - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class UserExtract(BaseModel): - name: str - age: int - - -try: - client.create_with_completion( - response_model=UserExtract, - messages=[ - {"role": "user", "content": "Extract jason is 25 years old"}, - ], - ) -except IncompleteOutputException as e: - token_count = e.last_completion.usage.total_tokens # type: ignore - # your logic here -``` diff --git a/참고/instructor-main/docs/concepts/validation.md b/참고/instructor-main/docs/concepts/validation.md deleted file mode 100644 index 7af9a59..0000000 --- a/참고/instructor-main/docs/concepts/validation.md +++ /dev/null @@ -1,202 +0,0 @@ ---- -title: Validation -description: Learn how to validate LLM outputs with Pydantic for type safety and data consistency. ---- - -# Validation - -Instructor uses Pydantic for validation, providing type checking, data coercion, custom validators, and field constraints. - -## Validation Flow - -```mermaid -flowchart TD - A[Define Pydantic Model] --> B[Send Request to LLM] - B --> C[LLM Generates Response] - C --> D{Validate Response} - - D -->|Valid| E[Return Pydantic Object] - D -->|Invalid| F{Auto-Retry Enabled?} - - F -->|Yes| G[Send Error Context to LLM] - F -->|No| H[Raise ValidationError] - - G --> I[LLM Generates New Response] - I --> J{Validate Again} - - J -->|Valid| E - J -->|Invalid| K{Max Retries Reached?} - - K -->|No| G - K -->|Yes| H -``` - -## Basic Validation - -Define models with type hints and field constraints: - -```python -from typing import List -from pydantic import BaseModel, Field, field_validator - - -class User(BaseModel): - name: str = Field(..., min_length=2, description="User's full name") - age: int = Field(..., ge=0, le=150, description="User's age") - emails: List[str] = Field(description="List of email addresses") - - @field_validator('emails') - @classmethod - def validate_emails(cls, v): - if not all('@' in email for email in v): - raise ValueError('Invalid email format') - return v -``` - -## Field Validation - -Use `Field()` for basic constraints: - -```python -from pydantic import BaseModel, Field - - -class Product(BaseModel): - name: str = Field(..., min_length=1, max_length=100) - price: float = Field(..., gt=0) - quantity: int = Field(..., ge=0) -``` - -## Custom Validators - -Use `@field_validator` for complex validation: - -```python -from pydantic import BaseModel, Field, field_validator - - -class Order(BaseModel): - items: list[str] = Field(description="List of item names") - total: float = Field(description="Total order amount") - - @field_validator('total') - @classmethod - def validate_total(cls, v): - if v < 0: - raise ValueError('Total cannot be negative') - return v -``` - -## Pre-validation Transformation - -Transform data before validation: - -```python -from pydantic import BaseModel, field_validator - - -class UserProfile(BaseModel): - username: str - - @field_validator('username', mode='before') - @classmethod - def lowercase_username(cls, v): - return v.lower() if isinstance(v, str) else v -``` - -## Semantic Validation - -Use `llm_validator` for validations that are hard to express programmatically: - -```python -from typing import Annotated -from pydantic import BaseModel, BeforeValidator -import instructor -from instructor import llm_validator - -client = instructor.from_provider("openai/gpt-4.1-mini") - - -class ContentReview(BaseModel): - title: str - content: Annotated[ - str, - BeforeValidator( - llm_validator( - "Content must be family-friendly and not contain profanity", - client=client, - ) - ), - ] -``` - -Semantic validation works well for content moderation, tone validation, consistency checks, and complex relationships. For more patterns and details, see the [Semantic Validation](./semantic_validation.md) guide. - -## Nested Validation - -Validate nested structures: - -```python -from pydantic import BaseModel, Field - - -class Address(BaseModel): - street: str - city: str - country: str - - -class User(BaseModel): - name: str - addresses: list[Address] = Field(description="User's addresses") -``` - -## Error Handling - -Handle validation failures with appropriate error types: - -```python -import instructor -from pydantic import BaseModel, Field, field_validator - - -class User(BaseModel): - name: str - age: int - - @field_validator('age') - @classmethod - def validate_age(cls, v): - if v < 0: - raise ValueError("Age cannot be negative") - return v - - -client = instructor.from_provider("openai/gpt-4.1-mini") - -try: - user = client.create( - response_model=User, - messages=[ - {"role": "user", "content": "Extract: John, age: -5"}, - ], - ) -except instructor.exceptions.InstructorValidationError as e: - print(f"Validation error: {e}") -``` - -## Best Practices - -1. **Start simple**: Begin with basic type validation before adding complex rules -2. **Use type hints**: Always specify types for clarity -3. **Document constraints**: Add descriptions to Field() definitions -4. **Choose the right validation type**: Rule-based for objective criteria, semantic for subjective -5. **Handle errors**: Implement proper error handling for validation failures -6. **Consider costs**: Semantic validation with LLMs incurs API costs and latency - -## See Also - -- [Semantic Validation](./semantic_validation.md) - LLM-based validation patterns -- [Reask Validation](./reask_validation.md) - Automatic retry with validation feedback -- [Retrying](./retrying.md) - Configure retry behavior -- [Error Handling](./error_handling.md) - Handle validation failures diff --git a/참고/instructor-main/docs/contributing.md b/참고/instructor-main/docs/contributing.md deleted file mode 100644 index 262278a..0000000 --- a/참고/instructor-main/docs/contributing.md +++ /dev/null @@ -1,473 +0,0 @@ ---- -title: Contribute to Instructor: Evals, Issues, and Pull Requests -description: Join us in enhancing the Instructor library with evals, report issues, and submit pull requests on GitHub. Collaborate and contribute! ---- - -# Contributing to Instructor - -We welcome contributions to Instructor! This page covers the different ways you can help improve the library. - -## Ways to Contribute - -### Evaluation Tests (Evals) - -Evals help us monitor the quality of both the OpenAI models and the Instructor library. To contribute: - -1. **Explore Existing Evals**: Check out [our evals directory](https://github.com/instructor-ai/instructor/tree/main/tests/llm/test_openai/evals) -2. **Create a New Eval**: Add new pytest tests that evaluate specific capabilities or edge cases -3. **Follow the Pattern**: Structure your eval similar to existing ones -4. **Submit a PR**: We'll review and incorporate your eval - -Evals are run weekly, and results are tracked to monitor performance over time. - -### Reporting Issues - -If you encounter a bug or problem, please [file an issue on GitHub](https://github.com/instructor-ai/instructor/issues) with: - -1. A clear, descriptive title -2. Detailed information including: - - The `response_model` you're using - - The `messages` you sent - - The `model` you're using - - Steps to reproduce the issue - - Expected vs. actual behavior - - Your environment details (Python version, OS, package versions) - -### Contributing Code - -We welcome pull requests! Here's the process: - -1. **For Small Changes**: Feel free to submit a PR directly -2. **For Larger Changes**: [Start with an issue](https://github.com/instructor-ai/instructor/issues) to discuss approach -3. **Looking for Ideas?** Check issues labeled [help wanted](https://github.com/instructor-ai/instructor/labels/help%20wanted) or [good first issue](https://github.com/instructor-ai/instructor/labels/good%20first%20issue) - -## Setting Up Your Development Environment - -### Using UV (Recommended) - -UV is a fast Python package installer and resolver that makes development easier. - -1. **Install UV** (official method): - ```bash - # macOS/Linux - curl -LsSf https://astral.sh/uv/install.sh | sh - - # Windows PowerShell - powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" - ``` - -2. **Install Project in Development Mode**: - ```bash - # Clone the repository - git clone https://github.com/YOUR-USERNAME/instructor.git - cd instructor - - # Install with development dependencies - uv pip install -e ".[dev,docs]" - ``` - -3. **Adding New Dependencies**: - ```bash - # Add a regular dependency - uv pip install some-package - - # Install a specific version - uv pip install "some-package>=1.0.0,<2.0.0" - ``` - -4. **Common UV Commands**: - ```bash - # Update UV itself - uv self update - - # Create a requirements file - uv pip freeze > requirements.txt - ``` - -### Using Poetry - -Poetry provides comprehensive dependency management and packaging. - -1. **Install Poetry**: - ```bash - curl -sSL https://install.python-poetry.org | python3 - - ``` - -2. **Install Dependencies**: - ```bash - # Clone the repository - git clone https://github.com/YOUR-USERNAME/instructor.git - cd instructor - - # Install with development dependencies - poetry install --with dev,docs - ``` - -3. **Working with Poetry**: - ```bash - # Activate virtual environment - poetry shell - - # Run a command in the virtual environment - poetry run pytest - - # Add a dependency - poetry add package-name - - # Add a development dependency - poetry add --group dev package-name - ``` - -## Adding Support for New LLM Providers - -Instructor uses optional dependencies to support different LLM providers. Provider-specific utilities live in the `instructor/utils` directory. To add a new provider: - -1. **Add Dependencies to pyproject.toml**: - ```toml - [project.optional-dependencies] - # Add your provider - my-provider = ["my-provider-sdk>=1.0.0,<2.0.0"] - - [dependency-groups] - # Mirror in dependency groups - my-provider = ["my-provider-sdk>=1.0.0,<2.0.0"] - ``` - -2. **Create Provider Client**: - - Create a new file at `instructor/clients/client_myprovider.py` - - Implement `from_myprovider` function that patches the provider's client - -3. **Add Tests**: Create tests in `tests/llm/test_myprovider/` - -4. **Document Installation**: - ```bash - # Installation command for your provider - uv pip install "instructor[my-provider]" - # or with poetry - poetry install --with my-provider - ``` - -5. **Create Provider Utilities and Handlers**: - - Add `instructor/utils/myprovider.py` with `reask` and `handle_*` helpers - - Define `MYPROVIDER_HANDLERS` mapping `Mode` values to these functions - -6. **Register the Provider**: - - Update `instructor/utils/providers.py` with your provider enum value - - Extend `get_provider` detection for your base URL - -7. **Update `process_response.py`**: - - Import your handlers and add them to `mode_handlers` - - This script uses the handlers to prepare kwargs and parse results - -8. **Write Documentation**: - - Add a new markdown file in `docs/integrations/` for your provider - - Update `mkdocs.yml` to include your new page - - Make sure to include a complete example - -## Development Workflow - -1. **Fork the Repository**: Create your own fork of the project -2. **Clone and Set Up**: - ```bash - git clone https://github.com/YOUR-USERNAME/instructor.git - cd instructor - git remote add upstream https://github.com/instructor-ai/instructor.git - ``` -3. **Create a Branch**: - ```bash - git checkout -b feature/your-feature-name - ``` -4. **Make Changes, Test, and Commit**: - ```bash - # Run tests - pytest tests/ -k 'not llm and not openai' # Skip LLM tests for faster local dev - - # Commit changes - git add . - git commit -m "Your descriptive commit message" - ``` -5. **Keep Updated and Push**: - ```bash - git fetch upstream - git rebase upstream/main - git push origin feature/your-feature-name - ``` -6. **Create a Pull Request**: Submit your PR with a clear description of changes - -## Utility Scripts - -The `scripts/` directory contains utility scripts that help maintain code quality and documentation. These scripts are integrated into pre-commit hooks and can also be run manually. - -### Available Scripts - -#### `make_clean.py` - Markdown File Cleaner -Cleans markdown files by removing special whitespace characters and replacing em dashes with regular dashes. - -```bash -# Clean all markdown files -python scripts/make_clean.py - -# Preview changes without modifying files -python scripts/make_clean.py --dry-run -``` - -#### `check_blog_excerpts.py` - Blog Post Excerpt Validator -Ensures all blog posts contain the `` tag for proper excerpt handling. - -```bash -# Check all blog posts -python scripts/check_blog_excerpts.py -``` - -#### `make_sitemap.py` - Enhanced Documentation Sitemap Generator -Generates an enhanced sitemap (`sitemap.yaml`) with AI-powered content analysis and cross-link suggestions. - -```bash -# Generate sitemap with default settings -python scripts/make_sitemap.py - -# Customize settings -python scripts/make_sitemap.py \ - --root-dir docs \ - --output-file sitemap.yaml \ - --max-concurrency 10 -``` - -**Requirements for sitemap generation**: -- OpenAI API key (set as `OPENAI_API_KEY` environment variable) -- Additional dependencies: `openai`, `typer`, `rich`, `tenacity`, `pyyaml` - -### Pre-commit Integration - -These scripts run automatically during the commit process: - -- **Markdown cleaning**: Runs on commits with markdown files in `docs/` -- **Blog excerpt validation**: Runs on commits with blog post files - -### Manual Usage - -You can run scripts manually for testing or one-time operations: - -```bash -# Test markdown cleaning -python scripts/make_clean.py --dry-run - -# Check blog excerpts -python scripts/check_blog_excerpts.py - -# Generate fresh sitemap -python scripts/make_sitemap.py -``` - -For detailed documentation on each script, see the `scripts/README.md` file in the project repository. - -## Using Cursor to Build PRs - -[Cursor](https://cursor.sh) is an AI-powered code editor that can help you contribute to Instructor. - -1. **Getting Started with Cursor**: - - Download Cursor from [cursor.sh](https://cursor.sh) - - Open the Instructor project in Cursor - - Cursor will automatically detect our rules in `.cursor/rules/` - -2. **Using Cursor Rules**: - - `new-features-planning`: Helps plan and structure new features - - `simple-language`: Guidelines for writing clear documentation - - `documentation-sync`: Ensures documentation stays in sync with code changes - -3. **Creating PRs with Cursor**: - - Use Cursor's Git integration to create a new branch - - Make your changes with AI assistance - - Create a PR with: - ```bash - # Use GitHub CLI to create the PR - gh pr create -t "Your feature title" -b "Description of your changes" -r jxnl,ivanleomk - ``` - - Add `This PR was written by [Cursor](https://cursor.sh)` to your PR description - -4. **Benefits of Using Cursor**: - - AI helps generate code that follows our style guidelines - - Simplifies PR creation process - - Helps maintain documentation standards - -## Code Style Guidelines - -We use the following tools to maintain code quality: - -- **Ruff**: For linting and formatting -- **ty**: For type checking -- **Pre-commit**: For automatic checks before committing - -```bash -# Install pre-commit hooks -pip install pre-commit -pre-commit install -``` - -Key style guidelines: -- Use strict typing -- Follow import order: standard lib → third-party → local -- Use snake_case for functions/variables, PascalCase for classes -- Write comprehensive docstrings for public API functions - -### Conventional Comments - -When reviewing code or writing commit messages, we use conventional comments to make feedback clearer: - -``` -