lasta/AI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[버그수정]

Browse Source
This commit is contained in:
LASTA_DEV01\lasta
2026-05-11 13:02:11 +09:00
parent 5935c6cbe8
commit 3663734781
13 changed files with 686 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
crawler_platform/app/core/database/repository.py
View File

@@ -33,9 +33,10 @@ class KnowledgeRepository:
self.session.add(project)
self.session.flush()
else:
project.domain = config.domain
project.config = config_dict
project.updated_at = models.utcnow()
if project.domain != config.domain or project.config != config_dict:
project.domain = config.domain
project.config = config_dict
project.updated_at = models.utcnow()
for source_config in config.sources:
self.upsert_source(project, source_config)
return project
@@ -51,12 +52,20 @@ class KnowledgeRepository:
source = models.Source(project_id=project.id, name=source_config.name)
self.session.add(source)
self.session.flush()
source.type = source_config.type
source.base_url = source_config.base_url
source.trust_level = source_config.trust_level
source.respect_robots_txt = source_config.respect_robots_txt
source.rate_limit_per_minute = source_config.rate_limit_per_minute
source.updated_at = models.utcnow()
changed = (
source.type != source_config.type
or source.base_url != source_config.base_url
or source.trust_level != source_config.trust_level
or source.respect_robots_txt != source_config.respect_robots_txt
or source.rate_limit_per_minute != source_config.rate_limit_per_minute
)
if changed:
source.type = source_config.type
source.base_url = source_config.base_url
source.trust_level = source_config.trust_level
source.respect_robots_txt = source_config.respect_robots_txt
source.rate_limit_per_minute = source_config.rate_limit_per_minute
source.updated_at = models.utcnow()
return source
def get_project(self, project_name: str) -> models.Project:

22
crawler_platform/app/core/database/session.py
View File

@@ -3,15 +3,28 @@ from __future__ import annotations
from contextlib import contextmanager
from typing import Iterator
from sqlalchemy import create_engine
from sqlalchemy import create_engine, event
from sqlalchemy.orm import Session, sessionmaker
from crawler_platform.app.core.database.models import Base
def make_engine(database_url: str = "sqlite:///crawler_platform.db"):
connect_args = {"check_same_thread": False} if database_url.startswith("sqlite") else {}
return create_engine(database_url, future=True, connect_args=connect_args)
connect_args = {"check_same_thread": False, "timeout": 60} if database_url.startswith("sqlite") else {}
engine = create_engine(database_url, future=True, connect_args=connect_args)
if database_url.startswith("sqlite"):
install_sqlite_pragmas(engine)
return engine
def install_sqlite_pragmas(engine) -> None:
@event.listens_for(engine, "connect")
def _set_sqlite_pragmas(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA busy_timeout=60000")
cursor.execute("PRAGMA journal_mode=WAL")
cursor.execute("PRAGMA synchronous=NORMAL")
cursor.close()
def init_db(database_url: str = "sqlite:///crawler_platform.db") -> None:
@@ -21,7 +34,7 @@ def init_db(database_url: str = "sqlite:///crawler_platform.db") -> None:
def make_session_factory(database_url: str = "sqlite:///crawler_platform.db") -> sessionmaker[Session]:
engine = make_engine(database_url)
return sessionmaker(bind=engine, expire_on_commit=False, class_=Session, future=True)
return sessionmaker(bind=engine, expire_on_commit=False, autoflush=False, class_=Session, future=True)
@contextmanager
@@ -36,4 +49,3 @@ def session_scope(database_url: str = "sqlite:///crawler_platform.db") -> Iterat
raise
finally:
session.close()