[crawler_platform 삭제]
This commit is contained in:
LASTA_DEV01\lasta
140
ontology_platform/crawler_platform/app/cli/main.py
Normal file
140
ontology_platform/crawler_platform/app/cli/main.py
Normal file
@@ -0,0 +1,140 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import asdict
|
||||
import json
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from crawler_platform.app.config.loader import load_project_config
|
||||
from crawler_platform.app.core.crawler.pipeline import CrawlPipeline
|
||||
from crawler_platform.app.core.database import models
|
||||
from crawler_platform.app.core.database.repository import KnowledgeRepository
|
||||
from crawler_platform.app.core.database.session import init_db, session_scope
|
||||
from crawler_platform.app.core.extractor.factory import extractor_for_domain
|
||||
from crawler_platform.app.core.ontology.definitions import ontology_for_domain
|
||||
from crawler_platform.app.core.ontology.mapper import ontology_to_dict
|
||||
from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Ontology crawler platform admin CLI")
|
||||
parser.add_argument("--db", default="sqlite:///crawler_platform.db", help="SQLAlchemy database URL")
|
||||
sub = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
sub.add_parser("init-db")
|
||||
|
||||
create_project = sub.add_parser("create-project")
|
||||
create_project.add_argument("--config", required=True)
|
||||
|
||||
ontology = sub.add_parser("ontology")
|
||||
ontology.add_argument("--domain", required=True)
|
||||
|
||||
crawl = sub.add_parser("crawl-url")
|
||||
crawl.add_argument("--config", required=True)
|
||||
crawl.add_argument("--source", required=True)
|
||||
crawl.add_argument("--url", required=True)
|
||||
crawl.add_argument("--extractor-provider", default="rule_based", choices=["rule_based", "openai", "ollama", "lm_studio"])
|
||||
crawl.add_argument("--extractor-model")
|
||||
crawl.add_argument("--extractor-base-url")
|
||||
|
||||
claims = sub.add_parser("claims")
|
||||
claims.add_argument("--project", required=True)
|
||||
claims.add_argument("--limit", type=int, default=20)
|
||||
|
||||
entities = sub.add_parser("entities")
|
||||
entities.add_argument("--project", required=True)
|
||||
entities.add_argument("--type")
|
||||
entities.add_argument("--limit", type=int, default=20)
|
||||
|
||||
recommend = sub.add_parser("recommend")
|
||||
recommend.add_argument("--project", required=True)
|
||||
recommend.add_argument("--target-type", default="Perfume")
|
||||
recommend.add_argument("--preferences-json", default="{}")
|
||||
recommend.add_argument("--limit", type=int, default=10)
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = build_parser().parse_args()
|
||||
if args.command == "init-db":
|
||||
init_db(args.db)
|
||||
print(f"initialized database: {args.db}")
|
||||
return
|
||||
if args.command == "ontology":
|
||||
print(json.dumps(ontology_to_dict(ontology_for_domain(args.domain)), ensure_ascii=False, indent=2))
|
||||
return
|
||||
if args.command == "create-project":
|
||||
config = load_project_config(args.config)
|
||||
with session_scope(args.db) as session:
|
||||
project = KnowledgeRepository(session).upsert_project(config)
|
||||
print(json.dumps({"project_id": project.id, "name": project.name, "domain": project.domain}, ensure_ascii=False))
|
||||
return
|
||||
if args.command == "crawl-url":
|
||||
config = load_project_config(args.config)
|
||||
with session_scope(args.db) as session:
|
||||
repo = KnowledgeRepository(session)
|
||||
pipeline = CrawlPipeline(
|
||||
repo,
|
||||
extractor_for_domain(
|
||||
config.domain,
|
||||
provider=args.extractor_provider,
|
||||
model=args.extractor_model,
|
||||
base_url=args.extractor_base_url,
|
||||
),
|
||||
)
|
||||
result = pipeline.crawl_url(config, args.source, args.url)
|
||||
print(json.dumps(asdict(result), ensure_ascii=False))
|
||||
return
|
||||
if args.command == "claims":
|
||||
with session_scope(args.db) as session:
|
||||
project = KnowledgeRepository(session).get_project(args.project)
|
||||
rows = session.execute(
|
||||
select(models.Claim, models.Entity)
|
||||
.join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
|
||||
.where(models.Claim.project_id == project.id)
|
||||
.limit(args.limit)
|
||||
).all()
|
||||
print(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"claim_id": claim.id,
|
||||
"subject": subject.name,
|
||||
"predicate": claim.predicate,
|
||||
"object_entity_id": claim.object_entity_id,
|
||||
"object_value": claim.object_value,
|
||||
"confidence": claim.confidence,
|
||||
}
|
||||
for claim, subject in rows
|
||||
],
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return
|
||||
if args.command == "entities":
|
||||
with session_scope(args.db) as session:
|
||||
project = KnowledgeRepository(session).get_project(args.project)
|
||||
query = select(models.Entity).where(models.Entity.project_id == project.id)
|
||||
if args.type:
|
||||
query = query.where(models.Entity.entity_type == args.type)
|
||||
result = session.scalars(query.limit(args.limit)).all()
|
||||
print(
|
||||
json.dumps(
|
||||
[{"id": entity.id, "type": entity.entity_type, "name": entity.name} for entity in result],
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return
|
||||
if args.command == "recommend":
|
||||
preference = PreferenceInput(**json.loads(args.preferences_json))
|
||||
with session_scope(args.db) as session:
|
||||
project = KnowledgeRepository(session).get_project(args.project)
|
||||
result = RuleBasedRecommender(session).recommend(project.id, args.target_type, preference, args.limit)
|
||||
print(json.dumps([asdict(item) for item in result], ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user