141 lines
5.9 KiB
Python
141 lines
5.9 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
from dataclasses import asdict
|
|
import json
|
|
|
|
from sqlalchemy import select
|
|
|
|
from crawler_platform.app.config.loader import load_project_config
|
|
from crawler_platform.app.core.crawler.pipeline import CrawlPipeline
|
|
from crawler_platform.app.core.database import models
|
|
from crawler_platform.app.core.database.repository import KnowledgeRepository
|
|
from crawler_platform.app.core.database.session import init_db, session_scope
|
|
from crawler_platform.app.core.extractor.factory import extractor_for_domain
|
|
from crawler_platform.app.core.ontology.definitions import ontology_for_domain
|
|
from crawler_platform.app.core.ontology.mapper import ontology_to_dict
|
|
from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(description="Ontology crawler platform admin CLI")
|
|
parser.add_argument("--db", default="sqlite:///crawler_platform.db", help="SQLAlchemy database URL")
|
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
|
|
sub.add_parser("init-db")
|
|
|
|
create_project = sub.add_parser("create-project")
|
|
create_project.add_argument("--config", required=True)
|
|
|
|
ontology = sub.add_parser("ontology")
|
|
ontology.add_argument("--domain", required=True)
|
|
|
|
crawl = sub.add_parser("crawl-url")
|
|
crawl.add_argument("--config", required=True)
|
|
crawl.add_argument("--source", required=True)
|
|
crawl.add_argument("--url", required=True)
|
|
crawl.add_argument("--extractor-provider", default="rule_based", choices=["rule_based", "openai", "ollama", "lm_studio"])
|
|
crawl.add_argument("--extractor-model")
|
|
crawl.add_argument("--extractor-base-url")
|
|
|
|
claims = sub.add_parser("claims")
|
|
claims.add_argument("--project", required=True)
|
|
claims.add_argument("--limit", type=int, default=20)
|
|
|
|
entities = sub.add_parser("entities")
|
|
entities.add_argument("--project", required=True)
|
|
entities.add_argument("--type")
|
|
entities.add_argument("--limit", type=int, default=20)
|
|
|
|
recommend = sub.add_parser("recommend")
|
|
recommend.add_argument("--project", required=True)
|
|
recommend.add_argument("--target-type", default="Perfume")
|
|
recommend.add_argument("--preferences-json", default="{}")
|
|
recommend.add_argument("--limit", type=int, default=10)
|
|
return parser
|
|
|
|
|
|
def main() -> None:
|
|
args = build_parser().parse_args()
|
|
if args.command == "init-db":
|
|
init_db(args.db)
|
|
print(f"initialized database: {args.db}")
|
|
return
|
|
if args.command == "ontology":
|
|
print(json.dumps(ontology_to_dict(ontology_for_domain(args.domain)), ensure_ascii=False, indent=2))
|
|
return
|
|
if args.command == "create-project":
|
|
config = load_project_config(args.config)
|
|
with session_scope(args.db) as session:
|
|
project = KnowledgeRepository(session).upsert_project(config)
|
|
print(json.dumps({"project_id": project.id, "name": project.name, "domain": project.domain}, ensure_ascii=False))
|
|
return
|
|
if args.command == "crawl-url":
|
|
config = load_project_config(args.config)
|
|
with session_scope(args.db) as session:
|
|
repo = KnowledgeRepository(session)
|
|
pipeline = CrawlPipeline(
|
|
repo,
|
|
extractor_for_domain(
|
|
config.domain,
|
|
provider=args.extractor_provider,
|
|
model=args.extractor_model,
|
|
base_url=args.extractor_base_url,
|
|
),
|
|
)
|
|
result = pipeline.crawl_url(config, args.source, args.url)
|
|
print(json.dumps(asdict(result), ensure_ascii=False))
|
|
return
|
|
if args.command == "claims":
|
|
with session_scope(args.db) as session:
|
|
project = KnowledgeRepository(session).get_project(args.project)
|
|
rows = session.execute(
|
|
select(models.Claim, models.Entity)
|
|
.join(models.Entity, models.Claim.subject_entity_id == models.Entity.id)
|
|
.where(models.Claim.project_id == project.id)
|
|
.limit(args.limit)
|
|
).all()
|
|
print(
|
|
json.dumps(
|
|
[
|
|
{
|
|
"claim_id": claim.id,
|
|
"subject": subject.name,
|
|
"predicate": claim.predicate,
|
|
"object_entity_id": claim.object_entity_id,
|
|
"object_value": claim.object_value,
|
|
"confidence": claim.confidence,
|
|
}
|
|
for claim, subject in rows
|
|
],
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
return
|
|
if args.command == "entities":
|
|
with session_scope(args.db) as session:
|
|
project = KnowledgeRepository(session).get_project(args.project)
|
|
query = select(models.Entity).where(models.Entity.project_id == project.id)
|
|
if args.type:
|
|
query = query.where(models.Entity.entity_type == args.type)
|
|
result = session.scalars(query.limit(args.limit)).all()
|
|
print(
|
|
json.dumps(
|
|
[{"id": entity.id, "type": entity.entity_type, "name": entity.name} for entity in result],
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
return
|
|
if args.command == "recommend":
|
|
preference = PreferenceInput(**json.loads(args.preferences_json))
|
|
with session_scope(args.db) as session:
|
|
project = KnowledgeRepository(session).get_project(args.project)
|
|
result = RuleBasedRecommender(session).recommend(project.id, args.target_type, preference, args.limit)
|
|
print(json.dumps([asdict(item) for item in result], ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|