from __future__ import annotations import argparse from dataclasses import asdict import json from sqlalchemy import select from crawler_platform.app.config.loader import load_project_config from crawler_platform.app.core.crawler.pipeline import CrawlPipeline from crawler_platform.app.core.database import models from crawler_platform.app.core.database.repository import KnowledgeRepository from crawler_platform.app.core.database.session import init_db, session_scope from crawler_platform.app.core.extractor.factory import extractor_for_domain from crawler_platform.app.core.ontology.definitions import ontology_for_domain from crawler_platform.app.core.ontology.mapper import ontology_to_dict from crawler_platform.app.core.recommendation.scorer import PreferenceInput, RuleBasedRecommender def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Ontology crawler platform admin CLI") parser.add_argument("--db", default="sqlite:///crawler_platform.db", help="SQLAlchemy database URL") sub = parser.add_subparsers(dest="command", required=True) sub.add_parser("init-db") create_project = sub.add_parser("create-project") create_project.add_argument("--config", required=True) ontology = sub.add_parser("ontology") ontology.add_argument("--domain", required=True) crawl = sub.add_parser("crawl-url") crawl.add_argument("--config", required=True) crawl.add_argument("--source", required=True) crawl.add_argument("--url", required=True) crawl.add_argument("--extractor-provider", default="rule_based", choices=["rule_based", "openai", "ollama", "lm_studio"]) crawl.add_argument("--extractor-model") crawl.add_argument("--extractor-base-url") claims = sub.add_parser("claims") claims.add_argument("--project", required=True) claims.add_argument("--limit", type=int, default=20) entities = sub.add_parser("entities") entities.add_argument("--project", required=True) entities.add_argument("--type") entities.add_argument("--limit", type=int, default=20) recommend = sub.add_parser("recommend") recommend.add_argument("--project", required=True) recommend.add_argument("--target-type", default="Perfume") recommend.add_argument("--preferences-json", default="{}") recommend.add_argument("--limit", type=int, default=10) return parser def main() -> None: args = build_parser().parse_args() if args.command == "init-db": init_db(args.db) print(f"initialized database: {args.db}") return if args.command == "ontology": print(json.dumps(ontology_to_dict(ontology_for_domain(args.domain)), ensure_ascii=False, indent=2)) return if args.command == "create-project": config = load_project_config(args.config) with session_scope(args.db) as session: project = KnowledgeRepository(session).upsert_project(config) print(json.dumps({"project_id": project.id, "name": project.name, "domain": project.domain}, ensure_ascii=False)) return if args.command == "crawl-url": config = load_project_config(args.config) with session_scope(args.db) as session: repo = KnowledgeRepository(session) pipeline = CrawlPipeline( repo, extractor_for_domain( config.domain, provider=args.extractor_provider, model=args.extractor_model, base_url=args.extractor_base_url, ), ) result = pipeline.crawl_url(config, args.source, args.url) print(json.dumps(asdict(result), ensure_ascii=False)) return if args.command == "claims": with session_scope(args.db) as session: project = KnowledgeRepository(session).get_project(args.project) rows = session.execute( select(models.Claim, models.Entity) .join(models.Entity, models.Claim.subject_entity_id == models.Entity.id) .where(models.Claim.project_id == project.id) .limit(args.limit) ).all() print( json.dumps( [ { "claim_id": claim.id, "subject": subject.name, "predicate": claim.predicate, "object_entity_id": claim.object_entity_id, "object_value": claim.object_value, "confidence": claim.confidence, } for claim, subject in rows ], ensure_ascii=False, indent=2, ) ) return if args.command == "entities": with session_scope(args.db) as session: project = KnowledgeRepository(session).get_project(args.project) query = select(models.Entity).where(models.Entity.project_id == project.id) if args.type: query = query.where(models.Entity.entity_type == args.type) result = session.scalars(query.limit(args.limit)).all() print( json.dumps( [{"id": entity.id, "type": entity.entity_type, "name": entity.name} for entity in result], ensure_ascii=False, indent=2, ) ) return if args.command == "recommend": preference = PreferenceInput(**json.loads(args.preferences_json)) with session_scope(args.db) as session: project = KnowledgeRepository(session).get_project(args.project) result = RuleBasedRecommender(session).recommend(project.id, args.target_type, preference, args.limit) print(json.dumps([asdict(item) for item in result], ensure_ascii=False, indent=2)) if __name__ == "__main__": main()