From 3663734781d24bc057b67b952706c5b180277ff5 Mon Sep 17 00:00:00 2001 From: "LASTA_DEV01\\lasta" Date: Mon, 11 May 2026 13:02:11 +0900 Subject: [PATCH] =?UTF-8?q?[=EB=B2=84=EA=B7=B8=EC=88=98=EC=A0=95]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 + README.md | 40 ++++ crawler_platform.db | Bin 331776 -> 335872 bytes crawler_platform/app/api/routes.py | 36 +++ crawler_platform/app/core/crawler/pipeline.py | 8 +- .../app/core/crawler/site_crawler.py | 224 ++++++++++++++++++ .../app/core/database/repository.py | 27 ++- crawler_platform/app/core/database/session.py | 22 +- .../app/domains/perfume/extractor.py | 157 +++++++++++- crawler_platform/app/web/static/app.js | 222 +++++++++-------- crawler_platform/app/web/static/index.html | 45 ++-- crawler_platform/app/web/static/styles.css | 26 +- tests/test_site_crawler.py | 11 + 13 files changed, 686 insertions(+), 135 deletions(-) create mode 100644 crawler_platform/app/core/crawler/site_crawler.py create mode 100644 tests/test_site_crawler.py diff --git a/.gitignore b/.gitignore index e0fe3e4..cd9d6b2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ __pycache__/ *.py[cod] .pytest_cache/ crawler_platform.db +crawler_platform.db-* *.sqlite *.sqlite3 +*.sqlite-* +*.sqlite3-* uvicorn.*.log diff --git a/README.md b/README.md index 6229077..a492b47 100644 --- a/README.md +++ b/README.md @@ -214,6 +214,7 @@ http://127.0.0.1:8000/ - `POST /projects` - `GET /ontology/{domain}` - `POST /crawl` +- `POST /crawl-site` - `GET /projects/{project_name}/entities` - `GET /projects/{project_name}/claims` - `PATCH /claims/{claim_id}/confidence` @@ -221,6 +222,45 @@ http://127.0.0.1:8000/ - `GET /projects/{project_name}/recommendation-tags` - `POST /recommend` +## 사이트 순회 수집 + +단일 상품 URL뿐 아니라 Seed URL에서 시작해 같은 도메인의 링크를 따라가는 수집도 지원합니다. + +```text +Seed URL +→ robots 확인 +→ 링크 추출 +→ same-domain 필터 +→ URL queue 저장 +→ depth / max pages 제한 +→ 각 페이지 fetch +→ 상품/브랜드/리뷰 페이지 판별 +→ 분석 +→ DB 저장 +→ 다음 링크 반복 +``` + +웹 UI에서는 `Crawl site from seed`를 사용합니다. + +API 예시: + +```bash +curl -X POST http://127.0.0.1:8000/crawl-site \ + -H "Content-Type: application/json" \ + -d '{ + "config_path": "configs/perfume_subscription.yaml", + "source_name": "official_brand_site", + "url": "https://example-brand.com", + "extractor_provider": "rule_based", + "max_depth": 2, + "max_pages": 50, + "same_domain_only": true, + "analyze_page_types": ["product", "brand", "review"] + }' +``` + +주의: 검색 결과 페이지나 robots가 막는 페이지는 수집하지 않습니다. 그런 데이터는 공식 API Provider로 붙이는 방식이 맞습니다. + ## 향수 도메인 MVP 기본 엔티티: diff --git a/crawler_platform.db b/crawler_platform.db index a14e2b287cf69c4d6dd6cda8bc5f6e6a34c47f5b..033f4c27adf8e33abffd03c1228ac5fcbf9fd1e7 100644 GIT binary patch delta 16599 zcmeG@eS8$vwez)`>}F?fBHxCPNucr}konlz*$HY45GsfeKtT}1{g{Q6kYEz9$irn3 z5y1~M!k_|1rG8?G23e^79$2+nt)jJ7Yg_B96)n3#?6c}q{PE76*-dsg8%PA7|2%%o z-OW97&OP_sbIv{Yp0jIO`mQhgZmC=l#ws37@(Pk5dBzCJm@bgElf}-m(P`bP==Qj?BAAe7 zWxCosgl*F=$UqR>9x3|>81i5>Ka_=8QQYeU(z(~UUvqWbK=u{3i7j>g&b7idB=6O{ zGH0};l?hO7>l+GkOC$ZbJi^hhu&^LfP+56deSNU1R$t_+YtpTcRjZA+@|c0)%E^3# z-n6`-P7m^R_5AWipTAb;^}42-rW(DGuW#T3wZ59AjiswI*b~ZS&sbHERIk!KBr=c% zE(#M|RO=0)J_tdf*~_qPQoY~$qY*fUiOwt(pcH8 z%38+Krb3MHNM>~JFy0->jLN|9bUOxy7-L2;qXbLgl4GA5qdFJQWgJu5E3BD0fc2+Q z%&=ZzO|t{mnDZGvV@^u9bHb43fj49d;WP+!H;kB@E`w7e%%+{7-bFk&20tz{@!3M4X+?5V1Mf1^u`=h579c1|$Fi9H2+pFr=UIQMrn2R(r{pq1!e zB%ng{9`^;J9cW@3zvdyXvcLu|g2dh-;TBL|8<;Bzmf8ixb%0WgxH zSv{r`Y4+cL^SBfk<)jN82?t{h7$;C125duh=0O?_BeBdzN?nWztQ|$D;d{k#n3X26i{2=0gV=J7Q=D$+aCiSS92*2NBLe zxg**N@(1UJHwCGD!r@SfaS@iPY_2CF6Bai$Eo&?(nX&T z2nZpcqy)ua!0YpBx)clr!o({Xs$zO>4`{O_CMA{mIbrJbX;u8BnUz1CSvhS&mGfxZ zbM0I1iMK@JJNEPO2U^>={vy8px!42S`Pd`3#+rAo^VO(5b>6EvE3cS*X};ry2c_7 z^b%xY&<)#Szj)5C*^RJGq!dwrWU_%bNiNU zv5j~0@vZygE!*K^Z|v^7W6dof_m-pkZ{cBDNB6hlX~myf7r(bPJ9$`_*rwg_9ozWW z(=G9hyV^JIPoMI}U7Lbra?Qg(#-gXJdDselQanfIVkiGTi;JGJ=HVY>(Noqu{C~FS z&hdI<6F;fJx2VBa7to#Y8zb#^Z3xF6zsXtdyh=J>A@1R^BZvt#!6`d)drDU%)USc>gIb1j~ zUfj6T($!eY1|Ce2Z@=xi_U8Sk8&eytbRE+pWF0eeit!tF#cv63=3}jo;R%Apbxd$k zmib9p$5Px`$Lg40Qk({MLg&R=?*^0ZnA@KH*#BSyO^hrG*<~wgqrF|{DSNVxojK3g zV{6(swencJ;Q3lMfP+YmCn@1>2h1YaZU*c!S!?Oqx9o?7Y>(`YJq$k4(j6spxUi)> z+Gwro{KW3&2Rl)Us)8E?7s`WR+nXPaw}4-^#*(@WW{s7$kl_yfb$gT2On97CL*bNB ztFLw@JrH)otvh0yw&5)W{A+yu-SI~v?OQ-u?^_2PIja}fH`Vhq^%XVxbU^~ePx;nr90ZW)2!W2EKS2=iG@5oU<50=0G^z6##|%dkc!$QaP|YBC z&yn>#i{`TgdKNtkG;5G?{|MR^KGO(*n#e?0$M>evClN?k+?^oM3G`K4!J(KL_7qSw z?QmoT6`3IO*ou}A=r?EyD&qdmt>;ws1lt1h+rWnBfm(FC;M~5?4Qp z$0JG=jx>}$W1OeQxe&WjM;Q!bT>A!DI4nDL1*m(lV{~9OFrIpY>>JMhrW;sS?D)-M z%sR-wG7zY52B z>XM->*>Mx75}p;EoE7G^Lsw3TCt^H@={ikuymh!+aflv|=?6{g>C_FX8{M#+pd@OF zWNnV%jgG;!Lkl{N_M&|76tAKxG!pr^6Wm+ye9Aovz7qdy8`-jwTinNv4V6qa)BL{B zHxztHs!UOT=p1f!rq~vjz)v+BHrdz7J?Ap=|3-dLUSP-i;OGQU7SsPCz_pD*4+G0z zpcl~v=wtLpRD))q2pVF2f@67!o6gZCWG8YyqKA?xOripMIR3knKMo^wt`eJ$?=mj< zjNIp8>5>$VEQY=%(6?}P^BLSBncfyW`)jbN97G2apAP^-E0OWx9+Ej1V8+OOWGCa;!;?HOa9iIo4&o zil}lL$l)uY5O@bM%nv9%)$n7&^{7)H~XK|M7 zs37wmVh#yy%%Kxy55Ady*KuY9#r7w-?QDN`m1{@d3(jNC{thL#I%kRfX4@8~jCuqe zq{fo7h}F#n`7@%c4^gA2@QpUEKTMD$i-?7V5jH3ClR1N6rjY^)Yhrb;BD`9-;A`;J z1~s_q^66fms4J=x3I;`=;!%8><_(HqzaB+^D|}T^ydFX8D};nVQ1HsCPgNC7*L-qN zF>Zf{DKefoPNNY$O)(ft7tuc{_XV12R_N)AlIrxkhx9hJvNiys#$qcW`whZk8{Q$d zPP5|bmeUApmep7^A7W()O9;eA0fGG z;*ECI>WD9|ufgwOp6)H4-`RKTKIS|_c!{|!J<(zA$JxikJGC+G73#p)bT088n;io~ zAl-Fv2Gf>ENhbNnyKQzHfEYVwGEZi(bxH)$h`&lbmVtry*X%=Ji1GYu)GsqI@V1&A z14E4aen+he-$+wfbsdgOi~qj%j+Ipp21IYjC;9_kuj2Eox+kEkflx^BNwVy5ha|ru z_`Q;@2qTlBh6pt_D33?>A;#NbNUkrFXgEUc5L%xtEg+kyUJzhNsxmm9- z=s&yGo{)&3DH$4@ln93ORwC9(v5`QH=vVA-(P&qQ+sTeI|61!R=H|G*a6M^%pWB&t zXU>;azoRAhLRT@n*6|63I-koa$yw}rGG~t!MOmTL;QX8H;Yl_}ge5^#UatpaCTp^w z`@L?zKcqmKL<~qF9~^yxx~4%+1QIDepQK7&N%99V&@F)URj;?iBBO_b0XgKCd?0_# z69VrV3I!xN6bOVARnqj3Pxh&DKvLYg5O9awfk0SwE3z64YO1cOA(6M}gj7-T$_ngl zUY{f@*Ps45+*tY&7We zh+atyxJAhw3iveGBvm+n>HdH(EW>*EKtTk>9njQ3Pz`zEzv72vQcxaZ(NV>K4$=w9 zst#rZoQ33&6jBrk^5X$5=+#1sPxV1!DIj|Tz_%=^x}ZS{Ne=05uPhWtL z;PH#1PZPzU8WhBk=nlGrUIEM$Hs_!W?1$W%>IcDxU^pL^zpN=CN!11e9pix!^u*|E z?@{yUaBnvMYyZ=JHPKA5to4UFtstW6(qlX6F#h!DM2EFe? zipi(p9>K_Ip(j9Wd5#ruUbBbZ#iw5{ST_cVu-1%}jbSbHQ2H7)5AgEM*w$t*!ErHV z=zHkPjRO_*rIRT1F^>H|?uh-%P@mNJpFM%fxCQK+=mPF%=Bc0$Vz_Bs5y#&EtsolO?%o)2Hi^}OeO^2fi zXZ18DnKO1YCY5Y{CdJK^p5D(y;by)0Ech3bo$u>jWM?8AXR8>a^)J-K=*iLast%vmB5bhsWV6m;2wMks9YhP^lJN>u9X(H=hjh4{k;du@`Xb}mlhkDJ8(-qv z-fz1(rtgJ=`-5mEIHm1qDqMnWMem}`aERXs$MA^;DI(=zAe$hOTb1eR99`Du5@mGqWNr!ovaXPOEH7HNP)$}94 z-VU@4HN#!aEl?n`8p=ejh0KKxQ=f}w0b>s9I)LX));NBG-7bj^27s0!t_twXuSzL9`HE5u(@T!1SmeY_m7yJAVtSh9%Lemv17 zj)cjO7#|X>8!Yq6>C22Or#C>ap0>8)=C=Z~My^|0iE&seQD(zuRPgXUso)9stb~V8E8(f)&sqgf*epKcdr-o|r&n|E z9n}?lIp5o2j-`-hSqK^EMt)RHU7&V(5Gre{tUBE6a>qFZ!_0aMyCR+#=4;E#R)?WQ&d1GFxS5BWxwyF!H*;_^8#hn2z)ZVY!Sn4@zM(;5h*y&v8g`{{ynK??IaT zcTnwc2>lAuCci{}?rrW?P6p^hXg-wgujj_VdppoIZCR}1lg`baQQ+yR}hs?um(OldPz7n^tIk?T6 zja%mxxOH5P+uT{W&6$Z?`wZOLs&UIq$1Ob#x0JQ6_-o;%xIK3YZU;@qt?QEL{5R;C zq!}1-Ru;uLaG3t-mFA5NLG*>s3V41B&&BZIJ!cX;m%uX_o+M0lzoJ5dr{`6j)g*a3f%&hH54cTwkY`oEn88ws+Xb$4l9yQ_x# zIDMV*#5>Fw<8LQvHa(IN6hV`t@S_Qtv6Iek2g{CYjiOW3AIALe5H$Hrx z8fFL#<23RPQ?3kBkgM=uQb(Mf05fMU!4FNMzwOT~XR-mR3z?lM032UIYDks$@yyng zev=Z;gGV17#$1~TfaH)jdj3e}1$)L2IOV=iU@o?G_M0QP1x6|wEG5c`aP0S&t4?T%4q~4>pP*+ks@dU9F)>{k<3sOSQlI}u;bug9SK3a}UnVlL-r-4RYks3>1NW-Q$| z&0zE6&K(3r_9IIm{Wr56_W;}HleMuI`{L2jS2t6eWM4oEgafB{Ow|A3+ z=+qkKfZgW&f^dEjbuMF$ItOlaFqYRCZf%?uBF^$}USeEy1=E&JLQn-w?btQHCY)bK zookq5K*MIUXbiPzj7!qEmSOgsiiWj;Z)VI5+-}h*wrD8E$*s(F#>rVsc?Jcqpm{sE zu>4<(<^OUr{NKQb!T+_qN}ZCz=Y;d~sI!&%1m@w&wJ4NXJY1NZ$JF0a`%cA0JK=1P zIu9`)K?srWuxQ8@jY{LQKQiYU*S|tNa(W7HS`?gJDcp?bG1d@X=@NfvP&v{ej$?%L zSk(C;^ABJI1lb~!M(+?r6Vt!+12!hFv{^3i; z#SGk7KvtGD{?6laj__l^b|*s28`>*)FGzPifn@MD`mD)7(h0Ps;97FvJiqW%`FBi;66+ z60u*BtIpacrZbOwrgz`T>P3h`^q)61KzWMs4Q(5l!8Z1e+1bVrlkI_#MT4`lZB!sz z|B5V0yp+rJ)84ITlclVBGRsR7QpOOIspvA=w%NtudU4t{oIw59J+9$!Y#HQ&s-lu^ zs*1Q|Ua7}*8t`d*WRlGMstN2M!5J3bl6jtR@h@v34f~)9eU-g6#5B$vcrKbD6zy5ml zUcFbZ7jR_qnWZAj>byMmbPX)gQelVGM(-)B)+|n zPNP#ad@3GFC#CvdU&wzfq(!N^>kGugumSQYJu5HI>rEu4)Hf$bCmXM4@rS|xZ1zJQLDr#%y=oCm4ar&}TW-5urNvW-sUss|8QYEcE-aeaDo+`cG$BVtEI2H_s!a;$O4gPGBX5r87 zy+pk2zKdjK7^AD3lKwgja?^1C>vmGssa|6Tu z0eekvpvWJXT~k**_p-iXS73qIq72(5u-0^v{$_{Y^Sq)Crh1ibPc+i{RT8YiT|Eox zSlVXlw)s7(AJ97;-p;Rpe^;TnG#UB`rYbIv#-bAZ3mPDgZDzNyDAEjMJUT_5IyhP$ zj06*^j$V>;iX91MrU*b_+%?s;4Re$J$<;MY^>qPWGeyqJj)sEa_+)Bhv(7?nU3G2J z_NAOgs5XM4cK_1S zNKm*!gXbmKkC8tL@8C&z9(VIFoat8e99eD<{#Ug0i@m0zx>P+&mIbLo@i|9Nqe%MM z23bcglfmQBV7&wf;cfUYdp1jF*ay21x(U20k!3}*N7WvWgHOgAJ;6%_IT3?G zh_0LFHImCrNM-`o!)-7f`mIJ8;LvI))zmxW@X;r8J_qlaK}&8%5-lcLnC6<_@6z#VfM zpAjU3!lj{DS(F^NLb!8AeS^T!U%}ZqBCPF$BYklL!@*FjRGfSlcqEvQ0c?ju@EmyH zW2}JVUCvWrXQpQ;k00N6@AwhP$A&=Pp6yGujS(P%ZtMEROGl&5&-&D5WkVGkAM_ij?=^Y-)T1-48VGBo8p!keFkQ~Xvn~!o)+ueKBU*Y& zA|ZMCh(n}-KfaY_EJSZkGjW*e!eNRC_GHmPOmgBd(SgGR(My~ooPRdOVLZVhq2N#{ z3$)L9qr^PMFg?RHf$cRnIB#^%b`kwT=NqnVP-L{Q`(1aKBcKUh)Z5+p=3>2zUEu!4 zc**@FPL`yOY=6Q@#;ZDipn#0E-q@yGMEDF)D;*4lZKljmKT0dDCq7c%RK#X09m+T% z*ZGt~q;lmhg?fk&uH%kFinz~4_8hb8yOoDR)onAfVJ#v zHWEtMB`}g*WhX{IplpgY%0rf#DZgh9FB(9)av3>IK9S1IP4dL6D3rW7pFDrzPxZe@ zC7*Md#Bz)=`cT&&*yrZk&SmZ6W|LyhZ(?pSgX5;%ujntQ_>xRaMWNsR8V)41jS6g z218V5l?BNFz`YjQjTCRJAm{K^K+XyHV3q_g3JjKuS(tv$VKsOP>%pT~5gx>v@DCoW zB>mCG!N6xtB@+R!EyC;z%cMk-`ODq%cnnFa2vwPS11x}6M8|=*UPeaq7xI;f84hip z1n05kX16)R{g~rYZ60>!muuF%*<`W97#57hu=8juAbfL@+{F*n5}jY3BC11Vb6bJZ zN8FhrhEgD>B^y2>EVn#fUM8I2fE_<`0$TBU&#~D9N+9F65UD+@2MiLqpukaae9sbv zh+Szu=>E_hcQqU9_3!m9`aSwcYtZe4I{A(RM6-r%Bd$jEU@K?r0|* z7yr_|OSE?1mnKi`kp( zChS`7gY(&Xwg@{T4%65gb{!VUJ75A^#hS5Sx&_9v6|4?6!JlCidz#IG*I*&U*kf!a zbmB)s8GDFbY{$EZ-N()c4C=j|#D~^CVp?N2ldEJG*s*1H`|`pTa@HVGMe?O);wJLh_URS%voFC#`_q z5i~n59LN590UEXSw<=&U@A`)HYO1 zMnrkV9*Bw_$a>pF_Zd8@sf0K1tVMarxk+}8f zAbNG5*P%n`t6HyiWsp{?_FH+Vyxf`|ru8<~DPGaIxA(m+d6wZ+GsP2SH5Key(hhcNGuex=1-(ewwBq1;6p#C zlBQb5G<1Zu^B6ge@*h^vy$<0crcdII%V{1SiUQr5R6}3TRP9qq`_wWT=zfEHmeZH{ zn-@_Z6^)Sf_AwHqy3I)>jabGadeGoI2dHjq*C3i>6Bmk0ca@QjzyV1+U>UblJQPcE z>3BN?yl4jPmtw6B-gPte_L*&%a4K4>Nt$LE?ZVm}$4G$k@9@g+;oB`~yDj4x+GX(W z4fGYhXgbXhRI1$+9=DH4vigOjeqkA%^pl=Fi+k^D?%Ik5r%n4<(mu9~w-CS==F;

*AXIt+brEbc1-9z(webB(Sh13T E1Ai<2asU7T diff --git a/crawler_platform/app/api/routes.py b/crawler_platform/app/api/routes.py index 8a1dd7c..b67b601 100644 --- a/crawler_platform/app/api/routes.py +++ b/crawler_platform/app/api/routes.py @@ -11,6 +11,7 @@ from crawler_platform.app.config.loader import load_project_config from crawler_platform.app.core.crawler.discovery import discover_links from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher from crawler_platform.app.core.crawler.pipeline import CrawlPipeline +from crawler_platform.app.core.crawler.site_crawler import SiteCrawler from crawler_platform.app.core.database import models from crawler_platform.app.core.database.repository import KnowledgeRepository from crawler_platform.app.core.database.session import session_scope @@ -30,6 +31,13 @@ class CrawlRequest(BaseModel): extractor_base_url: str | None = None +class SiteCrawlRequest(CrawlRequest): + max_depth: int = 2 + max_pages: int = 50 + same_domain_only: bool = True + analyze_page_types: list[str] = Field(default_factory=lambda: ["product", "brand", "review"]) + + class DiscoverRequest(BaseModel): config_path: str source_name: str @@ -158,6 +166,34 @@ def register_routes(app, database_url: str) -> None: raise HTTPException(status_code=400, detail=str(exc)) from exc return {"page_id": result.page_id, "claim_count": result.claim_count, "entity_count": result.entity_count} + @app.post("/crawl-site") + def crawl_site(request: SiteCrawlRequest): + config = load_project_config(request.config_path) + with session_scope(database_url) as session: + repo = KnowledgeRepository(session) + crawler = SiteCrawler( + repo, + extractor_for_domain( + config.domain, + provider=request.extractor_provider, + model=request.extractor_model, + base_url=request.extractor_base_url, + ), + ) + try: + result = crawler.crawl_site( + config, + request.source_name, + request.url, + max_depth=max(request.max_depth, 0), + max_pages=max(min(request.max_pages, 500), 1), + same_domain_only=request.same_domain_only, + analyze_page_types=set(request.analyze_page_types), + ) + except Exception as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + return asdict(result) + @app.post("/discover") def discover(request: DiscoverRequest): config = load_project_config(request.config_path) diff --git a/crawler_platform/app/core/crawler/pipeline.py b/crawler_platform/app/core/crawler/pipeline.py index fc040ea..7650ad4 100644 --- a/crawler_platform/app/core/crawler/pipeline.py +++ b/crawler_platform/app/core/crawler/pipeline.py @@ -30,9 +30,7 @@ class CrawlPipeline: self.robots_policy = robots_policy or RobotsPolicy() def crawl_url(self, project_config: ProjectConfig, source_name: str, url: str) -> CrawlResult: - project = self.repository.upsert_project(project_config) source_config = project_config.source_by_name(source_name) - source = self.repository.get_source(project.id, source_name) if not self.robots_policy.allowed(url, source_config.respect_robots_txt): raise PermissionError(f"robots.txt does not allow crawling: {url}") @@ -40,6 +38,10 @@ class CrawlPipeline: fetch_result = fetcher.fetch(url) parser = self.parser_registry.get(source_config.parser) parsed = parser.parse(fetch_result.html, fetch_result.final_url or url) + bundle = self.extractor.extract(parsed.text, project_config) + + project = self.repository.upsert_project(project_config) + source = self.repository.get_source(project.id, source_name) page = self.repository.upsert_page( project_id=project.id, source_id=source.id, @@ -49,7 +51,5 @@ class CrawlPipeline: cleaned_text=parsed.text, metadata={**parsed.metadata, "final_url": fetch_result.final_url}, ) - bundle = self.extractor.extract(parsed.text, project_config) claims = self.repository.save_extraction_bundle(project.id, source, page, bundle) return CrawlResult(page_id=page.id, claim_count=len(claims), entity_count=len(bundle.entities)) - diff --git a/crawler_platform/app/core/crawler/site_crawler.py b/crawler_platform/app/core/crawler/site_crawler.py new file mode 100644 index 0000000..67d9678 --- /dev/null +++ b/crawler_platform/app/core/crawler/site_crawler.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass, field +from urllib.parse import urldefrag, urlparse + +from crawler_platform.app.config.loader import ProjectConfig +from crawler_platform.app.core.crawler.discovery import discover_links +from crawler_platform.app.core.crawler.fetchers import RobotsPolicy, make_fetcher +from crawler_platform.app.core.crawler.plugins import ParserRegistry, default_parser_registry +from crawler_platform.app.core.database import models +from crawler_platform.app.core.database.repository import KnowledgeRepository +from crawler_platform.app.core.extractor.base import Extractor + + +@dataclass(slots=True) +class SiteCrawlPageResult: + url: str + depth: int + status: str + page_type: str + page_id: int | None = None + claim_count: int = 0 + entity_count: int = 0 + discovered_count: int = 0 + error: str | None = None + + +@dataclass(slots=True) +class SiteCrawlResult: + seed_url: str + visited_count: int = 0 + analyzed_count: int = 0 + queued_count: int = 0 + skipped_count: int = 0 + errors: list[str] = field(default_factory=list) + pages: list[SiteCrawlPageResult] = field(default_factory=list) + + +class SiteCrawler: + def __init__( + self, + repository: KnowledgeRepository, + extractor: Extractor, + parser_registry: ParserRegistry | None = None, + robots_policy: RobotsPolicy | None = None, + ): + self.repository = repository + self.extractor = extractor + self.parser_registry = parser_registry or default_parser_registry() + self.robots_policy = robots_policy or RobotsPolicy() + + def crawl_site( + self, + project_config: ProjectConfig, + source_name: str, + seed_url: str, + max_depth: int = 2, + max_pages: int = 50, + same_domain_only: bool = True, + analyze_page_types: set[str] | None = None, + ) -> SiteCrawlResult: + source_config = project_config.source_by_name(source_name) + fetcher = make_fetcher(source_config.fetcher, source_config.rate_limit_per_minute) + parser = self.parser_registry.get(source_config.parser) + seed_host = normalized_host(seed_url) + analyze_page_types = analyze_page_types or {"product", "brand", "review"} + + result = SiteCrawlResult(seed_url=seed_url) + queue: deque[tuple[str, int]] = deque([(normalize_url(seed_url), 0)]) + queued: set[str] = {normalize_url(seed_url)} + visited: set[str] = set() + + project = self.repository.upsert_project(project_config) + source = self.repository.get_source(project.id, source_name) + + while queue and result.visited_count < max_pages: + url, depth = queue.popleft() + if url in visited: + continue + visited.add(url) + + job = self._create_job(project.id, source.id, url, depth) + if depth > max_depth: + self._finish_job(job, "skipped", "max depth exceeded") + result.skipped_count += 1 + result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="skipped", page_type="unknown")) + continue + if same_domain_only and normalized_host(url) != seed_host: + self._finish_job(job, "skipped", "outside same-domain filter") + result.skipped_count += 1 + result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="skipped", page_type="external")) + continue + if not self.robots_policy.allowed(url, source_config.respect_robots_txt): + error = f"robots.txt does not allow crawling: {url}" + self._finish_job(job, "blocked", error) + result.skipped_count += 1 + result.errors.append(error) + result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="blocked", page_type="unknown", error=error)) + continue + + try: + fetch_result = fetcher.fetch(url) + parsed = parser.parse(fetch_result.html, fetch_result.final_url or url) + page_type = classify_page(fetch_result.final_url or url, parsed.title, parsed.text) + links = discover_links(fetch_result.html, fetch_result.final_url or url, limit=200) + discovered_count = 0 + if depth < max_depth: + for link in links: + next_url = normalize_url(link.url) + if next_url in queued or next_url in visited: + continue + if same_domain_only and normalized_host(next_url) != seed_host: + continue + queue.append((next_url, depth + 1)) + queued.add(next_url) + discovered_count += 1 + + if page_type in analyze_page_types: + bundle = self.extractor.extract(parsed.text, project_config) + page = self.repository.upsert_page( + project_id=project.id, + source_id=source.id, + url=url, + title=parsed.title, + status_code=fetch_result.status_code, + cleaned_text=parsed.text, + metadata={ + **parsed.metadata, + "final_url": fetch_result.final_url, + "page_type": page_type, + "depth": depth, + }, + ) + claims = self.repository.save_extraction_bundle(project.id, source, page, bundle) + self._finish_job(job, "completed") + result.analyzed_count += 1 + result.pages.append( + SiteCrawlPageResult( + url=url, + depth=depth, + status="completed", + page_type=page_type, + page_id=page.id, + claim_count=len(claims), + entity_count=len(bundle.entities), + discovered_count=discovered_count, + ) + ) + else: + self._finish_job(job, "discovered") + result.skipped_count += 1 + result.pages.append( + SiteCrawlPageResult( + url=url, + depth=depth, + status="discovered", + page_type=page_type, + discovered_count=discovered_count, + ) + ) + result.visited_count += 1 + result.queued_count = len(queue) + except Exception as exc: + error = str(exc) + self._finish_job(job, "failed", error) + result.errors.append(error) + result.pages.append(SiteCrawlPageResult(url=url, depth=depth, status="failed", page_type="unknown", error=error)) + + return result + + def _create_job(self, project_id: int, source_id: int, url: str, depth: int) -> models.CrawlJob: + job = models.CrawlJob( + project_id=project_id, + source_id=source_id, + url=url, + status="running", + metadata_json={"depth": depth}, + started_at=models.utcnow(), + ) + self.repository.session.add(job) + self.repository.session.flush() + return job + + def _finish_job(self, job: models.CrawlJob, status: str, error: str | None = None) -> None: + job.status = status + job.error = error + job.finished_at = models.utcnow() + + +def normalize_url(url: str) -> str: + clean, _fragment = urldefrag(url.strip()) + return clean.rstrip("/") + + +def normalized_host(url: str) -> str: + parsed = urlparse(url) + return parsed.netloc.lower() + + +def classify_page(url: str, title: str | None, text: str) -> str: + combined = f"{url}\n{title or ''}\n{text[:3000]}".lower() + product_tokens = [ + "add to cart", + "buy now", + "price", + "top notes", + "middle notes", + "base notes", + "장바구니", + "구매", + "가격", + "탑 노트", + "베이스 노트", + ] + review_tokens = ["review", "reviews", "rating", "stars", "후기", "리뷰", "평점"] + brand_tokens = ["about us", "brand story", "official", "브랜드", "소개"] + if any(token in combined for token in product_tokens): + return "product" + if any(token in combined for token in review_tokens): + return "review" + if any(token in combined for token in brand_tokens): + return "brand" + return "listing" diff --git a/crawler_platform/app/core/database/repository.py b/crawler_platform/app/core/database/repository.py index ceb822a..8e3f601 100644 --- a/crawler_platform/app/core/database/repository.py +++ b/crawler_platform/app/core/database/repository.py @@ -33,9 +33,10 @@ class KnowledgeRepository: self.session.add(project) self.session.flush() else: - project.domain = config.domain - project.config = config_dict - project.updated_at = models.utcnow() + if project.domain != config.domain or project.config != config_dict: + project.domain = config.domain + project.config = config_dict + project.updated_at = models.utcnow() for source_config in config.sources: self.upsert_source(project, source_config) return project @@ -51,12 +52,20 @@ class KnowledgeRepository: source = models.Source(project_id=project.id, name=source_config.name) self.session.add(source) self.session.flush() - source.type = source_config.type - source.base_url = source_config.base_url - source.trust_level = source_config.trust_level - source.respect_robots_txt = source_config.respect_robots_txt - source.rate_limit_per_minute = source_config.rate_limit_per_minute - source.updated_at = models.utcnow() + changed = ( + source.type != source_config.type + or source.base_url != source_config.base_url + or source.trust_level != source_config.trust_level + or source.respect_robots_txt != source_config.respect_robots_txt + or source.rate_limit_per_minute != source_config.rate_limit_per_minute + ) + if changed: + source.type = source_config.type + source.base_url = source_config.base_url + source.trust_level = source_config.trust_level + source.respect_robots_txt = source_config.respect_robots_txt + source.rate_limit_per_minute = source_config.rate_limit_per_minute + source.updated_at = models.utcnow() return source def get_project(self, project_name: str) -> models.Project: diff --git a/crawler_platform/app/core/database/session.py b/crawler_platform/app/core/database/session.py index 40f26cf..141a749 100644 --- a/crawler_platform/app/core/database/session.py +++ b/crawler_platform/app/core/database/session.py @@ -3,15 +3,28 @@ from __future__ import annotations from contextlib import contextmanager from typing import Iterator -from sqlalchemy import create_engine +from sqlalchemy import create_engine, event from sqlalchemy.orm import Session, sessionmaker from crawler_platform.app.core.database.models import Base def make_engine(database_url: str = "sqlite:///crawler_platform.db"): - connect_args = {"check_same_thread": False} if database_url.startswith("sqlite") else {} - return create_engine(database_url, future=True, connect_args=connect_args) + connect_args = {"check_same_thread": False, "timeout": 60} if database_url.startswith("sqlite") else {} + engine = create_engine(database_url, future=True, connect_args=connect_args) + if database_url.startswith("sqlite"): + install_sqlite_pragmas(engine) + return engine + + +def install_sqlite_pragmas(engine) -> None: + @event.listens_for(engine, "connect") + def _set_sqlite_pragmas(dbapi_connection, connection_record): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA busy_timeout=60000") + cursor.execute("PRAGMA journal_mode=WAL") + cursor.execute("PRAGMA synchronous=NORMAL") + cursor.close() def init_db(database_url: str = "sqlite:///crawler_platform.db") -> None: @@ -21,7 +34,7 @@ def init_db(database_url: str = "sqlite:///crawler_platform.db") -> None: def make_session_factory(database_url: str = "sqlite:///crawler_platform.db") -> sessionmaker[Session]: engine = make_engine(database_url) - return sessionmaker(bind=engine, expire_on_commit=False, class_=Session, future=True) + return sessionmaker(bind=engine, expire_on_commit=False, autoflush=False, class_=Session, future=True) @contextmanager @@ -36,4 +49,3 @@ def session_scope(database_url: str = "sqlite:///crawler_platform.db") -> Iterat raise finally: session.close() - diff --git a/crawler_platform/app/domains/perfume/extractor.py b/crawler_platform/app/domains/perfume/extractor.py index 931e114..2ddd472 100644 --- a/crawler_platform/app/domains/perfume/extractor.py +++ b/crawler_platform/app/domains/perfume/extractor.py @@ -86,6 +86,30 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): name = "perfume_rule_based" def extract_entities(self, page_text: str, project_config: ProjectConfig) -> list[ExtractedEntity]: + product_cards = extract_product_cards(page_text) + if product_cards: + brand = valid_brand(extract_brand(page_text, "")) or infer_site_brand(page_text) + entities: list[ExtractedEntity] = [] + if brand: + entities.append(ExtractedEntity("Brand", brand, confidence=0.62)) + for card in product_cards: + attrs: dict[str, object] = {"name": card["name"]} + if brand: + attrs["brand"] = brand + if card.get("price"): + attrs["price"] = card["price"] + entities.append( + ExtractedEntity( + "Perfume", + str(card["name"]), + attrs, + evidence_text=str(card.get("evidence") or card["name"]), + confidence=0.74, + metadata={"page_pattern": "product_listing"}, + ) + ) + return dedupe_entities(entities) + product_name = extract_product_name(page_text) attrs = {"name": product_name} brand = extract_brand(page_text, product_name) @@ -137,6 +161,38 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): page_text: str, project_config: ProjectConfig, ) -> list[ExtractedClaim]: + product_cards = extract_product_cards(page_text) + if product_cards: + claims: list[ExtractedClaim] = [] + brand = next((entity for entity in entities if entity.entity_type == "Brand"), None) + for card in product_cards: + if brand: + claims.append( + ExtractedClaim( + str(card["name"]), + "Perfume", + "hasBrand", + brand.name, + "Brand", + evidence_text=brand.evidence_text or brand.name, + confidence=0.72, + confidence_reason="site brand inferred from listing page", + ) + ) + if card.get("price"): + claims.append( + ExtractedClaim( + str(card["name"]), + "Perfume", + "hasPrice", + object_value=card["price"], + evidence_text=str(card.get("evidence") or card["name"]), + confidence=0.76, + confidence_reason="Korean product listing price pattern matched", + ) + ) + return claims + perfume = next((entity for entity in entities if entity.entity_type == "Perfume"), None) if perfume is None: return [] @@ -195,7 +251,7 @@ class PerfumeRuleBasedExtractor(GenericRuleBasedExtractor): def extract_product_name(page_text: str) -> str: for line in page_text.splitlines()[:8]: clean = line.strip() - if clean and not looks_like_navigation(clean): + if clean and not looks_like_navigation(clean) and not is_template_placeholder(clean): return clean[:240] return first_non_empty_line(page_text) or "Unknown Perfume" @@ -217,6 +273,95 @@ def extract_brand(page_text: str, product_name: str) -> str | None: return None +def extract_product_cards(page_text: str) -> list[dict[str, object]]: + lines = [line.strip() for line in page_text.splitlines() if line.strip()] + cards: list[dict[str, object]] = [] + idx = 0 + while idx < len(lines): + if lines[idx] != "상품명": + idx += 1 + continue + name, name_idx = next_value_after_label(lines, idx) + if not name or is_template_placeholder(name) or name in {":", "상품명"}: + idx += 1 + continue + card: dict[str, object] = {"name": cleanup_value(name), "evidence": f"상품명: {name}"} + scan_end = next_label_index(lines, "상품명", name_idx + 1) or min(len(lines), name_idx + 12) + for price_label in ("할인판매가", "판매가", "price", "Price"): + label_idx = find_label_index(lines, price_label, name_idx + 1, scan_end) + if label_idx is None: + continue + raw_price, _price_idx = next_value_after_label(lines, label_idx) + parsed = parse_price_value(raw_price) + if parsed: + card["price"] = parsed + card["evidence"] = f"{card['evidence']} / {price_label}: {raw_price}" + break + cards.append(card) + idx = scan_end + return dedupe_product_cards(cards) + + +def next_value_after_label(lines: list[str], label_idx: int) -> tuple[str | None, int]: + for idx in range(label_idx + 1, min(len(lines), label_idx + 5)): + value = cleanup_value(lines[idx]) + if not value or value == ":": + continue + return value, idx + return None, label_idx + + +def next_label_index(lines: list[str], label: str, start: int) -> int | None: + for idx in range(start, len(lines)): + if lines[idx] == label: + return idx + return None + + +def find_label_index(lines: list[str], label: str, start: int, end: int) -> int | None: + lower_label = label.lower() + for idx in range(start, min(end, len(lines))): + if lines[idx].lower() == lower_label: + return idx + return None + + +def parse_price_value(raw_price: str | None) -> dict[str, object] | None: + if not raw_price: + return None + match = re.search(r"(?P\d{1,3}(?:,\d{3})*|\d+)\s*(?P원|KRW|₩|USD|\$)?", raw_price) + if not match: + return None + currency = match.group("currency") or "KRW" + if currency in {"원", "₩"}: + currency = "KRW" + return {"amount": float(match.group("amount").replace(",", "")), "currency": currency} + + +def dedupe_product_cards(cards: list[dict[str, object]]) -> list[dict[str, object]]: + seen: set[str] = set() + result: list[dict[str, object]] = [] + for card in cards: + key = str(card["name"]).strip().lower() + if key in seen: + continue + seen.add(key) + result.append(card) + return result + + +def infer_site_brand(page_text: str) -> str | None: + if "912 공식 홈페이지" in page_text or "912" in page_text[:500]: + return "912" + return None + + +def valid_brand(value: str | None) -> str | None: + if not value or is_template_placeholder(value): + return None + return value + + def extract_field_values(field: str, page_text: str) -> list[tuple[str, str]]: if field in NOTE_LABELS: return extract_labeled_values(page_text, NOTE_LABELS[field]) @@ -292,6 +437,15 @@ def looks_like_navigation(value: str) -> bool: return value.lower() in {"home", "shop", "menu", "cart", "login", "검색", "장바구니", "홈"} +def is_template_placeholder(value: str) -> bool: + clean = value.strip() + return clean.startswith("{#") or clean.endswith("}") or clean in { + "CLONE FRAGRANCE", + "NICHE FRAGRANCE", + "HOME FRAGRANCE", + } + + def field_entity_type(field: str) -> str: return { "top_notes": "Note", @@ -347,4 +501,3 @@ def dedupe_entities(entities: list[ExtractedEntity]) -> list[ExtractedEntity]: seen.add(key) result.append(entity) return result - diff --git a/crawler_platform/app/web/static/app.js b/crawler_platform/app/web/static/app.js index 79b9b57..7f1db76 100644 --- a/crawler_platform/app/web/static/app.js +++ b/crawler_platform/app/web/static/app.js @@ -8,10 +8,7 @@ const state = { const $ = (id) => document.getElementById(id); function csv(value) { - return value - .split(",") - .map((item) => item.trim()) - .filter(Boolean); + return value.split(",").map((item) => item.trim()).filter(Boolean); } async function api(path, options = {}) { @@ -25,7 +22,7 @@ async function api(path, options = {}) { const body = await response.json(); detail = body.detail || body.error || detail; } catch { - // Keep the HTTP status text. + // Keep status text. } throw new Error(detail); } @@ -39,16 +36,15 @@ function toast(message) { window.setTimeout(() => node.classList.remove("show"), 2400); } -function renderProjects() { - const list = $("projectList"); - list.innerHTML = ""; - state.projects.forEach((project) => { - const button = document.createElement("button"); - button.className = `project-item ${state.selectedProject === project.name ? "active" : ""}`; - button.innerHTML = `${project.name}${project.domain}`; - button.addEventListener("click", () => selectProject(project.name)); - list.appendChild(button); - }); +function requestBase() { + return { + config_path: $("configPath").value.trim(), + source_name: $("sourceSelect").value, + url: $("crawlUrl").value.trim(), + extractor_provider: $("extractorProvider").value, + extractor_model: $("extractorModel").value.trim() || null, + extractor_base_url: $("extractorBaseUrl").value.trim() || null, + }; } async function loadProjects() { @@ -62,6 +58,18 @@ async function loadProjects() { } } +function renderProjects() { + const list = $("projectList"); + list.innerHTML = ""; + state.projects.forEach((project) => { + const button = document.createElement("button"); + button.className = `project-item ${state.selectedProject === project.name ? "active" : ""}`; + button.innerHTML = `${escapeHtml(project.name)}${escapeHtml(project.domain)}`; + button.addEventListener("click", () => selectProject(project.name)); + list.appendChild(button); + }); +} + async function selectProject(projectName) { state.selectedProject = projectName; state.projectDetail = await api(`/projects/${encodeURIComponent(projectName)}`); @@ -77,6 +85,7 @@ function renderOverview() { $("metricProject").textContent = detail?.name ?? "-"; $("metricDomain").textContent = detail?.domain ?? "-"; $("metricSources").textContent = detail?.sources?.length ?? 0; + const sourceSelect = $("sourceSelect"); sourceSelect.innerHTML = ""; (detail?.sources ?? []).forEach((source) => { @@ -102,8 +111,7 @@ function renderOntology() { const predicates = state.ontology?.predicates ?? []; $("ontologyEntities").innerHTML = entityTypes.map(chip).join(""); $("ontologyPredicates").innerHTML = predicates.map(chip).join(""); - const filter = $("entityTypeFilter"); - filter.innerHTML = `${entityTypes + $("entityTypeFilter").innerHTML = `${entityTypes .map((type) => ``) .join("")}`; } @@ -115,40 +123,104 @@ async function createProject() { method: "POST", body: JSON.stringify({ config_path: configPath }), }); - toast(`프로젝트 생성: ${result.name}`); + toast(`Project created: ${result.name}`); state.selectedProject = result.name; await loadProjects(); } async function crawl() { if (!state.selectedProject) return; - const sourceName = $("sourceSelect").value; - const url = $("crawlUrl").value.trim(); - const provider = $("extractorProvider").value; - const model = $("extractorModel").value.trim(); - const baseUrl = $("extractorBaseUrl").value.trim(); - $("crawlResult").textContent = "수집 중..."; + $("crawlResult").textContent = "Crawling one URL..."; try { const result = await api("/crawl", { method: "POST", - body: JSON.stringify({ - config_path: $("configPath").value.trim(), - source_name: sourceName, - url, - extractor_provider: provider, - extractor_model: model || null, - extractor_base_url: baseUrl || null, - }), + body: JSON.stringify(requestBase()), }); - $("crawlResult").textContent = `analyzer ${provider}, page ${result.page_id}, claims ${result.claim_count}, entities ${result.entity_count}`; - toast("수집 완료"); + $("crawlResult").textContent = `URL done: page ${result.page_id}, claims ${result.claim_count}, entities ${result.entity_count}`; + toast("URL crawl completed"); await Promise.all([loadEntities(), loadClaims(), loadTags()]); } catch (error) { - $("crawlResult").textContent = `수집 실패: ${error.message}`; - toast("수집 실패"); + $("crawlResult").textContent = `Crawl failed: ${error.message}`; + toast("Crawl failed"); } } +async function crawlSite() { + if (!state.selectedProject) return; + $("crawlResult").textContent = "Crawling site from seed..."; + $("discoveredLinks").innerHTML = ""; + try { + const result = await api("/crawl-site", { + method: "POST", + body: JSON.stringify({ + ...requestBase(), + max_depth: Number($("siteMaxDepth").value || 0), + max_pages: Number($("siteMaxPages").value || 1), + same_domain_only: $("sameDomainOnly").checked, + analyze_page_types: ["product", "brand", "review"], + }), + }); + $("crawlResult").textContent = + `Site done: visited ${result.visited_count}, analyzed ${result.analyzed_count}, skipped ${result.skipped_count}, queued ${result.queued_count}`; + $("discoveredLinks").innerHTML = result.pages.map(renderSitePage).join(""); + toast("Site crawl completed"); + await Promise.all([loadEntities(), loadClaims(), loadTags()]); + } catch (error) { + $("crawlResult").textContent = `Site crawl failed: ${error.message}`; + toast("Site crawl failed"); + } +} + +function renderSitePage(page) { + return ` + + `; +} + +async function discover() { + $("crawlResult").textContent = "Discovering links..."; + $("discoveredLinks").innerHTML = ""; + try { + const result = await api("/discover", { + method: "POST", + body: JSON.stringify({ + config_path: $("configPath").value.trim(), + source_name: $("sourceSelect").value, + url: $("crawlUrl").value.trim(), + limit: 30, + }), + }); + if (!result.ok) { + $("crawlResult").textContent = result.error ?? "Discovery failed"; + return; + } + $("crawlResult").textContent = `Discovered ${result.links.length} links`; + $("discoveredLinks").innerHTML = result.links.map(renderDiscoveredLink).join(""); + document.querySelectorAll("[data-discovered-url]").forEach((button) => { + button.addEventListener("click", () => { + $("crawlUrl").value = button.dataset.discoveredUrl; + toast("URL copied to input"); + }); + }); + } catch (error) { + $("crawlResult").textContent = `Discovery failed: ${error.message}`; + toast("Discovery failed"); + } +} + +function renderDiscoveredLink(link) { + return ` + + `; +} + function updateExtractorOptions() { const provider = $("extractorProvider").value; $("extractorOptions").classList.toggle("active", provider !== "rule_based"); @@ -164,17 +236,14 @@ function updateExtractorOptions() { async function testExtractor() { const provider = $("extractorProvider").value; const baseUrl = $("extractorBaseUrl").value.trim(); - $("crawlResult").textContent = "분석기 연결 확인 중..."; + $("crawlResult").textContent = "Testing analyzer..."; const result = await api("/extractors/models", { method: "POST", - body: JSON.stringify({ - provider, - base_url: baseUrl || null, - }), + body: JSON.stringify({ provider, base_url: baseUrl || null }), }); if (!result.ok) { - $("crawlResult").textContent = `분석기 연결 실패: ${result.error}`; - toast("분석기 연결 실패"); + $("crawlResult").textContent = `Analyzer failed: ${result.error}`; + toast("Analyzer failed"); return; } const models = result.models ?? []; @@ -182,51 +251,9 @@ async function testExtractor() { $("extractorModel").value = models[0].id; } $("crawlResult").textContent = models.length - ? `연결됨. 모델 ${models.length}개: ${models.map((model) => model.id).join(", ")}` - : "연결됨. 모델 목록은 비어 있습니다."; - toast("분석기 연결 확인 완료"); -} - -async function discover() { - const sourceName = $("sourceSelect").value; - const url = $("crawlUrl").value.trim(); - $("crawlResult").textContent = "주소 발견 중..."; - $("discoveredLinks").innerHTML = ""; - try { - const result = await api("/discover", { - method: "POST", - body: JSON.stringify({ - config_path: $("configPath").value.trim(), - source_name: sourceName, - url, - limit: 30, - }), - }); - if (!result.ok) { - $("crawlResult").textContent = result.error ?? "주소 발견 실패"; - return; - } - $("crawlResult").textContent = `발견된 주소 ${result.links.length}개`; - $("discoveredLinks").innerHTML = result.links.map(renderDiscoveredLink).join(""); - document.querySelectorAll("[data-discovered-url]").forEach((button) => { - button.addEventListener("click", () => { - $("crawlUrl").value = button.dataset.discoveredUrl; - toast("URL 입력칸에 넣었습니다."); - }); - }); - } catch (error) { - $("crawlResult").textContent = `주소 발견 실패: ${error.message}`; - toast("주소 발견 실패"); - } -} - -function renderDiscoveredLink(link) { - return ` - - `; + ? `Analyzer connected. Models: ${models.map((model) => model.id).join(", ")}` + : "Analyzer connected. No models returned."; + toast("Analyzer connected"); } async function loadEntities() { @@ -251,7 +278,7 @@ async function mergeEntities() { const sourceId = Number($("mergeSourceId").value); const targetId = Number($("mergeTargetId").value); if (!sourceId || !targetId || sourceId === targetId) { - toast("병합할 ID와 남길 ID를 확인하세요."); + toast("Check entity IDs"); return; } const result = await api("/entities/merge", { @@ -263,10 +290,10 @@ async function mergeEntities() { }), }); if (!result.ok) { - toast(result.error ?? "병합 실패"); + toast(result.error ?? "Merge failed"); return; } - toast("Entity 병합 완료"); + toast("Entity merged"); $("mergeSourceId").value = ""; $("mergeTargetId").value = ""; await Promise.all([loadEntities(), loadClaims(), loadTags()]); @@ -296,7 +323,7 @@ function renderClaim(claim) {

- +
`; @@ -309,7 +336,7 @@ async function updateClaimConfidence(claimId) { method: "PATCH", body: JSON.stringify({ confidence, reason }), }); - toast("신뢰도 수정 완료"); + toast("Claim updated"); await loadClaims(); } @@ -357,16 +384,12 @@ function chip(value) { function table(headers, rows) { if (!rows.length) { - return `
데이터가 없습니다.
`; + return `
No data.
`; } return ` ${headers.map((header) => ``).join("")} - - ${rows - .map((row) => `${row.map((cell) => ``).join("")}`) - .join("")} - + ${rows.map((row) => `${row.map((cell) => ``).join("")}`).join("")}
${escapeHtml(header)}
${String(cell)}
${String(cell)}
`; } @@ -393,6 +416,7 @@ $("refreshBtn").addEventListener("click", loadProjects); $("createProjectBtn").addEventListener("click", createProject); $("discoverBtn").addEventListener("click", discover); $("crawlBtn").addEventListener("click", crawl); +$("siteCrawlBtn").addEventListener("click", crawlSite); $("extractorProvider").addEventListener("change", updateExtractorOptions); $("testExtractorBtn").addEventListener("click", testExtractor); $("loadEntitiesBtn").addEventListener("click", loadEntities); diff --git a/crawler_platform/app/web/static/index.html b/crawler_platform/app/web/static/index.html index d444fe7..1ca1e3a 100644 --- a/crawler_platform/app/web/static/index.html +++ b/crawler_platform/app/web/static/index.html @@ -10,9 +10,9 @@

Ontology Crawler

-

프로젝트별 수집, Claim 검수, 온톨로지 매핑, 추천 태그 확인

+

Project crawler, ontology mapping, claim review, and recommendation tags

- +
@@ -23,7 +23,7 @@
- +
@@ -37,7 +37,7 @@ - +
diff --git a/crawler_platform/app/web/static/styles.css b/crawler_platform/app/web/static/styles.css index 79a426d..c19f3ca 100644 --- a/crawler_platform/app/web/static/styles.css +++ b/crawler_platform/app/web/static/styles.css @@ -194,6 +194,29 @@ h2 { gap: 8px; } +button.full { + width: 100%; +} + +.site-controls { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 8px; + align-items: end; +} + +.check-row { + display: flex; + align-items: center; + gap: 8px; + min-height: 36px; +} + +.check-row input { + width: 16px; + min-height: 16px; +} + .extractor-options { display: none; gap: 8px; @@ -446,7 +469,8 @@ th { .split, .recommend-grid, .claim-actions, - .merge-bar { + .merge-bar, + .site-controls { grid-template-columns: 1fr; } } diff --git a/tests/test_site_crawler.py b/tests/test_site_crawler.py new file mode 100644 index 0000000..5e77f3e --- /dev/null +++ b/tests/test_site_crawler.py @@ -0,0 +1,11 @@ +from crawler_platform.app.core.crawler.site_crawler import classify_page, normalize_url + + +def test_classify_perfume_product_page(): + text = "Top notes: Bergamot, Neroli\nMiddle notes: Jasmine\nBase notes: Musk\nPrice $89" + + assert classify_page("https://example.com/products/neroli", "Neroli Summer", text) == "product" + + +def test_normalize_url_removes_fragment_and_trailing_slash(): + assert normalize_url("https://example.com/path/#details") == "https://example.com/path"