466 lines
16 KiB
TypeScript
466 lines
16 KiB
TypeScript
import { useMemo, useState } from "react";
|
|
import { useNavigate, useParams } from "react-router-dom";
|
|
import {
|
|
AlertCircle,
|
|
ArrowLeft,
|
|
CheckCircle2,
|
|
Database,
|
|
ExternalLink,
|
|
FileSearch,
|
|
Layers3,
|
|
ListChecks,
|
|
PlayCircle,
|
|
RotateCw,
|
|
} from "lucide-react";
|
|
import {
|
|
Card,
|
|
CardContent,
|
|
CardDescription,
|
|
CardHeader,
|
|
CardTitle,
|
|
} from "@/components/ui/card";
|
|
import { Badge, BadgeProps } from "@/components/ui/badge";
|
|
import { Button } from "@/components/ui/button";
|
|
import { Progress } from "@/components/ui/progress";
|
|
import { Skeleton } from "@/components/ui/skeleton";
|
|
import { useProject } from "@/hooks/useProjects";
|
|
import { usePipeline, useRerunPipelineStage } from "@/hooks/usePlatform";
|
|
import { formatDateTime, formatPercent } from "@/lib/display";
|
|
|
|
interface PipelineStep {
|
|
key: string;
|
|
name: string;
|
|
description: string;
|
|
count: number;
|
|
target: number;
|
|
status: "done" | "running" | "waiting" | "issue";
|
|
extra?: string;
|
|
}
|
|
|
|
const navigableStages = new Set(["human-review", "ontology-commit", "export"]);
|
|
|
|
function numberFrom(value: unknown): number {
|
|
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
|
}
|
|
|
|
function statusVariant(status: PipelineStep["status"]): BadgeProps["variant"] {
|
|
switch (status) {
|
|
case "done":
|
|
return "success";
|
|
case "running":
|
|
return "default";
|
|
case "issue":
|
|
return "warning";
|
|
default:
|
|
return "outline";
|
|
}
|
|
}
|
|
|
|
function actionLabel(step: PipelineStep): string {
|
|
if (step.key === "human-review") return "Open review";
|
|
if (step.key === "ontology-commit") return "Open graph";
|
|
if (step.key === "export") return "Open export";
|
|
return "Rerun";
|
|
}
|
|
|
|
export default function BuildPipelinePage() {
|
|
const navigate = useNavigate();
|
|
const { projectId } = useParams<{ projectId: string }>();
|
|
const projectName = projectId ?? "";
|
|
const { data: project } = useProject(projectName);
|
|
const pipeline = usePipeline(projectName);
|
|
const rerunStage = useRerunPipelineStage(projectName);
|
|
const [notice, setNotice] = useState<{
|
|
tone: "success" | "error" | "info";
|
|
text: string;
|
|
} | null>(null);
|
|
|
|
const steps = useMemo<PipelineStep[]>(() => {
|
|
const stages = pipeline.data?.stages ?? [];
|
|
const byKey = new Map(stages.map((stage) => [stage.key, stage]));
|
|
const crawled = numberFrom(byKey.get("crawled")?.count);
|
|
const extracted = numberFrom(byKey.get("extracted")?.count);
|
|
const extractionEvents = numberFrom(byKey.get("extracted")?.extra?.events);
|
|
const extractionErrors = numberFrom(byKey.get("extracted")?.extra?.errors);
|
|
const claimStatuses = byKey.get("claims")?.extra ?? {};
|
|
const claims = numberFrom(byKey.get("claims")?.count);
|
|
const validated = numberFrom(byKey.get("validated")?.count);
|
|
const rejected = numberFrom(claimStatuses.rejected);
|
|
const candidates =
|
|
numberFrom(claimStatuses.active) +
|
|
numberFrom(claimStatuses.rule_candidate) +
|
|
numberFrom(claimStatuses.candidate_claim);
|
|
const graph = numberFrom(byKey.get("graph")?.count);
|
|
const entities = numberFrom(byKey.get("graph")?.extra?.entities);
|
|
const pageTyped =
|
|
pipeline.data?.recent_pages.filter((page) => page.page_type).length ?? 0;
|
|
|
|
return [
|
|
{
|
|
key: "source-crawl",
|
|
name: "Source Crawl",
|
|
description: "Fetch source HTML, metadata, and in-domain links.",
|
|
count: crawled,
|
|
target: Math.max(crawled, 1),
|
|
status: crawled > 0 ? "done" : "waiting",
|
|
},
|
|
{
|
|
key: "page-clean",
|
|
name: "Page Clean",
|
|
description: "Normalize page text and remove repeated UI noise.",
|
|
count: crawled,
|
|
target: Math.max(crawled, 1),
|
|
status: crawled > 0 ? "done" : "waiting",
|
|
},
|
|
{
|
|
key: "page-classification",
|
|
name: "Page Classification",
|
|
description: "Classify product, brand, review, and supporting pages.",
|
|
count: pageTyped,
|
|
target: Math.max(crawled, 1),
|
|
status: pageTyped > 0 ? "done" : crawled > 0 ? "running" : "waiting",
|
|
},
|
|
{
|
|
key: "entity-extraction",
|
|
name: "Entity Extraction",
|
|
description: "Extract candidate entities from typed pages.",
|
|
count: entities,
|
|
target: Math.max(entities, 1),
|
|
status: entities > 0 ? "done" : extracted > 0 ? "running" : "waiting",
|
|
extra: `${extractionEvents} extraction events`,
|
|
},
|
|
{
|
|
key: "claim-generation",
|
|
name: "Claim Generation",
|
|
description: "Turn entities and page evidence into relation claims.",
|
|
count: claims,
|
|
target: Math.max(claims, 1),
|
|
status: claims > 0 ? "done" : entities > 0 ? "running" : "waiting",
|
|
},
|
|
{
|
|
key: "deduplication",
|
|
name: "Deduplication",
|
|
description: "Collapse repeated entities and duplicate relation claims.",
|
|
count: Math.max(claims - candidates, 0),
|
|
target: Math.max(claims, 1),
|
|
status: claims > 0 ? "done" : "waiting",
|
|
},
|
|
{
|
|
key: "validation",
|
|
name: "Validation",
|
|
description: "Check schema fit, evidence quality, and conflicts.",
|
|
count: Math.max(claims - candidates, 0),
|
|
target: Math.max(claims, 1),
|
|
status: extractionErrors > 0 ? "issue" : claims > 0 ? "done" : "waiting",
|
|
extra: extractionErrors > 0 ? `${extractionErrors} errors` : undefined,
|
|
},
|
|
{
|
|
key: "human-review",
|
|
name: "Human Review",
|
|
description: "Approve, reject, or send uncertain claims back for cleanup.",
|
|
count: validated + rejected,
|
|
target: Math.max(claims, 1),
|
|
status:
|
|
validated + rejected > 0
|
|
? "done"
|
|
: claims > 0
|
|
? "running"
|
|
: "waiting",
|
|
extra: `${validated} approved / ${rejected} rejected`,
|
|
},
|
|
{
|
|
key: "ontology-commit",
|
|
name: "Ontology Commit",
|
|
description: "Project accepted claims into the graph triple store.",
|
|
count: graph,
|
|
target: Math.max(validated, 1),
|
|
status: graph > 0 ? "done" : validated > 0 ? "running" : "waiting",
|
|
},
|
|
{
|
|
key: "export",
|
|
name: "Export",
|
|
description: "Prepare JSON, CSV, Turtle, and API access for downstream use.",
|
|
count: graph,
|
|
target: Math.max(graph, 1),
|
|
status: graph > 0 ? "done" : "waiting",
|
|
},
|
|
];
|
|
}, [pipeline.data]);
|
|
|
|
const completed = steps.filter((step) => step.status === "done").length;
|
|
const overall = steps.length ? (completed / steps.length) * 100 : 0;
|
|
|
|
async function handleStageAction(step: PipelineStep) {
|
|
setNotice(null);
|
|
try {
|
|
const response = await rerunStage.mutateAsync(step.key);
|
|
if (response.action === "navigate" && response.route) {
|
|
navigate(response.route);
|
|
return;
|
|
}
|
|
setNotice({
|
|
tone: "success",
|
|
text:
|
|
response.job_id != null
|
|
? `Started job #${response.job_id} for ${step.name}.`
|
|
: response.message ?? `${step.name} action completed.`,
|
|
});
|
|
pipeline.refetch();
|
|
} catch (error) {
|
|
setNotice({
|
|
tone: "error",
|
|
text: error instanceof Error ? error.message : "Stage action failed.",
|
|
});
|
|
}
|
|
}
|
|
|
|
return (
|
|
<div className="mx-auto max-w-7xl px-6 py-8">
|
|
<div className="mb-6 flex items-center gap-3">
|
|
<Button
|
|
variant="ghost"
|
|
size="icon"
|
|
onClick={() => navigate(`/crawl/${projectName}`)}
|
|
aria-label="Back"
|
|
>
|
|
<ArrowLeft className="h-4 w-4" />
|
|
</Button>
|
|
<div className="flex-1">
|
|
<h1 className="flex items-center gap-2 text-2xl font-bold tracking-tight">
|
|
<Layers3 className="h-6 w-6 text-primary" />
|
|
Build Pipeline
|
|
</h1>
|
|
<p className="text-sm text-muted-foreground">
|
|
{project?.name ?? projectName} pipeline status and recovery actions
|
|
</p>
|
|
</div>
|
|
<Button variant="outline" onClick={() => pipeline.refetch()}>
|
|
<RotateCw className="h-4 w-4" />
|
|
Refresh
|
|
</Button>
|
|
</div>
|
|
|
|
{notice && (
|
|
<Card
|
|
className={
|
|
notice.tone === "error"
|
|
? "mb-6 border-destructive"
|
|
: "mb-6 border-green-200"
|
|
}
|
|
>
|
|
<CardContent className="flex items-center gap-2 py-4 text-sm">
|
|
<AlertCircle className="h-4 w-4" />
|
|
{notice.text}
|
|
</CardContent>
|
|
</Card>
|
|
)}
|
|
|
|
{pipeline.isError && (
|
|
<Card className="mb-6 border-destructive">
|
|
<CardContent className="flex items-center gap-2 py-4 text-sm text-destructive">
|
|
<AlertCircle className="h-4 w-4" />
|
|
{(pipeline.error as Error).message}
|
|
</CardContent>
|
|
</Card>
|
|
)}
|
|
|
|
<div className="mb-6 grid gap-4 md:grid-cols-4">
|
|
<MetricCard
|
|
icon={Database}
|
|
label="Collected pages"
|
|
value={pipeline.data?.stages.find((s) => s.key === "crawled")?.count}
|
|
/>
|
|
<MetricCard
|
|
icon={FileSearch}
|
|
label="Extraction pages"
|
|
value={pipeline.data?.stages.find((s) => s.key === "extracted")?.count}
|
|
/>
|
|
<MetricCard
|
|
icon={ListChecks}
|
|
label="Claims"
|
|
value={pipeline.data?.stages.find((s) => s.key === "claims")?.count}
|
|
/>
|
|
<MetricCard
|
|
icon={CheckCircle2}
|
|
label="Overall"
|
|
value={formatPercent(overall / 100)}
|
|
/>
|
|
</div>
|
|
|
|
<Card className="mb-6">
|
|
<CardHeader>
|
|
<div className="flex items-center justify-between gap-3">
|
|
<div>
|
|
<CardTitle>Ontology Build Flow</CardTitle>
|
|
<CardDescription>
|
|
The ten major build stages, current counts, and recovery entry points.
|
|
</CardDescription>
|
|
</div>
|
|
<Badge variant="outline">
|
|
{completed}/{steps.length} complete
|
|
</Badge>
|
|
</div>
|
|
</CardHeader>
|
|
<CardContent>
|
|
<Progress value={overall} className="mb-5" />
|
|
{pipeline.isLoading ? (
|
|
<div className="space-y-3">
|
|
{Array.from({ length: 6 }).map((_, index) => (
|
|
<Skeleton key={index} className="h-20" />
|
|
))}
|
|
</div>
|
|
) : (
|
|
<div className="grid gap-3 lg:grid-cols-2">
|
|
{steps.map((step, index) => {
|
|
const progress =
|
|
step.target > 0
|
|
? Math.min(100, (step.count / step.target) * 100)
|
|
: 0;
|
|
const isNavigable = navigableStages.has(step.key);
|
|
return (
|
|
<article
|
|
key={step.key}
|
|
className="rounded-md border bg-background p-4"
|
|
>
|
|
<div className="mb-3 flex items-start justify-between gap-3">
|
|
<div>
|
|
<div className="text-xs text-muted-foreground">
|
|
Step {index + 1}
|
|
</div>
|
|
<h3 className="font-semibold">{step.name}</h3>
|
|
<p className="mt-1 text-sm text-muted-foreground">
|
|
{step.description}
|
|
</p>
|
|
</div>
|
|
<Badge variant={statusVariant(step.status)}>
|
|
{step.status}
|
|
</Badge>
|
|
</div>
|
|
<div className="flex items-center gap-3">
|
|
<Progress value={progress} className="h-2" />
|
|
<span className="w-16 text-right text-xs text-muted-foreground">
|
|
{Math.round(progress)}%
|
|
</span>
|
|
</div>
|
|
<div className="mt-3 flex items-center justify-between gap-3 text-xs text-muted-foreground">
|
|
<div>
|
|
<span>{step.count.toLocaleString()} items</span>
|
|
{step.extra && <span> / {step.extra}</span>}
|
|
</div>
|
|
<Button
|
|
size="sm"
|
|
variant={isNavigable ? "outline" : "secondary"}
|
|
disabled={!projectName || rerunStage.isPending}
|
|
onClick={() => handleStageAction(step)}
|
|
>
|
|
{isNavigable ? (
|
|
<ExternalLink className="h-3.5 w-3.5" />
|
|
) : (
|
|
<RotateCw className="h-3.5 w-3.5" />
|
|
)}
|
|
{rerunStage.isPending
|
|
? "Working"
|
|
: actionLabel(step)}
|
|
</Button>
|
|
</div>
|
|
</article>
|
|
);
|
|
})}
|
|
</div>
|
|
)}
|
|
</CardContent>
|
|
</Card>
|
|
|
|
<div className="grid gap-6 lg:grid-cols-2">
|
|
<Card>
|
|
<CardHeader>
|
|
<CardTitle>Recent Pages</CardTitle>
|
|
<CardDescription>Most recently collected and typed pages.</CardDescription>
|
|
</CardHeader>
|
|
<CardContent>
|
|
<ul className="divide-y">
|
|
{(pipeline.data?.recent_pages ?? []).map((page) => (
|
|
<li key={page.id} className="py-3 text-sm">
|
|
<div className="flex items-center gap-2">
|
|
<Badge variant="outline">{page.page_type ?? "unknown"}</Badge>
|
|
<span className="truncate font-medium">
|
|
{page.title || page.url}
|
|
</span>
|
|
</div>
|
|
<div className="mt-1 truncate text-xs text-muted-foreground">
|
|
{page.url}
|
|
</div>
|
|
<div className="mt-1 text-xs text-muted-foreground">
|
|
{formatDateTime(page.fetched_at)}
|
|
</div>
|
|
</li>
|
|
))}
|
|
{pipeline.data?.recent_pages.length === 0 && (
|
|
<li className="py-6 text-center text-sm text-muted-foreground">
|
|
No collected pages yet.
|
|
</li>
|
|
)}
|
|
</ul>
|
|
</CardContent>
|
|
</Card>
|
|
|
|
<Card>
|
|
<CardHeader>
|
|
<CardTitle>Recent Claims</CardTitle>
|
|
<CardDescription>Latest relation claims from the build flow.</CardDescription>
|
|
</CardHeader>
|
|
<CardContent>
|
|
<ul className="divide-y">
|
|
{(pipeline.data?.recent_claims ?? []).map((claim) => (
|
|
<li key={claim.id} className="py-3 text-sm">
|
|
<div className="flex items-center gap-2">
|
|
<PlayCircle className="h-4 w-4 text-muted-foreground" />
|
|
<span className="truncate font-medium">
|
|
{claim.subject || "-"}
|
|
</span>
|
|
<Badge variant="secondary">{claim.predicate}</Badge>
|
|
</div>
|
|
<div className="mt-1 flex flex-wrap gap-2 text-xs text-muted-foreground">
|
|
<span>{formatPercent(claim.confidence)}</span>
|
|
<span>{claim.status}</span>
|
|
<span>{formatDateTime(claim.last_seen_at)}</span>
|
|
</div>
|
|
</li>
|
|
))}
|
|
{pipeline.data?.recent_claims.length === 0 && (
|
|
<li className="py-6 text-center text-sm text-muted-foreground">
|
|
No claims have been generated yet.
|
|
</li>
|
|
)}
|
|
</ul>
|
|
</CardContent>
|
|
</Card>
|
|
</div>
|
|
</div>
|
|
);
|
|
}
|
|
|
|
function MetricCard({
|
|
icon: Icon,
|
|
label,
|
|
value,
|
|
}: {
|
|
icon: React.ComponentType<{ className?: string }>;
|
|
label: string;
|
|
value: number | string | undefined;
|
|
}) {
|
|
return (
|
|
<Card>
|
|
<CardContent className="flex items-center justify-between gap-4 py-4">
|
|
<div>
|
|
<div className="text-xs text-muted-foreground">{label}</div>
|
|
<div className="mt-1 text-2xl font-semibold">
|
|
{typeof value === "number" ? value.toLocaleString() : value ?? "-"}
|
|
</div>
|
|
</div>
|
|
<Icon className="h-5 w-5 text-muted-foreground" />
|
|
</CardContent>
|
|
</Card>
|
|
);
|
|
}
|