docs
This commit is contained in:
LASTA_DEV01\lasta
@@ -0,0 +1,465 @@
|
||||
import { useMemo, useState } from "react";
|
||||
import { useNavigate, useParams } from "react-router-dom";
|
||||
import {
|
||||
AlertCircle,
|
||||
ArrowLeft,
|
||||
CheckCircle2,
|
||||
Database,
|
||||
ExternalLink,
|
||||
FileSearch,
|
||||
Layers3,
|
||||
ListChecks,
|
||||
PlayCircle,
|
||||
RotateCw,
|
||||
} from "lucide-react";
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@/components/ui/card";
|
||||
import { Badge, BadgeProps } from "@/components/ui/badge";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Progress } from "@/components/ui/progress";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { useProject } from "@/hooks/useProjects";
|
||||
import { usePipeline, useRerunPipelineStage } from "@/hooks/usePlatform";
|
||||
import { formatDateTime, formatPercent } from "@/lib/display";
|
||||
|
||||
interface PipelineStep {
|
||||
key: string;
|
||||
name: string;
|
||||
description: string;
|
||||
count: number;
|
||||
target: number;
|
||||
status: "done" | "running" | "waiting" | "issue";
|
||||
extra?: string;
|
||||
}
|
||||
|
||||
const navigableStages = new Set(["human-review", "ontology-commit", "export"]);
|
||||
|
||||
function numberFrom(value: unknown): number {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
||||
}
|
||||
|
||||
function statusVariant(status: PipelineStep["status"]): BadgeProps["variant"] {
|
||||
switch (status) {
|
||||
case "done":
|
||||
return "success";
|
||||
case "running":
|
||||
return "default";
|
||||
case "issue":
|
||||
return "warning";
|
||||
default:
|
||||
return "outline";
|
||||
}
|
||||
}
|
||||
|
||||
function actionLabel(step: PipelineStep): string {
|
||||
if (step.key === "human-review") return "Open review";
|
||||
if (step.key === "ontology-commit") return "Open graph";
|
||||
if (step.key === "export") return "Open export";
|
||||
return "Rerun";
|
||||
}
|
||||
|
||||
export default function BuildPipelinePage() {
|
||||
const navigate = useNavigate();
|
||||
const { projectId } = useParams<{ projectId: string }>();
|
||||
const projectName = projectId ?? "";
|
||||
const { data: project } = useProject(projectName);
|
||||
const pipeline = usePipeline(projectName);
|
||||
const rerunStage = useRerunPipelineStage(projectName);
|
||||
const [notice, setNotice] = useState<{
|
||||
tone: "success" | "error" | "info";
|
||||
text: string;
|
||||
} | null>(null);
|
||||
|
||||
const steps = useMemo<PipelineStep[]>(() => {
|
||||
const stages = pipeline.data?.stages ?? [];
|
||||
const byKey = new Map(stages.map((stage) => [stage.key, stage]));
|
||||
const crawled = numberFrom(byKey.get("crawled")?.count);
|
||||
const extracted = numberFrom(byKey.get("extracted")?.count);
|
||||
const extractionEvents = numberFrom(byKey.get("extracted")?.extra?.events);
|
||||
const extractionErrors = numberFrom(byKey.get("extracted")?.extra?.errors);
|
||||
const claimStatuses = byKey.get("claims")?.extra ?? {};
|
||||
const claims = numberFrom(byKey.get("claims")?.count);
|
||||
const validated = numberFrom(byKey.get("validated")?.count);
|
||||
const rejected = numberFrom(claimStatuses.rejected);
|
||||
const candidates =
|
||||
numberFrom(claimStatuses.active) +
|
||||
numberFrom(claimStatuses.rule_candidate) +
|
||||
numberFrom(claimStatuses.candidate_claim);
|
||||
const graph = numberFrom(byKey.get("graph")?.count);
|
||||
const entities = numberFrom(byKey.get("graph")?.extra?.entities);
|
||||
const pageTyped =
|
||||
pipeline.data?.recent_pages.filter((page) => page.page_type).length ?? 0;
|
||||
|
||||
return [
|
||||
{
|
||||
key: "source-crawl",
|
||||
name: "Source Crawl",
|
||||
description: "Fetch source HTML, metadata, and in-domain links.",
|
||||
count: crawled,
|
||||
target: Math.max(crawled, 1),
|
||||
status: crawled > 0 ? "done" : "waiting",
|
||||
},
|
||||
{
|
||||
key: "page-clean",
|
||||
name: "Page Clean",
|
||||
description: "Normalize page text and remove repeated UI noise.",
|
||||
count: crawled,
|
||||
target: Math.max(crawled, 1),
|
||||
status: crawled > 0 ? "done" : "waiting",
|
||||
},
|
||||
{
|
||||
key: "page-classification",
|
||||
name: "Page Classification",
|
||||
description: "Classify product, brand, review, and supporting pages.",
|
||||
count: pageTyped,
|
||||
target: Math.max(crawled, 1),
|
||||
status: pageTyped > 0 ? "done" : crawled > 0 ? "running" : "waiting",
|
||||
},
|
||||
{
|
||||
key: "entity-extraction",
|
||||
name: "Entity Extraction",
|
||||
description: "Extract candidate entities from typed pages.",
|
||||
count: entities,
|
||||
target: Math.max(entities, 1),
|
||||
status: entities > 0 ? "done" : extracted > 0 ? "running" : "waiting",
|
||||
extra: `${extractionEvents} extraction events`,
|
||||
},
|
||||
{
|
||||
key: "claim-generation",
|
||||
name: "Claim Generation",
|
||||
description: "Turn entities and page evidence into relation claims.",
|
||||
count: claims,
|
||||
target: Math.max(claims, 1),
|
||||
status: claims > 0 ? "done" : entities > 0 ? "running" : "waiting",
|
||||
},
|
||||
{
|
||||
key: "deduplication",
|
||||
name: "Deduplication",
|
||||
description: "Collapse repeated entities and duplicate relation claims.",
|
||||
count: Math.max(claims - candidates, 0),
|
||||
target: Math.max(claims, 1),
|
||||
status: claims > 0 ? "done" : "waiting",
|
||||
},
|
||||
{
|
||||
key: "validation",
|
||||
name: "Validation",
|
||||
description: "Check schema fit, evidence quality, and conflicts.",
|
||||
count: Math.max(claims - candidates, 0),
|
||||
target: Math.max(claims, 1),
|
||||
status: extractionErrors > 0 ? "issue" : claims > 0 ? "done" : "waiting",
|
||||
extra: extractionErrors > 0 ? `${extractionErrors} errors` : undefined,
|
||||
},
|
||||
{
|
||||
key: "human-review",
|
||||
name: "Human Review",
|
||||
description: "Approve, reject, or send uncertain claims back for cleanup.",
|
||||
count: validated + rejected,
|
||||
target: Math.max(claims, 1),
|
||||
status:
|
||||
validated + rejected > 0
|
||||
? "done"
|
||||
: claims > 0
|
||||
? "running"
|
||||
: "waiting",
|
||||
extra: `${validated} approved / ${rejected} rejected`,
|
||||
},
|
||||
{
|
||||
key: "ontology-commit",
|
||||
name: "Ontology Commit",
|
||||
description: "Project accepted claims into the graph triple store.",
|
||||
count: graph,
|
||||
target: Math.max(validated, 1),
|
||||
status: graph > 0 ? "done" : validated > 0 ? "running" : "waiting",
|
||||
},
|
||||
{
|
||||
key: "export",
|
||||
name: "Export",
|
||||
description: "Prepare JSON, CSV, Turtle, and API access for downstream use.",
|
||||
count: graph,
|
||||
target: Math.max(graph, 1),
|
||||
status: graph > 0 ? "done" : "waiting",
|
||||
},
|
||||
];
|
||||
}, [pipeline.data]);
|
||||
|
||||
const completed = steps.filter((step) => step.status === "done").length;
|
||||
const overall = steps.length ? (completed / steps.length) * 100 : 0;
|
||||
|
||||
async function handleStageAction(step: PipelineStep) {
|
||||
setNotice(null);
|
||||
try {
|
||||
const response = await rerunStage.mutateAsync(step.key);
|
||||
if (response.action === "navigate" && response.route) {
|
||||
navigate(response.route);
|
||||
return;
|
||||
}
|
||||
setNotice({
|
||||
tone: "success",
|
||||
text:
|
||||
response.job_id != null
|
||||
? `Started job #${response.job_id} for ${step.name}.`
|
||||
: response.message ?? `${step.name} action completed.`,
|
||||
});
|
||||
pipeline.refetch();
|
||||
} catch (error) {
|
||||
setNotice({
|
||||
tone: "error",
|
||||
text: error instanceof Error ? error.message : "Stage action failed.",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="mx-auto max-w-7xl px-6 py-8">
|
||||
<div className="mb-6 flex items-center gap-3">
|
||||
<Button
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
onClick={() => navigate(`/crawl/${projectName}`)}
|
||||
aria-label="Back"
|
||||
>
|
||||
<ArrowLeft className="h-4 w-4" />
|
||||
</Button>
|
||||
<div className="flex-1">
|
||||
<h1 className="flex items-center gap-2 text-2xl font-bold tracking-tight">
|
||||
<Layers3 className="h-6 w-6 text-primary" />
|
||||
Build Pipeline
|
||||
</h1>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{project?.name ?? projectName} pipeline status and recovery actions
|
||||
</p>
|
||||
</div>
|
||||
<Button variant="outline" onClick={() => pipeline.refetch()}>
|
||||
<RotateCw className="h-4 w-4" />
|
||||
Refresh
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{notice && (
|
||||
<Card
|
||||
className={
|
||||
notice.tone === "error"
|
||||
? "mb-6 border-destructive"
|
||||
: "mb-6 border-green-200"
|
||||
}
|
||||
>
|
||||
<CardContent className="flex items-center gap-2 py-4 text-sm">
|
||||
<AlertCircle className="h-4 w-4" />
|
||||
{notice.text}
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
{pipeline.isError && (
|
||||
<Card className="mb-6 border-destructive">
|
||||
<CardContent className="flex items-center gap-2 py-4 text-sm text-destructive">
|
||||
<AlertCircle className="h-4 w-4" />
|
||||
{(pipeline.error as Error).message}
|
||||
</CardContent>
|
||||
</Card>
|
||||
)}
|
||||
|
||||
<div className="mb-6 grid gap-4 md:grid-cols-4">
|
||||
<MetricCard
|
||||
icon={Database}
|
||||
label="Collected pages"
|
||||
value={pipeline.data?.stages.find((s) => s.key === "crawled")?.count}
|
||||
/>
|
||||
<MetricCard
|
||||
icon={FileSearch}
|
||||
label="Extraction pages"
|
||||
value={pipeline.data?.stages.find((s) => s.key === "extracted")?.count}
|
||||
/>
|
||||
<MetricCard
|
||||
icon={ListChecks}
|
||||
label="Claims"
|
||||
value={pipeline.data?.stages.find((s) => s.key === "claims")?.count}
|
||||
/>
|
||||
<MetricCard
|
||||
icon={CheckCircle2}
|
||||
label="Overall"
|
||||
value={formatPercent(overall / 100)}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<Card className="mb-6">
|
||||
<CardHeader>
|
||||
<div className="flex items-center justify-between gap-3">
|
||||
<div>
|
||||
<CardTitle>Ontology Build Flow</CardTitle>
|
||||
<CardDescription>
|
||||
The ten major build stages, current counts, and recovery entry points.
|
||||
</CardDescription>
|
||||
</div>
|
||||
<Badge variant="outline">
|
||||
{completed}/{steps.length} complete
|
||||
</Badge>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<Progress value={overall} className="mb-5" />
|
||||
{pipeline.isLoading ? (
|
||||
<div className="space-y-3">
|
||||
{Array.from({ length: 6 }).map((_, index) => (
|
||||
<Skeleton key={index} className="h-20" />
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div className="grid gap-3 lg:grid-cols-2">
|
||||
{steps.map((step, index) => {
|
||||
const progress =
|
||||
step.target > 0
|
||||
? Math.min(100, (step.count / step.target) * 100)
|
||||
: 0;
|
||||
const isNavigable = navigableStages.has(step.key);
|
||||
return (
|
||||
<article
|
||||
key={step.key}
|
||||
className="rounded-md border bg-background p-4"
|
||||
>
|
||||
<div className="mb-3 flex items-start justify-between gap-3">
|
||||
<div>
|
||||
<div className="text-xs text-muted-foreground">
|
||||
Step {index + 1}
|
||||
</div>
|
||||
<h3 className="font-semibold">{step.name}</h3>
|
||||
<p className="mt-1 text-sm text-muted-foreground">
|
||||
{step.description}
|
||||
</p>
|
||||
</div>
|
||||
<Badge variant={statusVariant(step.status)}>
|
||||
{step.status}
|
||||
</Badge>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
<Progress value={progress} className="h-2" />
|
||||
<span className="w-16 text-right text-xs text-muted-foreground">
|
||||
{Math.round(progress)}%
|
||||
</span>
|
||||
</div>
|
||||
<div className="mt-3 flex items-center justify-between gap-3 text-xs text-muted-foreground">
|
||||
<div>
|
||||
<span>{step.count.toLocaleString()} items</span>
|
||||
{step.extra && <span> / {step.extra}</span>}
|
||||
</div>
|
||||
<Button
|
||||
size="sm"
|
||||
variant={isNavigable ? "outline" : "secondary"}
|
||||
disabled={!projectName || rerunStage.isPending}
|
||||
onClick={() => handleStageAction(step)}
|
||||
>
|
||||
{isNavigable ? (
|
||||
<ExternalLink className="h-3.5 w-3.5" />
|
||||
) : (
|
||||
<RotateCw className="h-3.5 w-3.5" />
|
||||
)}
|
||||
{rerunStage.isPending
|
||||
? "Working"
|
||||
: actionLabel(step)}
|
||||
</Button>
|
||||
</div>
|
||||
</article>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<div className="grid gap-6 lg:grid-cols-2">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Recent Pages</CardTitle>
|
||||
<CardDescription>Most recently collected and typed pages.</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<ul className="divide-y">
|
||||
{(pipeline.data?.recent_pages ?? []).map((page) => (
|
||||
<li key={page.id} className="py-3 text-sm">
|
||||
<div className="flex items-center gap-2">
|
||||
<Badge variant="outline">{page.page_type ?? "unknown"}</Badge>
|
||||
<span className="truncate font-medium">
|
||||
{page.title || page.url}
|
||||
</span>
|
||||
</div>
|
||||
<div className="mt-1 truncate text-xs text-muted-foreground">
|
||||
{page.url}
|
||||
</div>
|
||||
<div className="mt-1 text-xs text-muted-foreground">
|
||||
{formatDateTime(page.fetched_at)}
|
||||
</div>
|
||||
</li>
|
||||
))}
|
||||
{pipeline.data?.recent_pages.length === 0 && (
|
||||
<li className="py-6 text-center text-sm text-muted-foreground">
|
||||
No collected pages yet.
|
||||
</li>
|
||||
)}
|
||||
</ul>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Recent Claims</CardTitle>
|
||||
<CardDescription>Latest relation claims from the build flow.</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<ul className="divide-y">
|
||||
{(pipeline.data?.recent_claims ?? []).map((claim) => (
|
||||
<li key={claim.id} className="py-3 text-sm">
|
||||
<div className="flex items-center gap-2">
|
||||
<PlayCircle className="h-4 w-4 text-muted-foreground" />
|
||||
<span className="truncate font-medium">
|
||||
{claim.subject || "-"}
|
||||
</span>
|
||||
<Badge variant="secondary">{claim.predicate}</Badge>
|
||||
</div>
|
||||
<div className="mt-1 flex flex-wrap gap-2 text-xs text-muted-foreground">
|
||||
<span>{formatPercent(claim.confidence)}</span>
|
||||
<span>{claim.status}</span>
|
||||
<span>{formatDateTime(claim.last_seen_at)}</span>
|
||||
</div>
|
||||
</li>
|
||||
))}
|
||||
{pipeline.data?.recent_claims.length === 0 && (
|
||||
<li className="py-6 text-center text-sm text-muted-foreground">
|
||||
No claims have been generated yet.
|
||||
</li>
|
||||
)}
|
||||
</ul>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function MetricCard({
|
||||
icon: Icon,
|
||||
label,
|
||||
value,
|
||||
}: {
|
||||
icon: React.ComponentType<{ className?: string }>;
|
||||
label: string;
|
||||
value: number | string | undefined;
|
||||
}) {
|
||||
return (
|
||||
<Card>
|
||||
<CardContent className="flex items-center justify-between gap-4 py-4">
|
||||
<div>
|
||||
<div className="text-xs text-muted-foreground">{label}</div>
|
||||
<div className="mt-1 text-2xl font-semibold">
|
||||
{typeof value === "number" ? value.toLocaleString() : value ?? "-"}
|
||||
</div>
|
||||
</div>
|
||||
<Icon className="h-5 w-5 text-muted-foreground" />
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user