56 lines
1.3 KiB
TypeScript
56 lines
1.3 KiB
TypeScript
/** Coarse paragraph chunks from PDFWorker plain text (MinerU MVP fallback). */
|
|
|
|
export interface TextChunk {
|
|
id: string;
|
|
orderIndex: number;
|
|
text: string;
|
|
}
|
|
|
|
const MIN_CHUNK_CHARS = 200;
|
|
const MAX_CHUNK_CHARS = 900;
|
|
const MAX_CHUNKS = 40;
|
|
|
|
export function chunkPaperText(text: string): TextChunk[] {
|
|
const paragraphs = text
|
|
.split(/\n\s*\n+/)
|
|
.map((s) => s.replace(/\s+/g, " ").trim())
|
|
.filter((s) => s.length > 40);
|
|
|
|
const merged: string[] = [];
|
|
let buffer = "";
|
|
|
|
for (const para of paragraphs) {
|
|
if (
|
|
buffer &&
|
|
(buffer.length + para.length > MAX_CHUNK_CHARS ||
|
|
(buffer.length >= MIN_CHUNK_CHARS && merged.length >= MAX_CHUNKS - 1))
|
|
) {
|
|
merged.push(buffer);
|
|
buffer = para;
|
|
} else {
|
|
buffer = buffer ? `${buffer} ${para}` : para;
|
|
}
|
|
}
|
|
if (buffer) merged.push(buffer);
|
|
|
|
if (!merged.length && text.trim()) {
|
|
merged.push(text.trim().slice(0, MAX_CHUNK_CHARS));
|
|
}
|
|
|
|
return merged.slice(0, MAX_CHUNKS).map((chunkText, orderIndex) => ({
|
|
id: `p-${orderIndex}`,
|
|
orderIndex,
|
|
text: chunkText.slice(0, 2000),
|
|
}));
|
|
}
|
|
|
|
/** Compact outline for LLM beats generation. */
|
|
export function formatChunkOutline(chunks: TextChunk[]): string {
|
|
return chunks
|
|
.map(
|
|
(c) =>
|
|
`[${c.id}] ${c.text.slice(0, 100)}${c.text.length > 100 ? "…" : ""}`,
|
|
)
|
|
.join("\n");
|
|
}
|