diff --git a/src/Fetcher.ts b/src/Fetcher.ts index 83fb95e..41031f2 100644 --- a/src/Fetcher.ts +++ b/src/Fetcher.ts @@ -4,6 +4,14 @@ import is_ip_private from "private-ip"; import { RequestPayload } from "./types.js"; export class Fetcher { + private static applyLengthLimits(text: string, maxLength: number, startIndex: number): string { + if (startIndex >= text.length) { + return ""; + } + + const end = Math.min(startIndex + maxLength, text.length); + return text.substring(startIndex, end); + } private static async _fetch({ url, headers, @@ -38,7 +46,15 @@ export class Fetcher { static async html(requestPayload: RequestPayload) { try { const response = await this._fetch(requestPayload); - const html = await response.text(); + let html = await response.text(); + + // Apply length limits + html = this.applyLengthLimits( + html, + requestPayload.max_length ?? 5000, + requestPayload.start_index ?? 0 + ); + return { content: [{ type: "text", text: html }], isError: false }; } catch (error) { return { @@ -52,8 +68,17 @@ export class Fetcher { try { const response = await this._fetch(requestPayload); const json = await response.json(); + let jsonString = JSON.stringify(json); + + // Apply length limits + jsonString = this.applyLengthLimits( + jsonString, + requestPayload.max_length ?? 5000, + requestPayload.start_index ?? 0 + ); + return { - content: [{ type: "text", text: JSON.stringify(json) }], + content: [{ type: "text", text: jsonString }], isError: false, }; } catch (error) { @@ -78,8 +103,14 @@ export class Fetcher { Array.from(styles).forEach((style) => style.remove()); const text = document.body.textContent || ""; - - const normalizedText = text.replace(/\s+/g, " ").trim(); + let normalizedText = text.replace(/\s+/g, " ").trim(); + + // Apply length limits + normalizedText = this.applyLengthLimits( + normalizedText, + requestPayload.max_length ?? 5000, + requestPayload.start_index ?? 0 + ); return { content: [{ type: "text", text: normalizedText }], @@ -98,7 +129,15 @@ export class Fetcher { const response = await this._fetch(requestPayload); const html = await response.text(); const turndownService = new TurndownService(); - const markdown = turndownService.turndown(html); + let markdown = turndownService.turndown(html); + + // Apply length limits + markdown = this.applyLengthLimits( + markdown, + requestPayload.max_length ?? 5000, + requestPayload.start_index ?? 0 + ); + return { content: [{ type: "text", text: markdown }], isError: false }; } catch (error) { return { diff --git a/src/index.ts b/src/index.ts index 03c076f..86bb5b4 100644 --- a/src/index.ts +++ b/src/index.ts @@ -39,6 +39,14 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { type: "object", description: "Optional headers to include in the request", }, + max_length: { + type: "number", + description: "Maximum number of characters to return (default: 5000)", + }, + start_index: { + type: "number", + description: "Start content from this character index (default: 0)", + }, }, required: ["url"], }, @@ -57,6 +65,14 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { type: "object", description: "Optional headers to include in the request", }, + max_length: { + type: "number", + description: "Maximum number of characters to return (default: 5000)", + }, + start_index: { + type: "number", + description: "Start content from this character index (default: 0)", + }, }, required: ["url"], }, @@ -76,6 +92,14 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { type: "object", description: "Optional headers to include in the request", }, + max_length: { + type: "number", + description: "Maximum number of characters to return (default: 5000)", + }, + start_index: { + type: "number", + description: "Start content from this character index (default: 0)", + }, }, required: ["url"], }, @@ -94,6 +118,14 @@ server.setRequestHandler(ListToolsRequestSchema, async () => { type: "object", description: "Optional headers to include in the request", }, + max_length: { + type: "number", + description: "Maximum number of characters to return (default: 5000)", + }, + start_index: { + type: "number", + description: "Start content from this character index (default: 0)", + }, }, required: ["url"], }, diff --git a/src/types.ts b/src/types.ts index 58851e5..bff6404 100644 --- a/src/types.ts +++ b/src/types.ts @@ -3,6 +3,14 @@ import { z } from "zod"; export const RequestPayloadSchema = z.object({ url: z.string().url(), headers: z.record(z.string()).optional(), + max_length: z.number().int().min(1).optional().default(5000), + start_index: z.number().int().min(0).optional().default(0), }); -export type RequestPayload = z.infer; +// Make sure TypeScript treats the fields as optional with defaults +export type RequestPayload = { + url: string; + headers?: Record; + max_length?: number; + start_index?: number; +};