feat: initial commit

2024-12-17 18:05:36 -07:00
commit 2dea4318b9
10 changed files with 3724 additions and 0 deletions
--- a/.gitignore
+++ b/.gitignore
@ -0,0 +1,2 @@
+node_modules
+dist
--- a/README.md
+++ b/README.md
@ -0,0 +1,89 @@
+# Fetch MCP Server
+
+This MCP server provides functionality to fetch web content in various formats, including HTML, JSON, plain text, and Markdown.
+
+## Components
+
+### Tools
+
+- **fetch_html**
+  - Fetch a website and return the content as HTML
+  - Input:
+    - `url` (string, required): URL of the website to fetch
+    - `headers` (object, optional): Custom headers to include in the request
+  - Returns the raw HTML content of the webpage
+
+- **fetch_json**
+  - Fetch a JSON file from a URL
+  - Input:
+    - `url` (string, required): URL of the JSON to fetch
+    - `headers` (object, optional): Custom headers to include in the request
+  - Returns the parsed JSON content
+
+- **fetch_txt**
+  - Fetch a website and return the content as plain text (no HTML)
+  - Input:
+    - `url` (string, required): URL of the website to fetch
+    - `headers` (object, optional): Custom headers to include in the request
+  - Returns the text content of the webpage with HTML tags, scripts, and styles removed
+
+- **fetch_markdown**
+  - Fetch a website and return the content as Markdown
+  - Input:
+    - `url` (string, required): URL of the website to fetch
+    - `headers` (object, optional): Custom headers to include in the request
+  - Returns the content of the webpage converted to Markdown format
+
+### Resources
+
+This server does not provide any persistent resources. It's designed to fetch and transform web content on demand.
+
+## Getting started
+
+1. Clone the repository
+2. Install dependencies: `npm install`
+3. Build the server: `npm run build`
+
+### Usage
+
+To use the server, you can run it directly:
+
+```bash
+npm start
+```
+
+This will start the Fetch MCP Server running on stdio.
+
+### Usage with Desktop App
+
+To integrate this server with a desktop app, add the following to your app's server configuration:
+
+```json
+{
+  "mcpServers": {
+    "fetch": {
+      "command": "node",
+      "args": [
+        "{ABSOLUTE PATH TO FILE HERE}/dist/index.js"
+      ]
+    }
+  }
+}
+```
+
+## Features
+
+- Fetches web content using modern fetch API
+- Supports custom headers for requests
+- Provides content in multiple formats: HTML, JSON, plain text, and Markdown
+- Uses JSDOM for HTML parsing and text extraction
+- Uses TurndownService for HTML to Markdown conversion
+
+## Development
+
+- Run `npm run dev` to start the TypeScript compiler in watch mode
+- Use `npm test` to run the test suite
+
+## License
+
+This project is licensed under the MIT License.
--- a/jest.config.js
+++ b/jest.config.js
@ -0,0 +1,4 @@
+export default {
+  preset: "ts-jest",
+  testEnvironment: "node",
+};
--- a/package.json
+++ b/package.json
@ -0,0 +1,32 @@
+{
+  "name": "fetch",
+  "version": "1.0.0",
+  "description": "",
+  "type": "module",
+  "main": "index.js",
+  "license": "MIT",
+  "author": "zcaceres (@zachcaceres zach.dev)",
+  "scripts": {
+    "build": "tsc && shx chmod +x dist/*.js",
+    "prepare": "npm run build",
+    "dev": "tsc --watch",
+    "start": "node dist/index.js",
+    "test": "jest"
+  },
+  "keywords": [],
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.0.4",
+    "jsdom": "^25.0.1",
+    "turndown": "^7.2.0",
+    "zod": "^3.24.1"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.14",
+    "@types/jsdom": "^21.1.7",
+    "@types/node": "^22.10.2",
+    "@types/turndown": "^5.0.5",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.2.5",
+    "typescript": "^5.7.2"
+  }
+}
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
--- a/src/Fetcher.test.ts
+++ b/src/Fetcher.test.ts
@ -0,0 +1,208 @@
+import { Fetcher } from "./Fetcher";
+import { JSDOM } from "jsdom";
+import TurndownService from "turndown";
+
+global.fetch = jest.fn();
+
+jest.mock("jsdom");
+
+jest.mock("turndown");
+
+describe("Fetcher", () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  const mockRequest = {
+    url: "https://example.com",
+    headers: { "Custom-Header": "Value" },
+  };
+
+  const mockHtml = `
+    <html>
+      <head>
+        <title>Test Page</title>
+        <script>console.log('This should be removed');</script>
+        <style>body { color: red; }</style>
+      </head>
+      <body>
+        <h1>Hello World</h1>
+        <p>This is a test paragraph.</p>
+      </body>
+    </html>
+  `;
+
+  describe("html", () => {
+    it("should return the raw HTML content", async () => {
+      (fetch as jest.Mock).mockResolvedValueOnce({
+        ok: true,
+        text: jest.fn().mockResolvedValueOnce(mockHtml),
+      });
+
+      const result = await Fetcher.html(mockRequest);
+      expect(result).toEqual({
+        content: [{ type: "text", text: mockHtml }],
+        isError: false,
+      });
+    });
+
+    it("should handle errors", async () => {
+      (fetch as jest.Mock).mockRejectedValueOnce(new Error("Network error"));
+
+      const result = await Fetcher.html(mockRequest);
+      expect(result).toEqual({
+        content: [
+          {
+            type: "text",
+            text: "Failed to fetch https://example.com: Network error",
+          },
+        ],
+        isError: true,
+      });
+    });
+  });
+
+  describe("json", () => {
+    it("should parse and return JSON content", async () => {
+      const mockJson = { key: "value" };
+      (fetch as jest.Mock).mockResolvedValueOnce({
+        ok: true,
+        json: jest.fn().mockResolvedValueOnce(mockJson),
+      });
+
+      const result = await Fetcher.json(mockRequest);
+      expect(result).toEqual({
+        content: [{ type: "text", text: JSON.stringify(mockJson) }],
+        isError: false,
+      });
+    });
+
+    it("should handle errors", async () => {
+      (fetch as jest.Mock).mockRejectedValueOnce(new Error("Invalid JSON"));
+
+      const result = await Fetcher.json(mockRequest);
+      expect(result).toEqual({
+        content: [
+          {
+            type: "text",
+            text: "Failed to fetch https://example.com: Invalid JSON",
+          },
+        ],
+        isError: true,
+      });
+    });
+  });
+
+  describe("txt", () => {
+    it("should return plain text content without HTML tags, scripts, and styles", async () => {
+      (fetch as jest.Mock).mockResolvedValueOnce({
+        ok: true,
+        text: jest.fn().mockResolvedValueOnce(mockHtml),
+      });
+
+      const mockTextContent = "Hello World This is a test paragraph.";
+      // @ts-expect-error Mocking JSDOM
+      (JSDOM as jest.Mock).mockImplementationOnce(() => ({
+        window: {
+          document: {
+            body: {
+              textContent: mockTextContent,
+            },
+            getElementsByTagName: jest.fn().mockReturnValue([]),
+          },
+        },
+      }));
+
+      const result = await Fetcher.txt(mockRequest);
+      expect(result).toEqual({
+        content: [{ type: "text", text: mockTextContent }],
+        isError: false,
+      });
+    });
+
+    it("should handle errors", async () => {
+      (fetch as jest.Mock).mockRejectedValueOnce(new Error("Parsing error"));
+
+      const result = await Fetcher.txt(mockRequest);
+      expect(result).toEqual({
+        content: [
+          {
+            type: "text",
+            text: "Failed to fetch https://example.com: Parsing error",
+          },
+        ],
+        isError: true,
+      });
+    });
+  });
+
+  describe("markdown", () => {
+    it("should convert HTML to markdown", async () => {
+      (fetch as jest.Mock).mockResolvedValueOnce({
+        ok: true,
+        text: jest.fn().mockResolvedValueOnce(mockHtml),
+      });
+
+      const mockMarkdown = "# Hello World\n\nThis is a test paragraph.";
+      (TurndownService as jest.Mock).mockImplementationOnce(() => ({
+        turndown: jest.fn().mockReturnValueOnce(mockMarkdown),
+      }));
+
+      const result = await Fetcher.markdown(mockRequest);
+      expect(result).toEqual({
+        content: [{ type: "text", text: mockMarkdown }],
+        isError: false,
+      });
+    });
+
+    it("should handle errors", async () => {
+      (fetch as jest.Mock).mockRejectedValueOnce(new Error("Conversion error"));
+
+      const result = await Fetcher.markdown(mockRequest);
+      expect(result).toEqual({
+        content: [
+          {
+            type: "text",
+            text: "Failed to fetch https://example.com: Conversion error",
+          },
+        ],
+        isError: true,
+      });
+    });
+  });
+
+  describe("error handling", () => {
+    it("should handle non-OK responses", async () => {
+      (fetch as jest.Mock).mockResolvedValueOnce({
+        ok: false,
+        status: 404,
+      });
+
+      const result = await Fetcher.html(mockRequest);
+      expect(result).toEqual({
+        content: [
+          {
+            type: "text",
+            text: "Failed to fetch https://example.com: HTTP error: 404",
+          },
+        ],
+        isError: true,
+      });
+    });
+
+    it("should handle unknown errors", async () => {
+      (fetch as jest.Mock).mockRejectedValueOnce("Unknown error");
+
+      const result = await Fetcher.html(mockRequest);
+      expect(result).toEqual({
+        content: [
+          {
+            type: "text",
+            text: "Failed to fetch https://example.com: Unknown error",
+          },
+        ],
+        isError: true,
+      });
+    });
+  });
+});
--- a/src/Fetcher.ts
+++ b/src/Fetcher.ts
@ -0,0 +1,104 @@
+import { JSDOM } from "jsdom";
+import TurndownService from "turndown";
+import { RequestPayload } from "./types.js";
+
+export class Fetcher {
+  private static async _fetch({
+    url,
+    headers,
+  }: RequestPayload): Promise<Response> {
+    try {
+      const response = await fetch(url, {
+        headers: {
+          "User-Agent":
+            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
+          ...headers,
+        },
+      });
+
+      if (!response.ok) {
+        throw new Error(`HTTP error: ${response.status}`);
+      }
+      return response;
+    } catch (e: unknown) {
+      if (e instanceof Error) {
+        throw new Error(`Failed to fetch ${url}: ${e.message}`);
+      } else {
+        throw new Error(`Failed to fetch ${url}: Unknown error`);
+      }
+    }
+  }
+
+  static async html(requestPayload: RequestPayload) {
+    try {
+      const response = await this._fetch(requestPayload);
+      const html = await response.text();
+      return { content: [{ type: "text", text: html }], isError: false };
+    } catch (error) {
+      return {
+        content: [{ type: "text", text: (error as Error).message }],
+        isError: true,
+      };
+    }
+  }
+
+  static async json(requestPayload: RequestPayload) {
+    try {
+      const response = await this._fetch(requestPayload);
+      const json = await response.json();
+      return {
+        content: [{ type: "text", text: JSON.stringify(json) }],
+        isError: false,
+      };
+    } catch (error) {
+      return {
+        content: [{ type: "text", text: (error as Error).message }],
+        isError: true,
+      };
+    }
+  }
+
+  static async txt(requestPayload: RequestPayload) {
+    try {
+      const response = await this._fetch(requestPayload);
+      const html = await response.text();
+
+      const dom = new JSDOM(html);
+      const document = dom.window.document;
+
+      const scripts = document.getElementsByTagName("script");
+      const styles = document.getElementsByTagName("style");
+      Array.from(scripts).forEach((script) => script.remove());
+      Array.from(styles).forEach((style) => style.remove());
+
+      const text = document.body.textContent || "";
+
+      const normalizedText = text.replace(/\s+/g, " ").trim();
+
+      return {
+        content: [{ type: "text", text: normalizedText }],
+        isError: false,
+      };
+    } catch (error) {
+      return {
+        content: [{ type: "text", text: (error as Error).message }],
+        isError: true,
+      };
+    }
+  }
+
+  static async markdown(requestPayload: RequestPayload) {
+    try {
+      const response = await this._fetch(requestPayload);
+      const html = await response.text();
+      const turndownService = new TurndownService();
+      const markdown = turndownService.turndown(html);
+      return { content: [{ type: "text", text: markdown }], isError: false };
+    } catch (error) {
+      return {
+        content: [{ type: "text", text: (error as Error).message }],
+        isError: true,
+      };
+    }
+  }
+}
--- a/src/index.ts
+++ b/src/index.ts
@ -0,0 +1,138 @@
+#!/usr/bin/env node
+
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
+import {
+  CallToolRequestSchema,
+  ListToolsRequestSchema,
+} from "@modelcontextprotocol/sdk/types.js";
+import { RequestPayloadSchema } from "./types.js";
+import { Fetcher } from "./Fetcher.js";
+
+const server = new Server(
+  {
+    name: "zcaceres/fetch",
+    version: "0.1.0",
+  },
+  {
+    capabilities: {
+      resources: {},
+      tools: {},
+    },
+  },
+);
+
+server.setRequestHandler(ListToolsRequestSchema, async () => {
+  return {
+    tools: [
+      {
+        name: "fetch_html",
+        description: "Fetch a website and return the content as HTML",
+        inputSchema: {
+          type: "object",
+          properties: {
+            url: {
+              type: "string",
+              description: "URL of the website to fetch",
+            },
+            headers: {
+              type: "object",
+              description: "Optional headers to include in the request",
+            },
+          },
+          required: ["url"],
+        },
+      },
+      {
+        name: "fetch_markdown",
+        description: "Fetch a website and return the content as Markdown",
+        inputSchema: {
+          type: "object",
+          properties: {
+            url: {
+              type: "string",
+              description: "URL of the website to fetch",
+            },
+            headers: {
+              type: "object",
+              description: "Optional headers to include in the request",
+            },
+          },
+          required: ["url"],
+        },
+      },
+      {
+        name: "fetch_txt",
+        description:
+          "Fetch a website, return the content as plain text (no HTML)",
+        inputSchema: {
+          type: "object",
+          properties: {
+            url: {
+              type: "string",
+              description: "URL of the website to fetch",
+            },
+            headers: {
+              type: "object",
+              description: "Optional headers to include in the request",
+            },
+          },
+          required: ["url"],
+        },
+      },
+      {
+        name: "fetch_json",
+        description: "Fetch a JSON file from a URL",
+        inputSchema: {
+          type: "object",
+          properties: {
+            url: {
+              type: "string",
+              description: "URL of the JSON to fetch",
+            },
+            headers: {
+              type: "object",
+              description: "Optional headers to include in the request",
+            },
+          },
+          required: ["url"],
+        },
+      },
+    ],
+  };
+});
+
+server.setRequestHandler(CallToolRequestSchema, async (request) => {
+  const { name, arguments: args } = request.params;
+
+  const validatedArgs = RequestPayloadSchema.parse(args);
+
+  if (request.params.name === "fetch_html") {
+    const fetchResult = await Fetcher.html(validatedArgs);
+    return fetchResult;
+  }
+  if (request.params.name === "fetch_json") {
+    const fetchResult = await Fetcher.json(validatedArgs);
+    return fetchResult;
+  }
+  if (request.params.name === "fetch_txt") {
+    const fetchResult = await Fetcher.txt(validatedArgs);
+    return fetchResult;
+  }
+  if (request.params.name === "fetch_markdown") {
+    const fetchResult = await Fetcher.markdown(validatedArgs);
+    return fetchResult;
+  }
+  throw new Error("Tool not found");
+});
+
+async function main() {
+  const transport = new StdioServerTransport();
+  await server.connect(transport);
+  console.log("Fetch MCP Server running on stdio");
+}
+
+main().catch((error) => {
+  console.error("Fatal error in main():", error);
+  process.exit(1);
+});
--- a/src/types.ts
+++ b/src/types.ts
@ -0,0 +1,8 @@
+import { z } from "zod";
+
+export const RequestPayloadSchema = z.object({
+  url: z.string().url(),
+  headers: z.record(z.string()).optional(),
+});
+
+export type RequestPayload = z.infer<typeof RequestPayloadSchema>;
--- a/tsconfig.json
+++ b/tsconfig.json
@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "Node16",
+    "moduleResolution": "node16",
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true
+  },
+  "include": ["src/**/*.ts"],
+  "exclude": ["node_modules", "dist", "src/**/*.test.ts"]
+}