gitmanager/fetch-mcp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: initial commit

Browse Source
This commit is contained in:
Zach Caceres
2024-12-17 18:05:36 -07:00
commit 2dea4318b9
10 changed files with 3724 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
node_modules
dist

89
README.md Normal file
View File

@ -0,0 +1,89 @@
# Fetch MCP Server
This MCP server provides functionality to fetch web content in various formats, including HTML, JSON, plain text, and Markdown.
## Components
### Tools
- **fetch_html**
- Fetch a website and return the content as HTML
- Input:
- `url` (string, required): URL of the website to fetch
- `headers` (object, optional): Custom headers to include in the request
- Returns the raw HTML content of the webpage
- **fetch_json**
- Fetch a JSON file from a URL
- Input:
- `url` (string, required): URL of the JSON to fetch
- `headers` (object, optional): Custom headers to include in the request
- Returns the parsed JSON content
- **fetch_txt**
- Fetch a website and return the content as plain text (no HTML)
- Input:
- `url` (string, required): URL of the website to fetch
- `headers` (object, optional): Custom headers to include in the request
- Returns the text content of the webpage with HTML tags, scripts, and styles removed
- **fetch_markdown**
- Fetch a website and return the content as Markdown
- Input:
- `url` (string, required): URL of the website to fetch
- `headers` (object, optional): Custom headers to include in the request
- Returns the content of the webpage converted to Markdown format
### Resources
This server does not provide any persistent resources. It's designed to fetch and transform web content on demand.
## Getting started
1. Clone the repository
2. Install dependencies: `npm install`
3. Build the server: `npm run build`
### Usage
To use the server, you can run it directly:
```bash
npm start
```
This will start the Fetch MCP Server running on stdio.
### Usage with Desktop App
To integrate this server with a desktop app, add the following to your app's server configuration:
```json
{
"mcpServers": {
"fetch": {
"command": "node",
"args": [
"{ABSOLUTE PATH TO FILE HERE}/dist/index.js"
]
}
}
}
```
## Features
- Fetches web content using modern fetch API
- Supports custom headers for requests
- Provides content in multiple formats: HTML, JSON, plain text, and Markdown
- Uses JSDOM for HTML parsing and text extraction
- Uses TurndownService for HTML to Markdown conversion
## Development
- Run `npm run dev` to start the TypeScript compiler in watch mode
- Use `npm test` to run the test suite
## License
This project is licensed under the MIT License.

4
jest.config.js Normal file
View File

@ -0,0 +1,4 @@
export default {
preset: "ts-jest",
testEnvironment: "node",
};

32
package.json Normal file
View File

@ -0,0 +1,32 @@
{
"name": "fetch",
"version": "1.0.0",
"description": "",
"type": "module",
"main": "index.js",
"license": "MIT",
"author": "zcaceres (@zachcaceres zach.dev)",
"scripts": {
"build": "tsc && shx chmod +x dist/*.js",
"prepare": "npm run build",
"dev": "tsc --watch",
"start": "node dist/index.js",
"test": "jest"
},
"keywords": [],
"dependencies": {
"@modelcontextprotocol/sdk": "^1.0.4",
"jsdom": "^25.0.1",
"turndown": "^7.2.0",
"zod": "^3.24.1"
},
"devDependencies": {
"@types/jest": "^29.5.14",
"@types/jsdom": "^21.1.7",
"@types/node": "^22.10.2",
"@types/turndown": "^5.0.5",
"jest": "^29.7.0",
"ts-jest": "^29.2.5",
"typescript": "^5.7.2"
}
}

3124
pnpm-lock.yaml generated Normal file
View File

File diff suppressed because it is too large Load Diff

208
src/Fetcher.test.ts Normal file
View File

@ -0,0 +1,208 @@
import { Fetcher } from "./Fetcher";
import { JSDOM } from "jsdom";
import TurndownService from "turndown";
global.fetch = jest.fn();
jest.mock("jsdom");
jest.mock("turndown");
describe("Fetcher", () => {
beforeEach(() => {
jest.clearAllMocks();
});
const mockRequest = {
url: "https://example.com",
headers: { "Custom-Header": "Value" },
};
const mockHtml = `
<html>
<head>
<title>Test Page</title>
<script>console.log('This should be removed');</script>
<style>body { color: red; }</style>
</head>
<body>
<h1>Hello World</h1>
<p>This is a test paragraph.</p>
</body>
</html>
`;
describe("html", () => {
it("should return the raw HTML content", async () => {
(fetch as jest.Mock).mockResolvedValueOnce({
ok: true,
text: jest.fn().mockResolvedValueOnce(mockHtml),
});
const result = await Fetcher.html(mockRequest);
expect(result).toEqual({
content: [{ type: "text", text: mockHtml }],
isError: false,
});
});
it("should handle errors", async () => {
(fetch as jest.Mock).mockRejectedValueOnce(new Error("Network error"));
const result = await Fetcher.html(mockRequest);
expect(result).toEqual({
content: [
{
type: "text",
text: "Failed to fetch https://example.com: Network error",
},
],
isError: true,
});
});
});
describe("json", () => {
it("should parse and return JSON content", async () => {
const mockJson = { key: "value" };
(fetch as jest.Mock).mockResolvedValueOnce({
ok: true,
json: jest.fn().mockResolvedValueOnce(mockJson),
});
const result = await Fetcher.json(mockRequest);
expect(result).toEqual({
content: [{ type: "text", text: JSON.stringify(mockJson) }],
isError: false,
});
});
it("should handle errors", async () => {
(fetch as jest.Mock).mockRejectedValueOnce(new Error("Invalid JSON"));
const result = await Fetcher.json(mockRequest);
expect(result).toEqual({
content: [
{
type: "text",
text: "Failed to fetch https://example.com: Invalid JSON",
},
],
isError: true,
});
});
});
describe("txt", () => {
it("should return plain text content without HTML tags, scripts, and styles", async () => {
(fetch as jest.Mock).mockResolvedValueOnce({
ok: true,
text: jest.fn().mockResolvedValueOnce(mockHtml),
});
const mockTextContent = "Hello World This is a test paragraph.";
// @ts-expect-error Mocking JSDOM
(JSDOM as jest.Mock).mockImplementationOnce(() => ({
window: {
document: {
body: {
textContent: mockTextContent,
},
getElementsByTagName: jest.fn().mockReturnValue([]),
},
},
}));
const result = await Fetcher.txt(mockRequest);
expect(result).toEqual({
content: [{ type: "text", text: mockTextContent }],
isError: false,
});
});
it("should handle errors", async () => {
(fetch as jest.Mock).mockRejectedValueOnce(new Error("Parsing error"));
const result = await Fetcher.txt(mockRequest);
expect(result).toEqual({
content: [
{
type: "text",
text: "Failed to fetch https://example.com: Parsing error",
},
],
isError: true,
});
});
});
describe("markdown", () => {
it("should convert HTML to markdown", async () => {
(fetch as jest.Mock).mockResolvedValueOnce({
ok: true,
text: jest.fn().mockResolvedValueOnce(mockHtml),
});
const mockMarkdown = "# Hello World\n\nThis is a test paragraph.";
(TurndownService as jest.Mock).mockImplementationOnce(() => ({
turndown: jest.fn().mockReturnValueOnce(mockMarkdown),
}));
const result = await Fetcher.markdown(mockRequest);
expect(result).toEqual({
content: [{ type: "text", text: mockMarkdown }],
isError: false,
});
});
it("should handle errors", async () => {
(fetch as jest.Mock).mockRejectedValueOnce(new Error("Conversion error"));
const result = await Fetcher.markdown(mockRequest);
expect(result).toEqual({
content: [
{
type: "text",
text: "Failed to fetch https://example.com: Conversion error",
},
],
isError: true,
});
});
});
describe("error handling", () => {
it("should handle non-OK responses", async () => {
(fetch as jest.Mock).mockResolvedValueOnce({
ok: false,
status: 404,
});
const result = await Fetcher.html(mockRequest);
expect(result).toEqual({
content: [
{
type: "text",
text: "Failed to fetch https://example.com: HTTP error: 404",
},
],
isError: true,
});
});
it("should handle unknown errors", async () => {
(fetch as jest.Mock).mockRejectedValueOnce("Unknown error");
const result = await Fetcher.html(mockRequest);
expect(result).toEqual({
content: [
{
type: "text",
text: "Failed to fetch https://example.com: Unknown error",
},
],
isError: true,
});
});
});
});

104
src/Fetcher.ts Normal file
View File

@ -0,0 +1,104 @@
import { JSDOM } from "jsdom";
import TurndownService from "turndown";
import { RequestPayload } from "./types.js";
export class Fetcher {
private static async _fetch({
url,
headers,
}: RequestPayload): Promise<Response> {
try {
const response = await fetch(url, {
headers: {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
...headers,
},
});
if (!response.ok) {
throw new Error(`HTTP error: ${response.status}`);
}
return response;
} catch (e: unknown) {
if (e instanceof Error) {
throw new Error(`Failed to fetch ${url}: ${e.message}`);
} else {
throw new Error(`Failed to fetch ${url}: Unknown error`);
}
}
}
static async html(requestPayload: RequestPayload) {
try {
const response = await this._fetch(requestPayload);
const html = await response.text();
return { content: [{ type: "text", text: html }], isError: false };
} catch (error) {
return {
content: [{ type: "text", text: (error as Error).message }],
isError: true,
};
}
}
static async json(requestPayload: RequestPayload) {
try {
const response = await this._fetch(requestPayload);
const json = await response.json();
return {
content: [{ type: "text", text: JSON.stringify(json) }],
isError: false,
};
} catch (error) {
return {
content: [{ type: "text", text: (error as Error).message }],
isError: true,
};
}
}
static async txt(requestPayload: RequestPayload) {
try {
const response = await this._fetch(requestPayload);
const html = await response.text();
const dom = new JSDOM(html);
const document = dom.window.document;
const scripts = document.getElementsByTagName("script");
const styles = document.getElementsByTagName("style");
Array.from(scripts).forEach((script) => script.remove());
Array.from(styles).forEach((style) => style.remove());
const text = document.body.textContent || "";
const normalizedText = text.replace(/\s+/g, " ").trim();
return {
content: [{ type: "text", text: normalizedText }],
isError: false,
};
} catch (error) {
return {
content: [{ type: "text", text: (error as Error).message }],
isError: true,
};
}
}
static async markdown(requestPayload: RequestPayload) {
try {
const response = await this._fetch(requestPayload);
const html = await response.text();
const turndownService = new TurndownService();
const markdown = turndownService.turndown(html);
return { content: [{ type: "text", text: markdown }], isError: false };
} catch (error) {
return {
content: [{ type: "text", text: (error as Error).message }],
isError: true,
};
}
}
}

138
src/index.ts Normal file
View File

@ -0,0 +1,138 @@
#!/usr/bin/env node
import { Server } from "@modelcontextprotocol/sdk/server/index.js";
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
import {
CallToolRequestSchema,
ListToolsRequestSchema,
} from "@modelcontextprotocol/sdk/types.js";
import { RequestPayloadSchema } from "./types.js";
import { Fetcher } from "./Fetcher.js";
const server = new Server(
{
name: "zcaceres/fetch",
version: "0.1.0",
},
{
capabilities: {
resources: {},
tools: {},
},
},
);
server.setRequestHandler(ListToolsRequestSchema, async () => {
return {
tools: [
{
name: "fetch_html",
description: "Fetch a website and return the content as HTML",
inputSchema: {
type: "object",
properties: {
url: {
type: "string",
description: "URL of the website to fetch",
},
headers: {
type: "object",
description: "Optional headers to include in the request",
},
},
required: ["url"],
},
},
{
name: "fetch_markdown",
description: "Fetch a website and return the content as Markdown",
inputSchema: {
type: "object",
properties: {
url: {
type: "string",
description: "URL of the website to fetch",
},
headers: {
type: "object",
description: "Optional headers to include in the request",
},
},
required: ["url"],
},
},
{
name: "fetch_txt",
description:
"Fetch a website, return the content as plain text (no HTML)",
inputSchema: {
type: "object",
properties: {
url: {
type: "string",
description: "URL of the website to fetch",
},
headers: {
type: "object",
description: "Optional headers to include in the request",
},
},
required: ["url"],
},
},
{
name: "fetch_json",
description: "Fetch a JSON file from a URL",
inputSchema: {
type: "object",
properties: {
url: {
type: "string",
description: "URL of the JSON to fetch",
},
headers: {
type: "object",
description: "Optional headers to include in the request",
},
},
required: ["url"],
},
},
],
};
});
server.setRequestHandler(CallToolRequestSchema, async (request) => {
const { name, arguments: args } = request.params;
const validatedArgs = RequestPayloadSchema.parse(args);
if (request.params.name === "fetch_html") {
const fetchResult = await Fetcher.html(validatedArgs);
return fetchResult;
}
if (request.params.name === "fetch_json") {
const fetchResult = await Fetcher.json(validatedArgs);
return fetchResult;
}
if (request.params.name === "fetch_txt") {
const fetchResult = await Fetcher.txt(validatedArgs);
return fetchResult;
}
if (request.params.name === "fetch_markdown") {
const fetchResult = await Fetcher.markdown(validatedArgs);
return fetchResult;
}
throw new Error("Tool not found");
});
async function main() {
const transport = new StdioServerTransport();
await server.connect(transport);
console.log("Fetch MCP Server running on stdio");
}
main().catch((error) => {
console.error("Fatal error in main():", error);
process.exit(1);
});

8
src/types.ts Normal file
View File

@ -0,0 +1,8 @@
import { z } from "zod";
export const RequestPayloadSchema = z.object({
url: z.string().url(),
headers: z.record(z.string()).optional(),
});
export type RequestPayload = z.infer<typeof RequestPayloadSchema>;

15
tsconfig.json Normal file
View File

@ -0,0 +1,15 @@
{
"compilerOptions": {
"target": "ES2022",
"module": "Node16",
"moduleResolution": "node16",
"outDir": "./dist",
"rootDir": "./src",
"strict": true,
"esModuleInterop": true,
"skipLibCheck": true,
"forceConsistentCasingInFileNames": true
},
"include": ["src/**/*.ts"],
"exclude": ["node_modules", "dist", "src/**/*.test.ts"]
}