diff --git a/packages/feed/package.json b/packages/feed/package.json index da36184..d4a0741 100644 --- a/packages/feed/package.json +++ b/packages/feed/package.json @@ -16,6 +16,7 @@ }, "dependencies": { "@mozilla/readability": "^0.6.0", + "@profullstack/x402-client": "0.2.0", "fast-xml-parser": "^5.2.5", "linkedom": "^0.18.13" } diff --git a/packages/feed/src/fetch.js b/packages/feed/src/fetch.js index ed75a56..b0b14d9 100644 --- a/packages/feed/src/fetch.js +++ b/packages/feed/src/fetch.js @@ -2,6 +2,7 @@ import dns from 'node:dns/promises'; import net from 'node:net'; import { looksLikeFeed, normalizeUrl, findFeedLinks, guessFeedUrls } from './discover.js'; +import { paidFetch } from './paid.js'; import { parseFeed } from './parse.js'; import { findPlaylistLinks } from './playlist.js'; @@ -145,7 +146,7 @@ export async function safeFetch(url, conditional = {}) { headers[name] = String(value); } - const res = await fetch(normalized, { + const res = await paidFetch(normalized, { headers, redirect: 'follow', signal: controller.signal, @@ -254,7 +255,7 @@ export async function safeFetchBytes(url, maxBytes = 64 * 1024) { const timer = setTimeout(() => controller.abort(), TIMEOUT_MS); try { - const res = await fetch(normalized, { + const res = await paidFetch(normalized, { headers: { 'user-agent': USER_AGENT, accept: 'image/*,*/*', diff --git a/packages/feed/src/frameable.js b/packages/feed/src/frameable.js index 7a6d0e6..422d352 100644 --- a/packages/feed/src/frameable.js +++ b/packages/feed/src/frameable.js @@ -1,5 +1,6 @@ import { normalizeUrl } from './discover.js'; import { isPublicHost } from './fetch.js'; +import { paidFetch } from './paid.js'; /** * Whether a page will load inside our reader's iframe. @@ -271,7 +272,7 @@ async function attempt(normalized, { origin, wantHtml, timeout }) { let timer = setTimeout(() => controller.abort(), timeout); try { - const res = await fetch(normalized, { + const res = await paidFetch(normalized, { method: 'GET', headers: { 'user-agent': USER_AGENT, accept: 'text/html,*/*' }, redirect: 'follow', diff --git a/packages/feed/src/paid.js b/packages/feed/src/paid.js new file mode 100644 index 0000000..c18eb9e --- /dev/null +++ b/packages/feed/src/paid.js @@ -0,0 +1,24 @@ +/** + * The fetch the crawler reaches other people's sites with — paying when asked. + * + * A site behind an x402 gateway answers a crawler with 402 and an offer. With + * `X402_PRIVATE_KEY` set, the client signs the offer with the shared crawler + * wallet, buys the pass, files it by origin and presents it on every later + * request to that site, so a gated site costs a dollar a day rather than + * being silently empty. Without the key this is the global fetch, unchanged. + * + * The ceiling is five dollars a payment: a hostile or misconfigured offer + * cannot drain the wallet by asking. Read through a non-literal accessor + * because Next inlines `process.env.NAME` at build time, and this module is + * shared by the poller and the web reader. + */ + +import { createClient } from '@profullstack/x402-client'; + +const key = globalThis.process?.env?.[['X402', 'PRIVATE_KEY'].join('_')]; + +/** The paying client, or null when no key is configured. */ +export const x402 = key ? createClient({ key, maxUsd: 5 }) : null; + +/** @type {typeof fetch} */ +export const paidFetch = x402 ? (input, init) => x402.fetch(input, init) : (input, init) => globalThis.fetch(input, init); diff --git a/packages/feed/test/paid.test.js b/packages/feed/test/paid.test.js new file mode 100644 index 0000000..8f4168a --- /dev/null +++ b/packages/feed/test/paid.test.js @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict'; +import { createServer } from 'node:http'; +import { test } from 'node:test'; + +import { paidFetch, x402 } from '../src/paid.js'; + +test('without X402_PRIVATE_KEY the crawler fetch is the plain fetch', async () => { + // The test process has no key; a process with one gets the paying client. + assert.equal(x402, null); + + const server = createServer((req, res) => { + res.writeHead(200, { 'content-type': 'text/plain' }); + res.end(`hello ${req.headers['user-agent'] ?? ''}`); + }); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const res = await paidFetch(`http://127.0.0.1:${server.address().port}/`, { headers: { 'user-agent': 'bot/1' } }); + assert.equal(res.status, 200); + assert.equal(await res.text(), 'hello bot/1'); + } finally { + await new Promise((resolve) => server.close(resolve)); + } +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 83b34fa..1ecd8b6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -139,6 +139,9 @@ importers: '@mozilla/readability': specifier: ^0.6.0 version: 0.6.0 + '@profullstack/x402-client': + specifier: 0.2.0 + version: 0.2.0 fast-xml-parser: specifier: ^5.2.5 version: 5.10.1 @@ -540,6 +543,14 @@ packages: cpu: [x64] os: [win32] + '@noble/curves@2.4.0': + resolution: {integrity: sha512-P4/62zrgfH33CneE3Dn4WhJVA22YUU0eR51wKIan4NVRvwsA0YnPTwWGpNbpuacSujmSFLvyzpyuR30+fbq2Ew==} + engines: {node: '>= 20.19.0'} + + '@noble/hashes@2.4.0': + resolution: {integrity: sha512-X5XaVWZIBCT7HHZGm5I7ZQXDwLG+bGXuSrMQAW+7Zvl87h1kmc1ZB1VSRJcpUfoUrGQp4Fkoxm5kZ+Ms+aW+eA==} + engines: {node: '>= 20.19.0'} + '@nodable/entities@3.0.0': resolution: {integrity: sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==} @@ -592,6 +603,11 @@ packages: react: optional: true + '@profullstack/x402-client@0.2.0': + resolution: {integrity: sha512-kJGAomE9Tf/ruhsCStaKVXpjklCbOh2UFcMGTIfTqcWaSsW2NFWQRaTxJBLvc6r7Iy2+fpAWzfJnerolI6IrRQ==} + engines: {node: '>=20.19'} + hasBin: true + '@profullstack/x402-gateway@0.3.0': resolution: {integrity: sha512-aaSMdtVInaAD6te/RnNnnqZW2+oo/dUsOTFTJCl0hOn58s85Yw3vZGk3KCtZmjrwV2O2rdTjKlZjbKp3CGgntw==} engines: {node: '>=20.11'} @@ -1222,6 +1238,12 @@ snapshots: '@next/swc-win32-x64-msvc@16.3.1': optional: true + '@noble/curves@2.4.0': + dependencies: + '@noble/hashes': 2.4.0 + + '@noble/hashes@2.4.0': {} + '@nodable/entities@3.0.0': {} '@peculiar/asn1-android@2.8.0': @@ -1331,6 +1353,11 @@ snapshots: optionalDependencies: react: 19.2.8 + '@profullstack/x402-client@0.2.0': + dependencies: + '@noble/curves': 2.4.0 + '@noble/hashes': 2.4.0 + '@profullstack/x402-gateway@0.3.0': {} '@simplewebauthn/browser@13.3.0': {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 39ea33b..effa365 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -14,3 +14,4 @@ allowBuilds: minimumReleaseAgeExclude: - '@profullstack/player@0.2.0 || 0.3.1' - '@profullstack/x402-gateway@0.1.0 || 0.2.1 || 0.3.0' + - '@profullstack/x402-client@0.2.0'