Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a243928436 | ||
|
|
c3adbc8704 | ||
|
|
811c99a548 | ||
|
|
2c7f5d4a8f | ||
|
|
53d4d90dae | ||
|
|
0dc59ccf9f | ||
|
|
f30b8c80e2 | ||
|
|
bea29211a3 | ||
|
|
a900c439a7 | ||
|
|
a7eac82fd7 | ||
|
|
b5ab1d130f | ||
|
|
4476e4d415 | ||
|
|
bba36067b5 | ||
|
|
e0db53ee2f | ||
|
|
b744ea88e6 | ||
|
|
ff245faf61 | ||
|
|
453566bf1f | ||
|
|
ec1fc3973f | ||
|
|
a56918d7d4 | ||
|
|
4fcf9b443b | ||
|
|
fe539db6f6 | ||
|
|
c25b51ed86 | ||
|
|
c800b605be | ||
|
|
8b5d7ed8bb | ||
|
|
c5b106be9a | ||
|
|
e807422754 | ||
|
|
4b17675310 | ||
|
|
9038dd005b | ||
|
|
9bfbb89781 | ||
|
|
ab57cf6cf3 | ||
|
|
bfcacf6ea4 | ||
|
|
ed08498f95 | ||
|
|
af1f94898e | ||
|
|
cb2daab6e4 | ||
|
|
6c83cd4a85 | ||
|
|
27916d3730 | ||
|
|
75004773dc | ||
|
|
b8ba228e92 | ||
|
|
d289c14f43 | ||
|
|
85ee76779f | ||
|
|
18f3db96a6 | ||
|
|
27296dab0d | ||
|
|
5a6bc96bdb | ||
|
|
20e7919543 | ||
|
|
dfb09fe89c | ||
|
|
44f8904de5 | ||
|
|
56014400ab | ||
|
|
e6e23c4f81 | ||
|
|
9b972ade24 | ||
|
|
b1302e10ed | ||
|
|
0143c64882 | ||
|
|
fd845e0847 | ||
|
|
71c4be886f | ||
|
|
c15230399a | ||
|
|
d9e6325731 | ||
|
|
e58497dbd1 | ||
|
|
b0c45df32b | ||
|
|
9b97317b1e | ||
|
|
367a633323 | ||
|
|
2d2e71318a | ||
|
|
d42b96cad2 | ||
|
|
5f2ca05b6f | ||
|
|
4afafe2d8c | ||
|
|
fdb785f4d2 | ||
|
|
679f0353b6 | ||
|
|
222259d5c2 | ||
|
|
b326daa423 | ||
|
|
c6d3bc1dcf | ||
|
|
14905bbb47 | ||
|
|
3e2f2a8770 | ||
|
|
5e1824c8a4 | ||
|
|
6ceb325fb5 | ||
|
|
8d79cb40b0 | ||
|
|
f571f737f1 | ||
|
|
c98dbe0bef | ||
|
|
10d6a5aedd | ||
|
|
d722de4bc6 | ||
|
|
5812c77e1b | ||
|
|
2f3ada4b9f | ||
|
|
6e2ea78bb9 | ||
|
|
117a5ec3c1 | ||
|
|
a7dcd27fe5 | ||
|
|
837ac38fd4 | ||
|
|
0e3b3df3f5 | ||
|
|
8af283d185 | ||
|
|
198b0e4c47 | ||
|
|
5ca9f6413c | ||
|
|
b78abb41bb | ||
|
|
74d92fdfde | ||
|
|
ae9e7faf7b | ||
|
|
936c8f6e2c | ||
|
|
9fc15d514d | ||
|
|
8b95c26d8b | ||
|
|
d019cc0bb4 | ||
|
|
f0f01ac34a | ||
|
|
bdcc82ccde | ||
|
|
5ed14ecf79 | ||
|
|
69d25a992d | ||
|
|
26af3f0b2a | ||
|
|
14b70c362a | ||
|
|
680f9c0232 | ||
|
|
d4a3c747ca | ||
|
|
dc8598c583 | ||
|
|
83c25d6112 | ||
|
|
5728daab98 | ||
|
|
b0f6a4e856 | ||
|
|
156bd0c4a8 | ||
|
|
92c8ba2026 | ||
|
|
0ef4851e2d | ||
|
|
7aaa8ced89 | ||
|
|
a88ff3a419 | ||
|
|
b7fc890486 | ||
|
|
2138ce6f7a | ||
|
|
72947e18ba | ||
|
|
4003b09c58 | ||
|
|
27ec172fcc | ||
|
|
6169d0744d | ||
|
|
2579432c2f | ||
|
|
4c00500791 | ||
|
|
5b78a049ca | ||
|
|
9111c91d58 | ||
|
|
acd8c69f00 | ||
|
|
84b20b4efa | ||
|
|
a7fa25c2f7 | ||
|
|
43e89d157d | ||
|
|
290183a593 | ||
|
|
2ff6a3cf4b | ||
|
|
4c40c32df9 | ||
|
|
0deedf5f26 | ||
|
|
1fcc889a12 | ||
|
|
fe94f00b0c | ||
|
|
5cf65688f6 | ||
|
|
e241794c7b | ||
|
|
c24b570761 | ||
|
|
4517db58c7 | ||
|
|
68c5d48684 | ||
|
|
90f2bd9d2d | ||
|
|
d59f135f7a | ||
|
|
6a4fe8bf77 | ||
|
|
5047f79bf8 | ||
|
|
335387989e | ||
|
|
eef7b01ad9 | ||
|
|
ce52fe40ef | ||
|
|
a493547eea | ||
|
|
7da704ebc4 | ||
|
|
13946bf269 | ||
|
|
787cb4be2c | ||
|
|
64613cb23e | ||
|
|
2c5241d69d |
No files matched your search
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"name": "podcastindex-web-ui",
|
||||
"image": "mcr.microsoft.com/devcontainers/javascript-node:1-16-bullseye",
|
||||
"features": {
|
||||
"ghcr.io/devcontainers/features/sshd:1": {
|
||||
"version": "latest"
|
||||
}
|
||||
},
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
"extensions": [
|
||||
"esbenp.prettier-vscode"
|
||||
]
|
||||
}
|
||||
},
|
||||
"forwardPorts": [9001, 5001],
|
||||
"portsAttributes": {
|
||||
"9001": {
|
||||
"label": "webpack-dev-server (UI)",
|
||||
"onAutoForward": "openPreview"
|
||||
},
|
||||
"5001": {
|
||||
"label": "express API server",
|
||||
"onAutoForward": "notify"
|
||||
}
|
||||
},
|
||||
"containerEnv": {
|
||||
"COREPACK_ENABLE_DOWNLOAD_PROMPT": "0"
|
||||
},
|
||||
"postCreateCommand": "sudo corepack enable && corepack prepare [email protected] --activate && { [ -f .env ] || cp .env-example .env; } && { [ -f .environments/.env.development ] || cp .env-example .environments/.env.development; } && yarn install",
|
||||
"remoteUser": "node"
|
||||
}
|
||||
@@ -12,11 +12,11 @@ jobs:
|
||||
# permissions:
|
||||
# issues: write # required for peter-evans/create-issue-from-file
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7.0.1
|
||||
|
||||
- name: Link Checker
|
||||
id: lychee
|
||||
uses: lycheeverse/lychee-action@a8c4c7cb88f0c7386610c35eb25108e448569cb0 # for v2.7.0
|
||||
uses: lycheeverse/lychee-action@e7477775783ea5526144ba13e8db5eec57747ce8 # for v2.9.0
|
||||
with:
|
||||
fail: false
|
||||
args: |
|
||||
@@ -24,9 +24,9 @@ jobs:
|
||||
--exclude-all-private
|
||||
--base-url https://podcastindex.org
|
||||
--remap 'https://developer.mozilla.org/docs/Web/HTTP/Guides/CORS https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/CORS'
|
||||
--include-verbatim --include-wikilinks --include-fragments
|
||||
--include-verbatim --include-fragments
|
||||
'./**/*.md' './**/*.tsx' './**/*.json' './ui/**/*.html'
|
||||
|
||||
#--include-wikilinks
|
||||
# --accept '100..=103,200..=299'
|
||||
# --root-dir "$(pwd)"
|
||||
# --suggest # Suggest link replacements for broken links, using a web archive
|
||||
|
||||
@@ -10,10 +10,10 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7.0.1
|
||||
|
||||
- name: Validate JSON
|
||||
uses: nhalstead/[email protected].3
|
||||
uses: nhalstead/validate-json-action@v0.1.12
|
||||
with:
|
||||
# Relative file path under the repository of a JSON schema file to validate the other JSON files with.
|
||||
schema: server/data/apps-schema.json
|
||||
|
||||
@@ -16,10 +16,10 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
|
||||
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
||||
|
||||
- name: Validate JSON
|
||||
uses: dsanders11/json-schema-validate-action@79b2d95a446a0d786f7f244ae7764f8370ff3657 # v1.4.0
|
||||
uses: dsanders11/json-schema-validate-action@f04ef3bca791388d2bd9ff3a50b27d1b2572158e # v2.1.0
|
||||
with:
|
||||
schema: ./server/data/apps-schema.json
|
||||
files: ./server/data/apps.json
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
services:
|
||||
init:
|
||||
image: node:16
|
||||
command: [ "bash", "-c", "corepack enable && yarn install" ]
|
||||
volumes:
|
||||
- .:/app
|
||||
working_dir: /app
|
||||
|
||||
dev:
|
||||
image: node:16
|
||||
command: [ "yarn", "run", "dev:docker"]
|
||||
environment:
|
||||
- PROXY_TARGET=http://server:5001
|
||||
ports:
|
||||
- "9001:9001"
|
||||
volumes:
|
||||
- .:/app
|
||||
working_dir: /app
|
||||
depends_on:
|
||||
init:
|
||||
condition: service_completed_successfully
|
||||
restart: on-failure
|
||||
|
||||
server:
|
||||
image: node:16
|
||||
command: [ "yarn", "run", "start" ]
|
||||
# ports:
|
||||
# - "5001:5001"
|
||||
volumes:
|
||||
- .:/app
|
||||
working_dir: /app
|
||||
depends_on:
|
||||
init:
|
||||
condition: service_completed_successfully
|
||||
restart: on-failure
|
||||
@@ -85,6 +85,7 @@
|
||||
"production": "NODE_ENV='production' && webpack --env.file=production --mode production && node server",
|
||||
"start": "node server",
|
||||
"test": "react-scripts test --env=jsdom",
|
||||
"test:server": "node server/test/well-known-signing-directory.test.js",
|
||||
"eject": "react-scripts eject",
|
||||
"flow": "flow",
|
||||
"format": "prettier --write 'src/**/*.js'",
|
||||
|
||||
|
After Width: | Height: | Size: 16 KiB |
|
After Width: | Height: | Size: 13 KiB |
|
After Width: | Height: | Size: 841 KiB |
|
After Width: | Height: | Size: 415 KiB |
|
After Width: | Height: | Size: 3.7 KiB |
|
After Width: | Height: | Size: 21 KiB |
|
After Width: | Height: | Size: 8.2 KiB |
|
After Width: | Height: | Size: 19 KiB |
|
After Width: | Height: | Size: 12 KiB |
|
After Width: | Height: | Size: 69 KiB |
|
After Width: | Height: | Size: 18 KiB |
|
After Width: | Height: | Size: 1.7 MiB |
|
After Width: | Height: | Size: 22 KiB |
|
After Width: | Height: | Size: 34 KiB |
|
Before Width: | Height: | Size: 17 KiB |
|
After Width: | Height: | Size: 14 KiB |
|
After Width: | Height: | Size: 41 KiB |
|
After Width: | Height: | Size: 93 KiB |
|
After Width: | Height: | Size: 61 KiB |
|
After Width: | Height: | Size: 635 KiB |
|
After Width: | Height: | Size: 409 KiB |
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"keys": [
|
||||
{
|
||||
"kty": "OKP",
|
||||
"crv": "Ed25519",
|
||||
"x": "REPLACE_ME",
|
||||
"kid": "REPLACE_ME",
|
||||
"use": "sig"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"title": "Public Data",
|
||||
"tagline": "Open datasets generated and published by the Podcast Index project.",
|
||||
"intro": "Podcast Index produces a number of public data exports - full feed catalogs, recommendation lists, value-block dumps, activity stats, and infrastructure pointers - all served as static files under public.podcastindex.org and a few sibling subdomains. They are free to mirror and use; please don't hammer them in tight loops, and prefer the SQLite/CSV catalog dumps over live API scraping when you need bulk data.",
|
||||
"sections": [
|
||||
{
|
||||
"id": "catalog",
|
||||
"name": "Catalog snapshots",
|
||||
"description": "Full feed catalogs. The SQLite dumps are the canonical bulk artifacts - load them directly with sqlite3 or any client.",
|
||||
"datasets": [
|
||||
{
|
||||
"name": "All live feeds - SQLite",
|
||||
"url": "https://public.podcastindex.org/podcastindex_feeds.db.tgz",
|
||||
"format": "tar.gz (SQLite)",
|
||||
"cadence": "weekly",
|
||||
"description": "Every feed currently being polled, with title, URL, owner, iTunes ID, host, language, categories, generator, and lastest-episode timestamps. The most heavily used artifact downstream - most third-party mirrors of the Podcast Index data start here."
|
||||
},
|
||||
{
|
||||
"name": "Dead feeds - CSV (id only)",
|
||||
"url": "https://public.podcastindex.org/podcastindex_dead_feeds.csv",
|
||||
"format": "CSV",
|
||||
"cadence": "daily",
|
||||
"description": "Lightweight CSV of dead feed IDs and the ID of the feed they duplicate (if any). Use this if you only need to reconcile your own list of feeds against the dead set."
|
||||
},
|
||||
{
|
||||
"name": "Problematic feeds - CSV",
|
||||
"url": "https://public.podcastindex.org/podcastindex_problematic_feeds.csv",
|
||||
"format": "CSV",
|
||||
"cadence": "daily",
|
||||
"description": "Dead feeds that have an entry in the problematic-feeds table. Includes feed URL, generator, author, owner, iTunes ID, the reason code, and the editor's note. Hosting-company brand names are stripped from the note column for cleaner pattern analysis."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "discovery",
|
||||
"name": "Discovery & recommendations",
|
||||
"description": "Curated and computed lists derived from podcast 2.0 tags and feed metadata.",
|
||||
"datasets": [
|
||||
{
|
||||
"name": "Recommendations - all media",
|
||||
"url": "https://public.podcastindex.org/recommendations.json",
|
||||
"format": "JSON",
|
||||
"cadence": "daily",
|
||||
"description": "Show recommendations mined from <podcast:podroll> tags across the whole index, ranked by mention count."
|
||||
},
|
||||
{
|
||||
"name": "Recommendations - video only",
|
||||
"url": "https://public.podcastindex.org/recommendations_video.json",
|
||||
"format": "JSON",
|
||||
"cadence": "daily",
|
||||
"description": "The same recommendation graph, filtered to feeds whose <podcast:medium> or content profile marks them as video."
|
||||
},
|
||||
{
|
||||
"name": "Recommendations - music only",
|
||||
"url": "https://public.podcastindex.org/recommendations_music.json",
|
||||
"format": "JSON",
|
||||
"cadence": "daily",
|
||||
"description": "The same recommendation graph, filtered to feeds whose <podcast:medium> or content profile marks them as music."
|
||||
},
|
||||
{
|
||||
"name": "Podcast locations",
|
||||
"url": "https://public.podcastindex.org/locations.json",
|
||||
"format": "JSON",
|
||||
"cadence": "daily",
|
||||
"description": "All live feeds carrying a <podcast:location> tag, with the place name, rel type (subject or creator), OpenStreetMap object ID, ISO country code, and latitude/longitude from the tag."
|
||||
},
|
||||
{
|
||||
"name": "Value-enabled feeds - CSV",
|
||||
"url": "https://public.podcastindex.org/valueEnabledFeeds.csv",
|
||||
"format": "CSV",
|
||||
"cadence": "daily",
|
||||
"description": "All feeds carrying a <podcast:value> block, grouped by host. Useful for surveying Lightning/Hive value-for-value adoption across the index."
|
||||
},
|
||||
{
|
||||
"name": "Hosting provider stats",
|
||||
"url": "https://public.podcastindex.org/hosting_stats.json",
|
||||
"format": "JSON",
|
||||
"cadence": "daily",
|
||||
"description": "Top hosting providers by live feed count. A quick read on the shape of the podcasting hosting market."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "activity",
|
||||
"name": "Activity & stats",
|
||||
"description": "Time-series counts and rolling new-feed reports.",
|
||||
"datasets": [
|
||||
{
|
||||
"name": "24-hour feed report - HTML",
|
||||
"url": "https://public.podcastindex.org/24hourFeedReport.html",
|
||||
"format": "HTML",
|
||||
"cadence": "daily",
|
||||
"description": "Human-readable daily report: counts of new feeds added in the last 24 hours, top generators, hour-by-hour add rate, and keyword-based scoring of the new-arrival mix."
|
||||
},
|
||||
{
|
||||
"name": "Newly-added feeds (last 24h) - CSV",
|
||||
"url": "https://public.podcastindex.org/newlyAddedFeeds24hours.csv",
|
||||
"format": "CSV",
|
||||
"cadence": "daily",
|
||||
"description": "Sliding-window CSV of every feed added to the index in the past 24 hours. Datestamped daily snapshots are also published as newlyAddedFeeds24hoursYYYY-MM-DD.csv."
|
||||
},
|
||||
{
|
||||
"name": "Hourly counts",
|
||||
"url": "https://stats.podcastindex.org/hourly_counts.json",
|
||||
"format": "JSON",
|
||||
"cadence": "hourly",
|
||||
"description": "Hour-bucketed aggregate counts: feeds added, episodes added, and API requests served. Useful for charting recent index activity."
|
||||
},
|
||||
{
|
||||
"name": "Daily counts",
|
||||
"url": "https://stats.podcastindex.org/daily_counts.json",
|
||||
"format": "JSON",
|
||||
"cadence": "daily",
|
||||
"description": "Day-bucketed aggregate counts with a longer retention window than the hourly file - better for long-term trend lines."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "infrastructure",
|
||||
"name": "Infrastructure feeds",
|
||||
"description": "Operational pointers used by clients of the Podcast Index ecosystem.",
|
||||
"datasets": [
|
||||
{
|
||||
"name": "Healthy Hive RPC nodes",
|
||||
"url": "https://public.podcastindex.org/hivenodes.txt",
|
||||
"format": "text",
|
||||
"cadence": "every few minutes",
|
||||
"description": "Plain-text list of Hive blockchain RPC nodes that have recently passed a health check. Consumed by Podping listeners that need a working node to subscribe to."
|
||||
},
|
||||
{
|
||||
"name": "Live tracking firehose - current window",
|
||||
"url": "https://tracking.podcastindex.org/current",
|
||||
"format": "JSON",
|
||||
"cadence": "every few minutes",
|
||||
"description": "Most recent ~200-second window of newly-added or updated feeds and items, as JSON. Polled by partner mobile apps to keep their local catalogs current. Per-window archives are also written under /<unix-timestamp>."
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -109,6 +109,50 @@ app.get('/.well-known/lnurlp/podcastindex', (req, res) => {
|
||||
res.redirect('https://getalby.com/.well-known/lnurlp/podcastindex')
|
||||
})
|
||||
|
||||
// ------------------------------------------------
|
||||
// ------ Web Bot Auth key directory (JWKS) -------
|
||||
// ------------------------------------------------
|
||||
//
|
||||
// Publishes the PUBLIC Ed25519 key(s) the Aggrivator crawler uses to sign its
|
||||
// requests, so Cloudflare and other verifiers can confirm a signed request
|
||||
// really comes from us (Web Bot Auth / RFC 9421 HTTP Message Signatures).
|
||||
//
|
||||
// The body is served from server/data/http-message-signatures-directory.json so
|
||||
// that adding/rotating a key is a content edit, not a code change. The exact
|
||||
// media type below is mandatory — verifiers reject application/json.
|
||||
//
|
||||
// TODO(ops): replace the REPLACE_ME placeholder values in that JSON file with
|
||||
// the real JWKS produced on the signing server. Never put private key material
|
||||
// (a "d" member) in that file. The `kid` must match the crawler's advertised
|
||||
// keyid.
|
||||
//
|
||||
// CDN/WAF: this endpoint must be reachable by automated clients with no bot
|
||||
// challenge, redirect, or auth. The app applies no such gating to this path
|
||||
// (the POW middleware only guards /api/*), but if Cloudflare (or any WAF) is in
|
||||
// front of the site, ensure this exact path is allowed through unchallenged.
|
||||
app.get('/.well-known/http-message-signatures-directory', (req, res) => {
|
||||
fs.readFile(
|
||||
'./server/data/http-message-signatures-directory.json',
|
||||
'utf8',
|
||||
(err, data) => {
|
||||
if (err) {
|
||||
res.status(500).json({ error: 'key directory unavailable' })
|
||||
return
|
||||
}
|
||||
// Use setHeader (raw Node) rather than res.set/res.type so Express does
|
||||
// not append "; charset=utf-8" — verifiers expect this media type exactly.
|
||||
res.setHeader(
|
||||
'Content-Type',
|
||||
'application/http-message-signatures-directory+json'
|
||||
)
|
||||
res.setHeader('Cache-Control', 'public, max-age=3600')
|
||||
// Send a Buffer, not a string: res.send() force-appends "; charset=utf-8"
|
||||
// to the Content-Type for string bodies, which we must avoid here.
|
||||
res.send(Buffer.from(data))
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
// ------------------------------------------------
|
||||
// ------------ Reverse proxy for API -------------
|
||||
// ------------------------------------------------
|
||||
@@ -222,28 +266,33 @@ app.use('/api/comments/byepisodeid', powMiddleware, async (req, res) => {
|
||||
|
||||
const sentCommenters = {}
|
||||
|
||||
const threadcap = await makeThreadcap(socialInteract[0].uri, {
|
||||
userAgent,
|
||||
cache,
|
||||
fetcher,
|
||||
})
|
||||
try {
|
||||
const threadcap = await makeThreadcap(socialInteract[0].uri, {
|
||||
userAgent,
|
||||
cache,
|
||||
fetcher,
|
||||
})
|
||||
|
||||
const callbacks = {
|
||||
onEvent: (e) => {
|
||||
if (e.kind === 'node-processed' && e.part === 'replies') {
|
||||
writeThreadcapChunk(e.nodeId, threadcap, sentCommenters, res)
|
||||
}
|
||||
},
|
||||
const callbacks = {
|
||||
onEvent: (e) => {
|
||||
if (e.kind === 'node-processed' && e.part === 'replies') {
|
||||
writeThreadcapChunk(e.nodeId, threadcap, sentCommenters, res)
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
await updateThreadcap(threadcap, {
|
||||
updateTime: new Date().toISOString(),
|
||||
userAgent,
|
||||
cache,
|
||||
fetcher,
|
||||
callbacks,
|
||||
})
|
||||
} catch (error) {
|
||||
console.error('Error fetching ActivityPub comments:', error)
|
||||
res.status(500).send('Error fetching ActivityPub comments')
|
||||
}
|
||||
|
||||
await updateThreadcap(threadcap, {
|
||||
updateTime: new Date().toISOString(),
|
||||
userAgent,
|
||||
cache,
|
||||
fetcher,
|
||||
callbacks,
|
||||
})
|
||||
|
||||
res.end()
|
||||
})
|
||||
|
||||
@@ -335,6 +384,17 @@ app.use('/api/apps', async (req, res) => {
|
||||
})
|
||||
})
|
||||
|
||||
app.use('/api/datasets', async (req, res) => {
|
||||
fs.readFile('./server/data/public_datasets.json', 'utf8', (err, data) => {
|
||||
if (err) {
|
||||
res.status(500).send({})
|
||||
return
|
||||
}
|
||||
res.set('Content-Type', 'application/json')
|
||||
res.send(data)
|
||||
})
|
||||
})
|
||||
|
||||
app.use('/api/images', express.static('./server/assets'))
|
||||
|
||||
// ------------------------------------------------
|
||||
@@ -417,6 +477,15 @@ app.use('/stats', (req, res) => {
|
||||
})
|
||||
})
|
||||
|
||||
app.use('/datasets', (req, res) => {
|
||||
res.render('index', {
|
||||
title: 'Datasets',
|
||||
description:
|
||||
'Open datasets generated and published by the Podcast Index project.',
|
||||
path: req.originalUrl,
|
||||
})
|
||||
})
|
||||
|
||||
// ------------------------------------------------
|
||||
// ---------- Static content for client -----------
|
||||
// ------------------------------------------------
|
||||
@@ -432,6 +501,11 @@ app.get('*', (req, res) => {
|
||||
const PORT = process.env.PORT || 333
|
||||
|
||||
// start express server on port 5001 (default)
|
||||
app.listen(PORT, () => {
|
||||
console.log(`server started on port ${PORT}`)
|
||||
})
|
||||
// Only listen when run directly, so the app can be imported by tests.
|
||||
if (require.main === module) {
|
||||
app.listen(PORT, () => {
|
||||
console.log(`server started on port ${PORT}`)
|
||||
})
|
||||
}
|
||||
|
||||
module.exports = app
|
||||
@@ -0,0 +1,94 @@
|
||||
/*
|
||||
* Self-contained test for the Web Bot Auth key directory endpoint:
|
||||
* GET /.well-known/http-message-signatures-directory
|
||||
*
|
||||
* There is no server-side test runner in this repo (`yarn test` is react-scripts
|
||||
* for the UI), so this file is a plain Node script: boot the Express app on an
|
||||
* ephemeral port, make real requests, assert, exit non-zero on failure.
|
||||
*
|
||||
* Run with: node server/test/well-known-signing-directory.test.js
|
||||
*/
|
||||
const assert = require('assert')
|
||||
const fetch = require('node-fetch')
|
||||
const app = require('../index')
|
||||
|
||||
const PATH = '/.well-known/http-message-signatures-directory'
|
||||
const EXPECTED_CONTENT_TYPE =
|
||||
'application/http-message-signatures-directory+json'
|
||||
|
||||
async function run() {
|
||||
const server = app.listen(0)
|
||||
await new Promise((resolve) => server.once('listening', resolve))
|
||||
const { port } = server.address()
|
||||
const url = `http://127.0.0.1:${port}${PATH}`
|
||||
|
||||
let failures = 0
|
||||
const check = (name, fn) => {
|
||||
try {
|
||||
fn()
|
||||
console.log(` ok - ${name}`)
|
||||
} catch (err) {
|
||||
failures++
|
||||
console.error(` FAIL - ${name}\n ${err.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// --- GET ---
|
||||
const getRes = await fetch(url)
|
||||
const body = await getRes.text()
|
||||
|
||||
check('GET returns 200', () => assert.strictEqual(getRes.status, 200))
|
||||
check('GET sets exact Content-Type', () =>
|
||||
assert.strictEqual(
|
||||
getRes.headers.get('content-type'),
|
||||
EXPECTED_CONTENT_TYPE
|
||||
)
|
||||
)
|
||||
check('GET sets a public Cache-Control', () => {
|
||||
const cc = getRes.headers.get('cache-control') || ''
|
||||
assert.ok(/public/.test(cc) && /max-age=\d+/.test(cc), `got "${cc}"`)
|
||||
})
|
||||
check('GET body is valid JSON with a keys array', () => {
|
||||
const json = JSON.parse(body)
|
||||
assert.ok(Array.isArray(json.keys), 'keys is not an array')
|
||||
assert.ok(json.keys.length >= 1, 'keys is empty')
|
||||
const k = json.keys[0]
|
||||
assert.strictEqual(k.kty, 'OKP')
|
||||
assert.strictEqual(k.crv, 'Ed25519')
|
||||
assert.ok('x' in k, 'missing x')
|
||||
assert.ok('kid' in k, 'missing kid')
|
||||
})
|
||||
check('GET body contains no private key material', () => {
|
||||
assert.ok(!/"d"\s*:/.test(body), 'response contains a private key "d"')
|
||||
})
|
||||
|
||||
// --- HEAD ---
|
||||
const headRes = await fetch(url, { method: 'HEAD' })
|
||||
const headBody = await headRes.text()
|
||||
check('HEAD returns 200', () => assert.strictEqual(headRes.status, 200))
|
||||
check('HEAD sets exact Content-Type', () =>
|
||||
assert.strictEqual(
|
||||
headRes.headers.get('content-type'),
|
||||
EXPECTED_CONTENT_TYPE
|
||||
)
|
||||
)
|
||||
check('HEAD has an empty body', () =>
|
||||
assert.strictEqual(headBody, '')
|
||||
)
|
||||
} finally {
|
||||
server.close()
|
||||
}
|
||||
|
||||
if (failures > 0) {
|
||||
console.error(`\n${failures} check(s) failed`)
|
||||
process.exit(1)
|
||||
}
|
||||
console.log('\nAll checks passed')
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
run().catch((err) => {
|
||||
console.error(err)
|
||||
process.exit(1)
|
||||
})
|
||||
@@ -148,9 +148,7 @@ export default class TopBar extends React.PureComponent<IProps, IState> {
|
||||
>
|
||||
<Button link href="/apps">Apps</Button>
|
||||
<Button link href="/podcast/value4value">Value 4 Value</Button>
|
||||
<Button link href="/stats">
|
||||
Stats
|
||||
</Button>
|
||||
<Button link href="/datasets">Datasets</Button>
|
||||
<Button link href="/add">Add</Button>
|
||||
<Button href="https://podcastindex-org.github.io/docs-api/">
|
||||
Docs
|
||||
|
||||
@@ -9,6 +9,8 @@ function SingleApp(app) {
|
||||
<div className="podcastIndexAppIcon">
|
||||
<img
|
||||
src={`${document.location.origin}/api/images/${app.appIconUrl}`}
|
||||
alt=''
|
||||
{...(app.tagLine ? { title: `${app.tagLine}` } : {})}
|
||||
></img>
|
||||
</div>
|
||||
<div className="podcastIndexAppTitleAndType">
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import * as React from 'react'
|
||||
|
||||
export interface Dataset {
|
||||
name: string
|
||||
url: string
|
||||
format: string
|
||||
cadence: string
|
||||
description: string
|
||||
}
|
||||
|
||||
interface IProps {
|
||||
dataset: Dataset
|
||||
}
|
||||
|
||||
const DatasetCard: React.FunctionComponent<IProps> = ({ dataset }) => {
|
||||
return (
|
||||
<div className="dataset-card">
|
||||
<h3 className="dataset-card-title">
|
||||
<a
|
||||
href={dataset.url}
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
{dataset.name}
|
||||
</a>
|
||||
</h3>
|
||||
<div className="dataset-card-badges">
|
||||
<span className="dataset-card-badge">{dataset.format}</span>
|
||||
<span className="dataset-card-badge">{dataset.cadence}</span>
|
||||
</div>
|
||||
<p className="dataset-card-description">{dataset.description}</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default DatasetCard
|
||||
@@ -0,0 +1,30 @@
|
||||
import * as React from 'react'
|
||||
|
||||
import DatasetCard, { Dataset } from './DatasetCard'
|
||||
|
||||
export interface Section {
|
||||
id: string
|
||||
name: string
|
||||
description: string
|
||||
datasets: Dataset[]
|
||||
}
|
||||
|
||||
interface IProps {
|
||||
section: Section
|
||||
}
|
||||
|
||||
const SectionBlock: React.FunctionComponent<IProps> = ({ section }) => {
|
||||
return (
|
||||
<section id={section.id} className="datasets-section">
|
||||
<h2 className="datasets-section-title">{section.name}</h2>
|
||||
<p className="datasets-section-description">{section.description}</p>
|
||||
<div className="datasets-section-list">
|
||||
{section.datasets.map((dataset) => (
|
||||
<DatasetCard key={dataset.url} dataset={dataset} />
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
export default SectionBlock
|
||||
@@ -0,0 +1,28 @@
|
||||
import * as React from 'react'
|
||||
|
||||
import { Section } from './Section'
|
||||
|
||||
interface IProps {
|
||||
sections: Section[]
|
||||
}
|
||||
|
||||
const TableOfContents: React.FunctionComponent<IProps> = ({ sections }) => {
|
||||
return (
|
||||
<nav className="datasets-toc" aria-label="Datasets sections">
|
||||
<ul className="datasets-toc-list">
|
||||
{sections.map((section) => (
|
||||
<li key={section.id} className="datasets-toc-item">
|
||||
<a
|
||||
className="datasets-toc-link"
|
||||
href={`#${section.id}`}
|
||||
>
|
||||
{section.name}
|
||||
</a>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</nav>
|
||||
)
|
||||
}
|
||||
|
||||
export default TableOfContents
|
||||
@@ -0,0 +1,90 @@
|
||||
import * as React from 'react'
|
||||
import { useEffect, useState } from 'react'
|
||||
|
||||
import { updateTitle } from '../../utils'
|
||||
import SectionBlock, { Section } from './Section'
|
||||
import TableOfContents from './TableOfContents'
|
||||
|
||||
import './styles.scss'
|
||||
|
||||
interface DatasetsDoc {
|
||||
title: string
|
||||
tagline: string
|
||||
intro: string
|
||||
sections: Section[]
|
||||
}
|
||||
|
||||
const Datasets: React.FunctionComponent = () => {
|
||||
const [data, setData] = useState<DatasetsDoc | null>(null)
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [error, setError] = useState(false)
|
||||
|
||||
useEffect(() => {
|
||||
updateTitle('Datasets')
|
||||
let cancelled = false
|
||||
|
||||
const load = async () => {
|
||||
try {
|
||||
const response = await fetch('/api/datasets', {
|
||||
credentials: 'same-origin',
|
||||
method: 'GET',
|
||||
})
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}`)
|
||||
}
|
||||
const json: DatasetsDoc = await response.json()
|
||||
if (!cancelled) {
|
||||
setData(json)
|
||||
setLoading(false)
|
||||
}
|
||||
} catch (e) {
|
||||
if (!cancelled) {
|
||||
setError(true)
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
load()
|
||||
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}, [])
|
||||
|
||||
if (loading) {
|
||||
return (
|
||||
<div className="datasets-page">
|
||||
<p className="datasets-loading">Loading datasets…</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
if (error || !data) {
|
||||
return (
|
||||
<div className="datasets-page">
|
||||
<p className="datasets-error">
|
||||
Couldn't load datasets. Try refreshing.
|
||||
</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="datasets-page">
|
||||
<header className="datasets-header">
|
||||
<h1 className="datasets-title">{data.title}</h1>
|
||||
<p className="datasets-tagline">{data.tagline}</p>
|
||||
<p className="datasets-intro">{data.intro}</p>
|
||||
</header>
|
||||
<TableOfContents sections={data.sections} />
|
||||
<div className="datasets-sections">
|
||||
{data.sections.map((section) => (
|
||||
<SectionBlock key={section.id} section={section} />
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
export default Datasets
|
||||
@@ -0,0 +1,158 @@
|
||||
.datasets-page {
|
||||
max-width: 900px;
|
||||
margin: 0 auto;
|
||||
padding: 28px 20px 80px;
|
||||
color: var(--fg-main);
|
||||
}
|
||||
|
||||
.datasets-header {
|
||||
margin-bottom: 28px;
|
||||
}
|
||||
|
||||
.datasets-title {
|
||||
font-family: var(--font-family-exp-bold);
|
||||
font-size: 2.6rem;
|
||||
line-height: 1.1;
|
||||
margin: 0 0 8px;
|
||||
}
|
||||
|
||||
.datasets-tagline {
|
||||
color: var(--text-color);
|
||||
font-size: 1.05rem;
|
||||
margin: 0 0 16px;
|
||||
}
|
||||
|
||||
.datasets-intro {
|
||||
line-height: 1.6;
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.datasets-toc {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 5;
|
||||
background: var(--bg-main);
|
||||
padding: 12px 0;
|
||||
margin-bottom: 40px;
|
||||
border-top: 1px solid rgba(127, 127, 127, 0.18);
|
||||
border-bottom: 1px solid rgba(127, 127, 127, 0.18);
|
||||
}
|
||||
|
||||
.datasets-toc-list {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 8px 24px;
|
||||
list-style: none;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.datasets-toc-item {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
.datasets-toc-link {
|
||||
color: var(--text-color);
|
||||
font-family: var(--font-family-bold);
|
||||
font-size: 0.85rem;
|
||||
letter-spacing: 0.04em;
|
||||
text-decoration: none;
|
||||
text-transform: uppercase;
|
||||
|
||||
&:hover {
|
||||
color: var(--link-hover-color);
|
||||
}
|
||||
}
|
||||
|
||||
.datasets-section {
|
||||
margin-bottom: 56px;
|
||||
scroll-margin-top: 80px;
|
||||
}
|
||||
|
||||
.datasets-section-title {
|
||||
font-family: var(--font-family-exp-bold);
|
||||
font-size: 2.25rem;
|
||||
line-height: 1.15;
|
||||
margin: 0;
|
||||
padding-bottom: 8px;
|
||||
border-bottom: 2px solid var(--color-primary);
|
||||
}
|
||||
|
||||
.datasets-section-description {
|
||||
color: var(--text-color);
|
||||
line-height: 1.5;
|
||||
margin: 14px 0 24px;
|
||||
max-width: 70ch;
|
||||
}
|
||||
|
||||
.datasets-section-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.dataset-card {
|
||||
padding: 16px 0;
|
||||
border-bottom: 1px solid rgba(127, 127, 127, 0.18);
|
||||
|
||||
&:first-child {
|
||||
padding-top: 4px;
|
||||
}
|
||||
|
||||
&:last-child {
|
||||
border-bottom: none;
|
||||
}
|
||||
}
|
||||
|
||||
.dataset-card-title {
|
||||
font-family: var(--font-family-bold);
|
||||
font-size: 1.15rem;
|
||||
margin: 0 0 4px;
|
||||
|
||||
a {
|
||||
color: var(--link-color);
|
||||
text-decoration: none;
|
||||
|
||||
&:hover {
|
||||
color: var(--link-hover-color);
|
||||
text-decoration: underline;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
.dataset-card-badges {
|
||||
color: var(--text-color);
|
||||
font-size: 0.85rem;
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.dataset-card-badge {
|
||||
text-transform: lowercase;
|
||||
|
||||
& + &::before {
|
||||
content: ' · ';
|
||||
margin: 0 4px;
|
||||
color: var(--text-color);
|
||||
}
|
||||
}
|
||||
|
||||
.dataset-card-description {
|
||||
line-height: 1.55;
|
||||
margin: 0;
|
||||
max-width: 70ch;
|
||||
}
|
||||
|
||||
.datasets-loading,
|
||||
.datasets-error {
|
||||
color: var(--text-color);
|
||||
padding: 24px 0;
|
||||
}
|
||||
|
||||
@media only screen and (max-width: 600px) {
|
||||
.datasets-title {
|
||||
font-size: 2rem;
|
||||
}
|
||||
|
||||
.datasets-section-title {
|
||||
font-size: 1.7rem;
|
||||
}
|
||||
}
|
||||
@@ -86,6 +86,9 @@ export default class Card extends React.Component<IProps> {
|
||||
last60={overallStats.feedCount60days}
|
||||
last90={overallStats.feedCount90days}
|
||||
/>
|
||||
<div style={{ textAlign: 'center' }}>
|
||||
Get an <a href="https://public.podcastindex.org/24hourFeedReport.html">overview of all new feeds</a> over the last 24 hours.
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -4,6 +4,7 @@ import { Route, Switch } from 'react-router-dom'
|
||||
import ScrollToTop from "./components/ScrollToTop";
|
||||
import AddFeed from "./pages/AddFeed";
|
||||
import Apps from './pages/Apps'
|
||||
import Datasets from './pages/Datasets'
|
||||
import DonationThankYou from './pages/Donations'
|
||||
|
||||
import Landing from './pages/landing'
|
||||
@@ -23,6 +24,7 @@ const Routes: React.FunctionComponent = () => (
|
||||
/>
|
||||
<Route path="/thankyou" component={DonationThankYou}/>
|
||||
<Route exact path="/stats" render={() => <Stats/>}/>
|
||||
<Route exact path="/datasets" component={Datasets}/>
|
||||
|
||||
<Route path="/podcast" component={Podcast}/>
|
||||
|
||||
|
||||
@@ -3,6 +3,8 @@ const HtmlWebpackPlugin = require('html-webpack-plugin')
|
||||
const FaviconsWebpackPlugin = require('favicons-webpack-plugin')
|
||||
const Dotenv = require('dotenv-webpack');
|
||||
|
||||
const proxyTarget = process.env.PROXY_TARGET || 'http://localhost:5001';
|
||||
|
||||
module.exports = env => ({
|
||||
watch: false,
|
||||
mode: 'production',
|
||||
@@ -15,22 +17,22 @@ module.exports = env => ({
|
||||
proxy: {
|
||||
'/': {
|
||||
changeOrigin: true,
|
||||
target: 'http://localhost:5001',
|
||||
target: proxyTarget,
|
||||
pathRewrite: { '^/': '/' },
|
||||
},
|
||||
'/api': {
|
||||
changeOrigin: true,
|
||||
target: 'http://localhost:5001',
|
||||
target: proxyTarget,
|
||||
pathRewrite: { '^/api': '/api' },
|
||||
},
|
||||
'/namespace': {
|
||||
changeOrigin: true,
|
||||
target: 'http://localhost:5001',
|
||||
target: proxyTarget,
|
||||
pathRewrite: { '^/namespace': '/namespace' },
|
||||
},
|
||||
'/podcast': {
|
||||
changeOrigin: true,
|
||||
target: 'http://localhost:5001',
|
||||
target: proxyTarget,
|
||||
pathRewrite: { '^/podcast': '/podcast' },
|
||||
}
|
||||
},
|
||||
|
||||