`
await writeFile(join(OUT,'index.html'),html)
-const sitemapUrls=[rootUrl,...hasSite.map(p=>p.site),rootUrl+'colophon.html']
+const sitemapUrls=[rootUrl,...hasSite.map(p=>p.site).filter(url=>servedFrom(ORIGIN,url)),rootUrl+'colophon.html']
await writeFile(join(OUT,'sitemap.xml'),`${sitemapUrls.map(url=>{const p=projects.find(p=>p.site===url);return `${e(url)}${p?.lastmod?`${p.lastmod}`:''}`}).join('')}\n`)
const sitemaps=[...new Set([rootUrl+'sitemap.xml',...projects.map(p=>p.sitemap).filter(Boolean)])].sort()
await writeFile(join(OUT,'robots.txt'),`User-agent: *\nAllow: /\n\n${sitemaps.map(url=>'Sitemap: '+url).join('\n')}\n`)
diff --git a/sites/home/scripts/catalog-policy.mjs b/sites/home/scripts/catalog-policy.mjs
index a0a8364..de0a0d1 100644
--- a/sites/home/scripts/catalog-policy.mjs
+++ b/sites/home/scripts/catalog-policy.mjs
@@ -2,12 +2,18 @@
// fallback without requiring a source edit for every newly published website.
export const projectAddress=(origin,name)=>`${origin}/${encodeURIComponent(name)}/`
-export function homepageProblem(repo,origin){
+// A project lives wherever its About URL says. The catalog refuses only an
+// address it cannot safely link to.
+export function homepageProblem(repo){
if(!repo.homepage)return null
- const expected=projectAddress(origin,repo.name)
- return repo.homepage===expected?null:`${repo.name} About URL is ${JSON.stringify(repo.homepage)}, expected ${expected}`
+ let url
+ try{url=new URL(repo.homepage)}catch{return `${repo.name} About URL ${JSON.stringify(repo.homepage)} is not a URL`}
+ return url.protocol==='https:'?null:`${repo.name} About URL ${JSON.stringify(repo.homepage)} is not an https address`
}
+// The central sitemap and robots.txt may list only addresses on their own host.
+export const servedFrom=(origin,url)=>typeof url==='string'&&url.startsWith(`${origin}/`)
+
export function automaticGroup(repo){
const name=repo.name.toLowerCase()
const topics=repo.topics.join(' ').toLowerCase()
diff --git a/sites/home/scripts/check-publication.mjs b/sites/home/scripts/check-publication.mjs
index a73c351..8bdd8b2 100644
--- a/sites/home/scripts/check-publication.mjs
+++ b/sites/home/scripts/check-publication.mjs
@@ -2,6 +2,7 @@ import assert from 'node:assert/strict'
import {readFile,access} from 'node:fs/promises'
import {gzipSync} from 'node:zlib'
import {catalog,ORIGIN,unitCount} from '../src/data/catalog.ts'
+import {servedFrom} from './catalog-policy.mjs'
import {escapeHTML as e} from './interface.mjs'
import {assignArtwork} from './artwork.mjs'
import {validateState} from './catalog-state.mjs'
@@ -38,7 +39,7 @@ for(const match of html.matchAll(/srcset="([^"]+)"/g))for(const candidate of mat
for(const name of ['UncutSans-Regular.woff2','UncutSans-Medium.woff2','UncutSans-LICENSE.txt'])await access('dist/assets/fonts/'+name)
const sitemap=await readFile('dist/sitemap.xml','utf8')
const robots=await readFile('dist/robots.txt','utf8')
-for(const url of [ORIGIN+'/',...projects.map(p=>p.site).filter(Boolean),ORIGIN+'/colophon.html'])assert(sitemap.includes(`${e(url)}`),`Sitemap omitted ${url}`)
+for(const url of [ORIGIN+'/',...projects.map(p=>p.site).filter(url=>servedFrom(ORIGIN,url)),ORIGIN+'/colophon.html'])assert(sitemap.includes(`${e(url)}`),`Sitemap omitted ${url}`)
for(const url of [ORIGIN+'/sitemap.xml',...projects.map(p=>p.sitemap).filter(Boolean)])assert(robots.includes(`Sitemap: ${url}`),`robots.txt omitted ${url}`)
assert(!/^Disallow:\s*\/$/m.test(robots),'Production blocks crawling')
assert(html.includes('href="https://appautomaton.renocrypt.com/"'),'Missing the related field guide')
diff --git a/sites/home/scripts/discovery.test.mjs b/sites/home/scripts/discovery.test.mjs
index 6de70fe..4da02f4 100644
--- a/sites/home/scripts/discovery.test.mjs
+++ b/sites/home/scripts/discovery.test.mjs
@@ -70,8 +70,8 @@ test('new public project websites enter the catalog automatically while editoria
assert.equal(selected.length,5)
const result=resolveShelves(shelves,selected)
assert.deepEqual(result.map(s=>s.items.map(p=>p.repo)),[['new-skill'],['source-only','new-system'],['mlx-new'],['studio']])
- assert.equal(homepageProblem(repo('mlx-new'),'https://appautomaton.com'),null)
- for(const homepage of ['https://appautomaton.renocrypt.com/mlx-new/','https://appautomaton.com/another/','http://appautomaton.com/mlx-new/','javascript:alert(1)'])assert(homepageProblem(repo('mlx-new',{homepage}),'https://appautomaton.com'))
+ for(const homepage of ['https://appautomaton.com/mlx-new/','https://mlx-new.appautomaton.com','https://appautomaton.renocrypt.com/mlx-new/'])assert.equal(homepageProblem(repo('mlx-new',{homepage})),null)
+ for(const homepage of ['http://appautomaton.com/mlx-new/','javascript:alert(1)','not a url'])assert(homepageProblem(repo('mlx-new',{homepage})))
assert.throws(()=>selectExhibits([],shelves,{}),/unavailable repositories/)
})
@@ -83,6 +83,7 @@ test('project indexing guards inspect canonical metadata and both HTML and HTTP
assert.deepEqual(audit(),[])
assert.match(audit(page.replace(url,'https://appautomaton.renocrypt.com/example/')).join(' '),/canonical/)
assert.match(audit('Missing canonical').join(' '),/missing canonical/)
+ assert.doesNotMatch(auditProjectPage('https://example.appautomaton.com',page.replace(url,'https://example.appautomaton.com/'),new Headers()).problems.join(' '),/canonical/)
for(const meta of ['','',''])assert.match(audit(page+meta).join(' '),/forbids indexing/)
assert.match(audit(page,new Headers({'X-Robots-Tag':'googlebot: noindex'})).join(' '),/forbids indexing/)
assert.deepEqual(audit(page+''),[])
diff --git a/sites/home/scripts/page-audit.mjs b/sites/home/scripts/page-audit.mjs
index eb59cd5..f7dcf95 100644
--- a/sites/home/scripts/page-audit.mjs
+++ b/sites/home/scripts/page-audit.mjs
@@ -1,4 +1,11 @@
// Read static project metadata without executing page scripts.
+
+// An address and its normalized form name the same page, e.g. a bare origin
+// and the origin with its root path.
+const sameAddress = (href, url) => {
+ try { return new URL(href, url).href === new URL(url).href } catch { return false }
+}
+
export function auditProjectPage(url, html, headers) {
const problems = []
const warnings = []
@@ -6,7 +13,7 @@ export function auditProjectPage(url, html, headers) {
const canonical = html.match(/]+rel=["']canonical["'][^>]*>/i)?.[0]
const href = canonical?.match(/href=["']([^"']+)["']/i)?.[1]
if (!href) problems.push('missing canonical')
- else if (href !== url) problems.push(`canonical points at ${href}, not ${url}`)
+ else if (!sameAddress(href, url)) problems.push(`canonical points at ${href}, not ${url}`)
const directives=[headers.get('x-robots-tag')||'']
for(const tag of html.match(/]*>/gi)||[]) {
diff --git a/sites/home/scripts/sync-catalog.mjs b/sites/home/scripts/sync-catalog.mjs
index e5a5fc6..131f6cd 100644
--- a/sites/home/scripts/sync-catalog.mjs
+++ b/sites/home/scripts/sync-catalog.mjs
@@ -17,7 +17,7 @@ import { join, dirname } from 'node:path'
import { fileURLToPath, pathToFileURL } from 'node:url'
import { serves } from './probe-response.mjs'
import { readRepositories } from './github-repositories.mjs'
-import { homepageProblem, projectAddress, selectExhibits } from './catalog-policy.mjs'
+import { homepageProblem, projectAddress, selectExhibits, servedFrom } from './catalog-policy.mjs'
import { auditProjectPage } from './page-audit.mjs'
import {emptyState,loadPublishedState,reconcileState} from './catalog-state.mjs'
const { shelves, notShown } = await import('../src/data/shelves.ts')
@@ -60,7 +60,8 @@ async function probe(repo) {
two conventional names answers is how that list stops being a thing
somebody has to remember to update. */
let sitemap = null
- for (const name_ of ['sitemap-index.xml', 'sitemap.xml']) {
+ // A project on its own host keeps its sitemap there, outside this catalog.
+ if (url === projectAddress(ORIGIN, name)) for (const name_ of ['sitemap-index.xml', 'sitemap.xml']) {
const url = `${ORIGIN}/${name}/${name_}`
try {
const response=await serves(url,'GET')
@@ -118,7 +119,7 @@ try {
process.exit(0)
}
-const invalidHomepages=repos.filter(r=>!Object.hasOwn(notShown,r.name)).map(r=>homepageProblem(r,ORIGIN)).filter(Boolean)
+const invalidHomepages=repos.filter(r=>!Object.hasOwn(notShown,r.name)).map(r=>homepageProblem(r)).filter(Boolean)
if(invalidHomepages.length)throw new Error(invalidHomepages.join('\n'))
const probes = await Promise.all(repos.map(r => Object.hasOwn(notShown,r.name)||(!r.homepage&&!placed.has(r.name))?{hasSite:false}:probe(r)))
diff --git a/tooling/sync-registry.mjs b/tooling/sync-registry.mjs
index 28615e3..62c12d9 100644
--- a/tooling/sync-registry.mjs
+++ b/tooling/sync-registry.mjs
@@ -10,7 +10,8 @@ const registry = await readJSON(path)
const byID = new Map(registry.sites.map(site => [site.id, site]))
for (const project of catalog.flatMap(group => group.items).filter(project => project.site)) {
const url = new URL(project.site)
- assert.equal(url.origin, registry.origin, `Project uses a different origin: ${project.repo}`)
+ // The registry records mounts on this origin; a project on its own host has none.
+ if (url.origin !== registry.origin) continue
const existing = byID.get(project.repo)
if (existing) {
assert.equal(existing.publicPath, url.pathname, `A public mount changed and needs review: ${project.repo}`)