diff --git a/.github/workflows/social-bot.yml b/.github/workflows/social-bot.yml new file mode 100644 index 0000000..9e43b7a --- /dev/null +++ b/.github/workflows/social-bot.yml @@ -0,0 +1,86 @@ +name: Morrow Brief bot + +# The Telegram bot's fallback path: answer commands by polling every five minutes. +# The faster path is the Cloudflare Worker in bot/worker.js; this workflow also sets its webhook. +# The scheduled run does nothing until the repo variable MORROW_BOT is set to "on". See docs/SOCIAL.md. + +on: + schedule: + - cron: "*/5 * * * *" + workflow_dispatch: + inputs: + action: + description: What to do + type: choice + default: poll + options: + - poll + - set-webhook + - delete-webhook + url: + description: Worker URL, for set-webhook (https://...workers.dev) + type: string + required: false + +permissions: + contents: read + +concurrency: + group: morrow-bot + cancel-in-progress: false + +jobs: + poll: + if: >- + (github.event_name == 'schedule' && vars.MORROW_BOT == 'on') || + (github.event_name == 'workflow_dispatch' && inputs.action == 'poll') + runs-on: ubuntu-latest + timeout-minutes: 8 + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: scripts + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: scripts/social_requirements.txt + - name: Install + run: pip install -r scripts/social_requirements.txt + - name: Fetch today's feed + run: | + mkdir -p data + curl -fsSL "https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/social-data/feed.json" -o data/feed.json \ + || echo "no feed published yet, the bot will say the edition is not ready" + - name: Answer messages + env: + TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} + run: python scripts/social_bot.py poll --feed data/feed.json --listen 240 + + webhook: + if: github.event_name == 'workflow_dispatch' && inputs.action != 'poll' + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + sparse-checkout: scripts + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: scripts/social_requirements.txt + - name: Install + run: pip install -r scripts/social_requirements.txt + - name: Point Telegram at the Worker + if: inputs.action == 'set-webhook' + env: + TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} + TELEGRAM_WEBHOOK_SECRET: ${{ secrets.TELEGRAM_WEBHOOK_SECRET }} + WORKER_URL: ${{ inputs.url }} + run: python scripts/social_bot.py set-webhook --url "$WORKER_URL" + - name: Remove the webhook + if: inputs.action == 'delete-webhook' + env: + TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} + run: python scripts/social_bot.py delete-webhook diff --git a/.github/workflows/social.yml b/.github/workflows/social.yml new file mode 100644 index 0000000..de18415 --- /dev/null +++ b/.github/workflows/social.yml @@ -0,0 +1,122 @@ +name: Morrow Brief + +# Posts the day's digest and a few alerts to the Morrow Brief pages when the edition changes. +# The schedule is a fallback in case the generator's pushes do not start workflows; a run with +# nothing new to post changes nothing. +# With no secrets set it sends nothing and records every target as skipped. See docs/SOCIAL.md. + +on: + push: + branches: [main] + paths: + - index.html + pull_request: + paths: + - "scripts/social_*" + - "tests/social/**" + - "bot/**" + - "assets/fonts/**" + - ".github/workflows/social*.yml" + schedule: + - cron: "17 */3 * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: scripts/social_requirements.txt + - uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Install + run: pip install -r scripts/social_requirements.txt + - name: Python tests + run: python -m unittest discover -s tests/social -v + - name: Worker tests + run: node --test bot/worker.test.mjs + - name: Render what today's page would post + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + run: python scripts/social_post.py preview --page index.html --out preview + - name: Keep the preview + if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + uses: actions/upload-artifact@v4 + with: + name: morrow-brief-preview + path: preview + retention-days: 14 + + publish: + needs: test + if: github.event_name != 'pull_request' && github.ref == 'refs/heads/main' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: write + concurrency: + group: morrow-social + cancel-in-progress: false + env: + RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: scripts/social_requirements.txt + - name: Install + run: pip install -r scripts/social_requirements.txt + + - name: Open the data branch + run: | + git config user.name "morrow-brief[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + set +e + git ls-remote --exit-code --heads origin social-data > /dev/null + found=$? + set -e + if [ "$found" -eq 0 ]; then + git fetch --depth 1 origin social-data + git worktree add --detach data FETCH_HEAD + elif [ "$found" -eq 2 ]; then + echo "social-data does not exist yet, starting empty" + git worktree add --detach data HEAD + git -C data rm -rf --quiet . + else + # Starting from an empty state here could post today's digest a second time. + echo "::error::could not read the social-data branch (git exit $found); nothing posted" + exit 1 + fi + + - name: Pick and freeze the next post + run: python scripts/social_post.py build --page index.html --out data --run "$RUN_ID" + + - name: Save the claim before anything is sent + run: | + sha="$(bash scripts/social_save.sh "Morrow Brief data, run $RUN_ID (before delivery)")" + echo "DATA_SHA=$sha" >> "$GITHUB_ENV" + + - name: Deliver + env: + BUFFER_API_KEY: ${{ secrets.BUFFER_API_KEY }} + TELEGRAM_BOT_TOKEN: ${{ secrets.TELEGRAM_BOT_TOKEN }} + TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }} + MORROW_BUFFER_CHANNELS: ${{ vars.MORROW_BUFFER_CHANNELS }} + run: | + python scripts/social_post.py deliver --out data --run "$RUN_ID" \ + --image-base "https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/${DATA_SHA}/" \ + --image-base "https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/social-data/" + + - name: Save the results + if: always() && env.DATA_SHA != '' + run: bash scripts/social_save.sh "Morrow Brief data, run $RUN_ID" "$DATA_SHA" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..01560e9 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.pyc +node_modules/ +/preview/ +/data/ diff --git a/README.md b/README.md index c2691a9..53d7cf7 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,16 @@ A planned add-on: last night's US move plus the rupiah, coal, oil and nickel, ma - `.github/workflows/waitlist.yml` thanks each sign-up and keeps a live tally issue up to date (`scripts/waitlist_tally.py`, tests in `tests/`). - Success line: 50 sign-ups between 2026-11-08 and 2026-11-22. 15-49 means rework the pitch; under 15 means stop. +## Morrow Brief (social page) + +The same edition as a page channel: one morning digest of five stories, plus a few alerts for stories that three or more outlets report. Every post carries its outlet count, a link and a typographic card. Telegram is posted to directly; Instagram, Threads and X go through Buffer. A Telegram bot answers section commands and `/find`. + +- `.github/workflows/social.yml` ("Morrow Brief") runs the tests on every pull request. On `main` it builds the posts from the published `index.html` and delivers them. +- `.github/workflows/social-bot.yml` ("Morrow Brief bot") sets the bot's webhook, or polls as a slower fallback. +- Setup, post formats, failure handling and what is not verified yet: [docs/SOCIAL.md](docs/SOCIAL.md). + +Nothing is posted until the secrets are added, and no post has been sent to a real platform yet. + ## Notes - Built by [Juwen Lie](https://github.com/juw12) as an independent personal project. Still being built, so expect changes. diff --git a/assets/fonts/OFL-SourceSans3.md b/assets/fonts/OFL-SourceSans3.md new file mode 100644 index 0000000..22c601b --- /dev/null +++ b/assets/fonts/OFL-SourceSans3.md @@ -0,0 +1,93 @@ +Copyright 2010-2024 Adobe (http://www.adobe.com/), with Reserved Font Name 'Source'. All Rights Reserved. Source is a trademark of Adobe in the United States and/or other countries. + +This Font Software is licensed under the SIL Open Font License, Version 1.1. + +This license is copied below, and is also available with a FAQ at: http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/assets/fonts/OFL-SourceSerif4.md b/assets/fonts/OFL-SourceSerif4.md new file mode 100644 index 0000000..5871e1f --- /dev/null +++ b/assets/fonts/OFL-SourceSerif4.md @@ -0,0 +1,93 @@ +Copyright 2014 - 2023 Adobe (http://www.adobe.com/), with Reserved Font Name ‘Source’. All Rights Reserved. Source is a trademark of Adobe in the United States and/or other countries. + +This Font Software is licensed under the SIL Open Font License, Version 1.1. + +This license is copied below, and is also available with a FAQ at: http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/assets/fonts/SourceSans3-Regular.ttf b/assets/fonts/SourceSans3-Regular.ttf new file mode 100644 index 0000000..c5dd0b8 Binary files /dev/null and b/assets/fonts/SourceSans3-Regular.ttf differ diff --git a/assets/fonts/SourceSans3-Semibold.ttf b/assets/fonts/SourceSans3-Semibold.ttf new file mode 100644 index 0000000..4aff9a3 Binary files /dev/null and b/assets/fonts/SourceSans3-Semibold.ttf differ diff --git a/assets/fonts/SourceSerif4-It.ttf b/assets/fonts/SourceSerif4-It.ttf new file mode 100644 index 0000000..e818087 Binary files /dev/null and b/assets/fonts/SourceSerif4-It.ttf differ diff --git a/assets/fonts/SourceSerif4-Regular.ttf b/assets/fonts/SourceSerif4-Regular.ttf new file mode 100644 index 0000000..964a2bf Binary files /dev/null and b/assets/fonts/SourceSerif4-Regular.ttf differ diff --git a/assets/fonts/SourceSerif4-Semibold.ttf b/assets/fonts/SourceSerif4-Semibold.ttf new file mode 100644 index 0000000..151e9a9 Binary files /dev/null and b/assets/fonts/SourceSerif4-Semibold.ttf differ diff --git a/bot/package.json b/bot/package.json new file mode 100644 index 0000000..2f0bda7 --- /dev/null +++ b/bot/package.json @@ -0,0 +1,8 @@ +{ + "name": "morrow-brief-bot", + "private": true, + "type": "module", + "scripts": { + "test": "node --test worker.test.mjs" + } +} diff --git a/bot/worker.js b/bot/worker.js new file mode 100644 index 0000000..47f5e6a --- /dev/null +++ b/bot/worker.js @@ -0,0 +1,135 @@ +// Morrow Brief: the Telegram bot as a Cloudflare Worker (free plan). +// +// Telegram calls this Worker for every message (a webhook). The Worker reads the day's pre-rendered +// replies from feed.json on the repo's data branch and answers in the HTTP response itself, so it +// holds no bot token, no database and no list of users. Its only secret is WEBHOOK_SECRET, which +// Telegram sends back in a header so nobody else can call it. +// +// scripts/social_bot.py gives the same answers; tests/social/fixtures/bot_cases.json keeps the two in step. + +export const FEED_URL = "https://raw.githubusercontent.com/juw12/morrow/social-data/feed.json"; +const SITE_URL = "https://juw12.github.io/morrow/"; +const NOT_READY = `Today's edition is not ready yet. Read the latest at juw12.github.io/morrow.`; +const SECTION_COMMANDS = ["/ai", "/bigtech", "/robotics", "/space", "/tech", "/markets", "/tape"]; +const MESSAGE_LIMIT = 4096; + +const escapeText = (text) => text.replace(/&/g, "&").replace(//g, ">"); +const escapeAttribute = (text) => escapeText(text).replace(/"/g, """).replace(/'/g, "'"); +const link = (url, label) => `${escapeText(label)}`; +const codePoints = (text) => Array.from(text).length; + +const NAMED_ENTITIES = { "&": "&", "<": "<", ">": ">", """: '"', "'": "'" }; + +function visibleLength(html) { + const text = html.replace(/<[^>]+>/g, "").replace(/&(amp|lt|gt|quot|#x27);/g, (entity) => NAMED_ENTITIES[entity]); + return codePoints(text); +} + +// Cut at a word boundary and end with an ellipsis; never longer than the limit. +export function truncate(text, limit) { + if (codePoints(text) <= limit) return text; + const words = text.split(" "); + while (words.length > 1) { + words.pop(); + const candidate = words.join(" ").replace(/[ ,;:\-—–]+$/, "") + "…"; + if (codePoints(candidate) <= limit) return candidate; + } + // A single word that does not fit: cut inside it. + const letters = Array.from(words[0]).slice(0, Math.max(limit - 1, 0)); + return letters.length ? letters.join("") + "…" : ""; +} + +function ranked(stories) { + return [...stories].sort( + (a, b) => Number(!a.is_lead) - Number(!b.is_lead) || b.coverage - a.coverage || a.order - b.order, + ); +} + +function storyLine(story) { + const extra = story.coverage > 1 ? ` +${story.coverage - 1}` : ""; + return `• ${link(story.url, truncate(story.title, 120))} (${escapeText(story.source)}${extra})`; +} + +function listReply(heading, stories) { + const build = (shown) => [`${escapeText(heading)}`, "", ...shown.map(storyLine)].join("\n"); + let shown = stories; + let text = build(shown); + while (visibleLength(text) > MESSAGE_LIMIT && shown.length) { + shown = shown.slice(0, -1); + text = build(shown); + } + return text; +} + +export function searchReply(stories, query) { + const words = (query.toLowerCase().match(/[a-z0-9]+/g) || []).filter((word) => word.length > 1); + if (!words.length) return "Send /find and a word, for example: /find nvidia"; + const hits = ranked(stories).filter((story) => { + const haystack = [story.title, story.dek, story.tag, story.source].join(" ").toLowerCase(); + return words.every((word) => haystack.includes(word)); + }); + if (!hits.length) return `Nothing on “${escapeText(query.trim())}” in today's edition. Try /today.`; + const noun = hits.length === 1 ? "story" : "stories"; + return listReply(`${hits.length} ${noun} on “${query.trim()}”`, hits.slice(0, 6)); +} + +// The bot's whole behaviour: message text in, HTML reply out. null means stay silent. +export function replyFor(feed, text, isPrivate = true) { + const message = (text || "").trim(); + const replies = (feed && feed.bot) || {}; + if (!message.startsWith("/")) { + // In a private chat any message gets the menu. In a group the bot only answers commands. + return isPrivate && message ? replies["/help"] || NOT_READY : null; + } + if (!Object.keys(replies).length) return NOT_READY; + const spaceAt = message.indexOf(" "); + const head = spaceAt === -1 ? message : message.slice(0, spaceAt); + const argument = spaceAt === -1 ? "" : message.slice(spaceAt + 1).trim(); + const command = head.split("@")[0].toLowerCase(); + if (command === "/find") return searchReply(feed.stories || [], argument); + if (replies[command]) return replies[command]; + if (SECTION_COMMANDS.includes(command)) return "Nothing under that heading in today's edition. Try /today."; + return replies["/help"]; +} + +async function loadFeed() { + // Cloudflare keeps the feed for two minutes, so a busy chat does not hammer GitHub. + const response = await fetch(FEED_URL, { cf: { cacheTtl: 120, cacheEverything: true } }); + if (!response.ok) return {}; + return response.json(); +} + +const empty = (status = 200) => new Response(null, { status }); + +export default { + async fetch(request, env) { + if (request.method !== "POST") return new Response("Morrow Brief bot", { status: 200 }); + // With no secret configured the Worker answers nobody, so a half-finished setup is never open. + if (!env || !env.WEBHOOK_SECRET) return empty(503); + if (request.headers.get("X-Telegram-Bot-Api-Secret-Token") !== env.WEBHOOK_SECRET) return empty(403); + let update; + try { + update = await request.json(); + } catch { + return empty(400); + } + const message = update && update.message; + if (!message || typeof message.text !== "string" || !message.chat) return empty(); + let feed = {}; + try { + feed = await loadFeed(); + } catch { + feed = {}; + } + const reply = replyFor(feed, message.text, message.chat.type === "private"); + if (reply === null) return empty(); + // Answering inside the webhook response: Telegram runs this as a sendMessage call. + return Response.json({ + method: "sendMessage", + chat_id: message.chat.id, + text: reply, + parse_mode: "HTML", + link_preview_options: { is_disabled: true }, + }); + }, +}; diff --git a/bot/worker.test.mjs b/bot/worker.test.mjs new file mode 100644 index 0000000..89f11b9 --- /dev/null +++ b/bot/worker.test.mjs @@ -0,0 +1,171 @@ +// Run with: node --test bot/ +// The Worker must give exactly the replies the Python bot gives. Both are checked against +// tests/social/fixtures/bot_cases.json, which tests/social/make_fixtures.py writes from a real edition. +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { afterEach, beforeEach, test } from "node:test"; + +import worker, { FEED_URL, replyFor, searchReply, truncate } from "./worker.js"; + +const fixture = JSON.parse( + readFileSync(new URL("../tests/social/fixtures/bot_cases.json", import.meta.url), "utf8"), +); +const SECRET = "webhook-secret-for-tests"; +const realFetch = globalThis.fetch; +let feedRequests = []; + +function serveFeed(respond = () => new Response(JSON.stringify(fixture.feed), { status: 200 })) { + globalThis.fetch = async (url, options) => { + feedRequests.push({ url: String(url), options }); + return respond(); + }; +} + +function webhook(update, { secret = SECRET, body } = {}) { + const headers = { "Content-Type": "application/json" }; + if (secret !== null) headers["X-Telegram-Bot-Api-Secret-Token"] = secret; + return new Request("https://bot.example/", { + method: "POST", + headers, + body: body === undefined ? JSON.stringify(update) : body, + }); +} + +const textMessage = (text, type = "private", id = 77) => ({ + update_id: 1, + message: { message_id: 5, text, chat: { id, type } }, +}); + +beforeEach(() => { + feedRequests = []; + serveFeed(); +}); + +afterEach(() => { + globalThis.fetch = realFetch; +}); + +test("every recorded message gets the same reply as the Python bot", () => { + assert.ok(fixture.cases.length >= 20); + for (const item of fixture.cases) { + assert.equal(replyFor(fixture.feed, item.text, item.private), item.reply, `message: ${JSON.stringify(item.text)}`); + } +}); + +test("before the first edition the bot says so", () => { + for (const empty of [{}, { bot: {} }, null]) { + assert.match(replyFor(empty, "/today"), /not ready/); + assert.match(replyFor(empty, "hi"), /not ready/); + } + assert.equal(replyFor({}, "hi", false), null); +}); + +test("search escapes the query and needs every word", () => { + const miss = searchReply(fixture.feed.stories, "zzzzqqq "); + assert.ok(miss.includes("<b>")); + assert.ok(!miss.includes("")); + const hit = searchReply(fixture.feed.stories, "nvidia"); + assert.ok(hit.includes(" { + const long = "One two three four five six seven eight nine ten"; + for (let limit = 1; limit <= long.length; limit += 1) { + const cut = truncate(long, limit); + assert.ok(Array.from(cut).length <= limit, `limit ${limit}: ${cut}`); + } + assert.equal(truncate(long, 15), "One two three…"); + assert.equal(truncate(long, long.length), long); +}); + +test("a browser visit gets a plain page and reads nothing", async () => { + const response = await worker.fetch(new Request("https://bot.example/"), { WEBHOOK_SECRET: SECRET }); + assert.equal(response.status, 200); + assert.match(await response.text(), /Morrow Brief/); + assert.equal(feedRequests.length, 0); +}); + +test("with no secret configured the Worker answers nobody", async () => { + for (const env of [{}, undefined, { WEBHOOK_SECRET: "" }]) { + const response = await worker.fetch(webhook(textMessage("/today")), env); + assert.equal(response.status, 503); + } + assert.equal(feedRequests.length, 0); +}); + +test("a call without Telegram's secret header is refused", async () => { + for (const secret of ["wrong", "", null]) { + const response = await worker.fetch(webhook(textMessage("/today"), { secret }), { WEBHOOK_SECRET: SECRET }); + assert.equal(response.status, 403); + assert.equal(await response.text(), ""); + } + assert.equal(feedRequests.length, 0); +}); + +test("a body that is not JSON is a 400", async () => { + const response = await worker.fetch(webhook(null, { body: "not json" }), { WEBHOOK_SECRET: SECRET }); + assert.equal(response.status, 400); +}); + +test("updates that are not text messages are accepted and ignored", async () => { + const ignored = [ + { update_id: 1 }, + { update_id: 2, edited_message: { text: "/today", chat: { id: 1, type: "private" } } }, + { update_id: 3, message: { photo: [], chat: { id: 1, type: "private" } } }, + { update_id: 4, message: { text: "/today" } }, + ]; + for (const update of ignored) { + const response = await worker.fetch(webhook(update), { WEBHOOK_SECRET: SECRET }); + assert.equal(response.status, 200); + assert.equal(await response.text(), ""); + } + assert.equal(feedRequests.length, 0); +}); + +test("a command is answered inside the webhook response", async () => { + const response = await worker.fetch(webhook(textMessage("/today", "private", 4242)), { WEBHOOK_SECRET: SECRET }); + assert.equal(response.status, 200); + assert.match(response.headers.get("Content-Type"), /application\/json/); + const payload = await response.json(); + assert.deepEqual(payload, { + method: "sendMessage", + chat_id: 4242, + text: fixture.feed.bot["/today"], + parse_mode: "HTML", + link_preview_options: { is_disabled: true }, + }); + assert.equal(feedRequests.length, 1); + assert.equal(feedRequests[0].url, FEED_URL); +}); + +test("a group message that is not a command gets no reply", async () => { + const response = await worker.fetch(webhook(textMessage("good morning", "supergroup")), { WEBHOOK_SECRET: SECRET }); + assert.equal(response.status, 200); + assert.equal(await response.text(), ""); +}); + +test("when the feed cannot be read the bot says the edition is not ready", async () => { + const failures = [ + () => new Response("missing", { status: 404 }), + () => new Response("not json", { status: 200 }), + () => { + throw new Error("network down"); + }, + ]; + for (const respond of failures) { + serveFeed(respond); + const response = await worker.fetch(webhook(textMessage("/today")), { WEBHOOK_SECRET: SECRET }); + assert.equal(response.status, 200); + const payload = await response.json(); + assert.equal(payload.method, "sendMessage"); + assert.match(payload.text, /not ready/); + } +}); + +test("the Worker holds no bot token and calls nothing but the feed", () => { + const source = readFileSync(new URL("./worker.js", import.meta.url), "utf8"); + assert.ok(!source.includes("api.telegram.org")); + assert.ok(!/BOT_TOKEN/.test(source)); + assert.equal(FEED_URL, "https://raw.githubusercontent.com/juw12/morrow/social-data/feed.json"); +}); diff --git a/docs/SOCIAL.md b/docs/SOCIAL.md new file mode 100644 index 0000000..1fd0608 --- /dev/null +++ b/docs/SOCIAL.md @@ -0,0 +1,241 @@ +# Morrow Brief: the social pages + +Morrow Brief is Morrow's page on Telegram, Instagram, Threads and X. It posts one digest a day and a few alerts, +all taken from the published edition at https://juw12.github.io/morrow/. A Telegram bot answers questions about +the day's edition. + +> Tech and markets, read across the outlets. One brief a day, every story sourced. + +Nothing here is live yet. The code is tested, but no page account, API key or bot token exists, so no post has +been sent. [What is not verified](#what-is-not-verified) lists every part that has only been tested against fakes. + +## Why this page, when AI news channels already exist + +The closest comparison is Girsta AI News (`t.me/girsta`), an automated English AI news channel on Telegram. Morrow +Brief is built to differ in ways a reader sees in the posts themselves: + +| | Girsta AI News | Morrow Brief | +|---|---|---| +| Volume | automated posts through the day | one digest of 5 stories, plus at most 4 alerts a day | +| Scope | AI | AI and chips, big tech, robotics, space, tech, plus a market line | +| Platforms | Telegram | Telegram, Instagram, Threads, X | +| Bot | none found | section commands and `/find ` | + +The table holds only what could be compared. The Girsta column comes from a secondary source (Telepick, about 486 +subscribers); the channel could not be opened from the machine this was built on, so how Girsta sources its +posts, what its images look like and how it writes were not checked and are not compared. + +Three more things are true of Morrow Brief on their own account, whatever a competitor does: + +- **Sourcing.** Every story says how many outlets report it, and the lead links to each one the page names. +- **Cards.** Each post carries its own typographic card: a red bar on the top edge, the headline, and the outlet + count as a large numeral. No publisher photos. +- **Tone.** No emoji, and a plain statement that the selection is automated. + +## What gets posted + +**The digest** goes out once a day, on the first edition published between 06:00 and 22:00 Jakarta time (WIB). +It holds 5 stories: the page's lead story, then the stories the most outlets report, with at most 2 from any +one section. If the edition has fewer than 3 stories, no digest is sent. + +**An alert** is a single story outside the digest that 3 or more outlets report. Alerts are limited to 4 a day, +at least 90 minutes apart, between 06:00 and 22:00 WIB. Most days have none or one. On Telegram an alert arrives +silently; only the digest sends a notification. + +No story is posted twice in a day, and a story in the digest is never sent again as an alert. + +Each platform gets its own format: + +| Platform | Digest | Alert | Limit enforced | +|---|---|---|---| +| Telegram | cover card with a caption: 5 linked headlines, section, outlet count, a link to each outlet the page names for the lead, market line | card, headline, summary, a link to each outlet the page names | 1024 | +| X | thread of 5, one post per story: headline, a one-sentence summary where it fits, outlets, source link; cover card on the first | one post with the card | 280, links counted as 23 | +| Threads | one post: as many whole headlines as fit (at least 3), the rest named by outlet, linking to the edition | headline, summary, source link | 500 | +| Instagram | carousel: cover and one card per story; the caption gives each story a summary and names its outlets | one card | 2200 | + +Instagram captions cannot hold links, so they name the outlets and point to the link in the bio. + +An X post often has no summary. After the headline, the outlets and the link there is rarely room for a whole +sentence, and half a sentence is not sent. In today's thread the first post has 76 characters free and the +lead's first sentence is 213 characters long, so it goes without. The card on that post carries the summary. + +Rules that hold for every post, each checked by a test: + +- every story names its source outlet and says how many outlets report it; +- no emoji, in text or on cards (`▲`, `▼` and `•` are the only marks used); +- a post over its platform limit is never sent; +- a summary is kept as whole sentences or left out, never cut mid-sentence, in captions and on cards alike. + When an outlet's own feed ends a summary mid-sentence, that fragment is dropped. A headline is shortened (at a + word boundary, with an ellipsis) only when the post would not fit otherwise; today's digest shortens none; +- cards are 1080 x 1350, drawn from text alone (no publisher photos), with every colour pair at 4.5:1 or better. + The cover sets the lead headline large (at least 1.6 times the rows when the text allows), with the other + four as numbered rows and the market moves below; +- Instagram captions carry this line: "Automated selection from published reporting. Headlines belong to their + outlets; every story names its source." + +Run `python scripts/social_post.py preview --page index.html --out preview` to see exactly what today's edition +would post. It writes `preview/preview.md`, the cards, and the request bodies, and sends nothing. The "Morrow +Brief" workflow attaches the same preview to every pull request as the artifact `morrow-brief-preview`. + +## How it runs + +``` +private generator --push--> index.html on main + | + .github/workflows/social.yml + | + tests -> build -> save -> deliver -> save + | + branch social-data: state.json, feed.json, cards/ + | | + Buffer fetches the cards the bot reads feed.json +``` + +1. The private generator pushes a new `index.html`. That push starts the workflow. The workflow also starts + itself every 3 hours as a fallback; a run that finds nothing new posts nothing. +2. The tests run. If they fail, nothing is posted. +3. `build` reads the page, decides whether a digest or alert is due, draws the cards and freezes the post into + `state.json`. A frozen post is never rewritten, even if the page changes before it is sent. +4. The state is saved to the `social-data` branch, and only then does `deliver` send: Telegram directly, and + Instagram, Threads and X through Buffer's API. +5. The results are saved again. + +State, cards and the bot's feed live on the `social-data` branch and never touch `main`, so the workflow cannot +collide with the generator's pushes. Each run replaces the branch with a fresh commit, so it does not grow. Posts +and cards older than 7 days are dropped. + +Buffer's free plan allows 3 channels, which is why there are three Buffer platforms. Telegram is sent directly +because Buffer does not support it. + +## What happens when something goes wrong + +Every target (one platform for one post) has a status in `state.json`. The run summary in the Actions tab lists them. + +| Status | Meaning | Sent again? | +|---|---|---| +| `pending` | frozen, not yet tried | yes | +| `sending` | claimed by a run that is about to send it | only by that same run | +| `delivered` | the platform accepted it | no | +| `failed` | a clear refusal or a network error before anything was sent | yes, up to 3 attempts | +| `gave_up` | failed 3 times | no | +| `unconfirmed` | the request may or may not have gone through | no | +| `skipped` | the platform has no credentials | no | +| `expired` | still unsent 3 hours after it was frozen | no | + +The design rule is that a post may be lost but must never appear twice. + +- **The run dies halfway.** Before sending, a run marks its targets `sending` with its own run id and saves that. + If it dies, the next run finds a claim that is not its own and marks the target `unconfirmed` without sending. +- **A timeout or an unreadable reply.** The target becomes `unconfirmed` and is left alone. +- **The data branch cannot be read.** The run fails before posting. Only a branch that does not exist yet + (the very first run) starts from an empty state. +- **The page is stale.** If the edition's date is not today in WIB, nothing is frozen. +- **A wrong channel.** Buffer posts go only to channel ids listed in the variable `MORROW_BUFFER_CHANNELS`. A + key that also holds other channels cannot post to them, and with the variable empty nothing is sent to Buffer. +- **Secrets are added later.** Targets skipped while a platform had no credentials stay skipped, so adding a + secret does not release a backlog of old posts. +- **A card is not yet served.** `deliver` waits up to 90 seconds for the first card URL to answer, then tries + the second URL, and fails the target (to be retried) if neither serves it. +- **Two runs at once.** The publish job is in one concurrency group, so runs queue. + +## The Telegram bot + +The bot answers from `feed.json`, which holds ready-made replies for the day. It stores nothing: no subscriber +list, no chat ids, no message history. + +| Command | Reply | +|---|---| +| `/start`, `/help` | what the bot does and the list of commands | +| `/today` | the day's digest | +| `/ai`, `/bigtech`, `/robotics`, `/space`, `/tech` | that section's stories | +| `/markets` | the markets stories | +| `/tape` | index moves, one per line | +| `/sources` | the story and outlet counts, and every outlet's name | +| `/find ` | up to 6 stories whose headline, summary, topic tag or outlet contains every word given | +| `/about` | what Morrow Brief is and how stories are chosen | + +There are two ways to run it. Use one. + +- **Cloudflare Worker (recommended).** `bot/worker.js` answers each message as it arrives. It holds no bot token: + it replies in the webhook response itself. Its only secret is `WEBHOOK_SECRET`, which Telegram sends back with + every call so the Worker can refuse anyone else. +- **GitHub cron (fallback).** The "Morrow Brief bot" workflow polls every 5 minutes and listens for 4 minutes each + time. GitHub often starts scheduled runs late, so replies can take several minutes. The schedule does nothing + until the variable `MORROW_BOT` is `on`; until then the Actions tab shows a skipped run every 5 minutes, and + the workflow can be disabled there if the Worker is used. + +If a webhook is set, polling is refused by Telegram; the poller logs that and stops. + +## Setup + +Until these steps are done the workflow runs, records every target as `skipped` and posts nothing. + +1. Create the page accounts: a Telegram channel, an Instagram professional account, Threads and X. Suggested + handle `morrowbrief`, with `readmorrow` or `morrowdaily` as fallbacks. None of these has been checked for + availability. +2. In Telegram, create a bot with @BotFather and add it to the channel as an admin that can post. +3. In Buffer, connect the Instagram, Threads and X accounts and create an API key. A separate Buffer account for + Morrow Brief keeps it apart from any personal channels. +4. List the channel ids: + ``` + BUFFER_API_KEY=... python scripts/social_post.py channels + ``` +5. In the repo settings, add: + + | Kind | Name | Value | + |---|---|---| + | secret | `BUFFER_API_KEY` | the Buffer key | + | secret | `TELEGRAM_BOT_TOKEN` | the token from @BotFather | + | secret | `TELEGRAM_CHAT_ID` | the channel, as `@handle` or its numeric id | + | variable | `MORROW_BUFFER_CHANNELS` | the three channel ids, comma separated | + +6. Run "Morrow Brief" once from the Actions tab and read the run summary. + +For the bot, either: + +- create a Cloudflare Worker, paste in `bot/worker.js`, and add a Worker secret `WEBHOOK_SECRET` (any long random + string). Add the same string to the repo as the secret `TELEGRAM_WEBHOOK_SECRET`. Then run "Morrow Brief bot" + from the Actions tab with action `set-webhook` and the Worker's URL; or +- set the repo variable `MORROW_BOT` to `on`. + +To stop posting, disable the "Morrow Brief" workflow in the Actions tab. To stop one platform, remove its id from +`MORROW_BUFFER_CHANNELS`, or remove the Telegram secrets. + +## What is not verified + +Tested here means unit tests against fake HTTP servers, plus a rehearsal of the workflow's git steps against a +local repository. The following have not been run against the real services: + +- a real Buffer `createPost`. The request shape follows Buffer's live schema, read by introspection; +- a real Telegram send, and a real webhook call to the Worker; +- the Worker deployed on Cloudflare (its tests run under Node); +- whether Buffer and Instagram accept card images served from `raw.githubusercontent.com`; +- whether X shows the cover card once or repeats it in the thread; +- whether Threads counts a link at full length (the code assumes it does); +- whether the generator's pushes start the workflow. A push made with the default `GITHUB_TOKEN` of another + workflow would not; a push with a personal token or deploy key does. If they do not, the 3-hour schedule still + posts, but the digest can be up to 3 hours later than the first edition and alerts arrive in 3-hour steps. + GitHub pauses schedules in a repository with no activity for 60 days; +- the publish job itself, which cannot run until this is on `main`; +- handle availability, and the Girsta figures above. + +## Files + +| Path | What it does | +|---|---| +| `scripts/social_feed.py` | reads the edition, picks the digest and alerts, writes the text for each platform | +| `scripts/social_cards.py` | draws the cards | +| `scripts/social_post.py` | `build`, `deliver`, `channels`, `preview` | +| `scripts/social_bot.py` | the polling bot, and `set-webhook` / `delete-webhook` | +| `scripts/social_save.sh` | saves the data folder to the `social-data` branch | +| `bot/worker.js` | the webhook bot | +| `tests/social/` | Python tests; `bot/worker.test.mjs` holds the Worker's | +| `assets/fonts/` | Source Serif 4 and Source Sans 3, under the SIL Open Font License | + +Run the tests with: + +``` +pip install -r scripts/social_requirements.txt +python -m unittest discover -s tests/social +node --test bot/worker.test.mjs +``` diff --git a/scripts/social_bot.py b/scripts/social_bot.py new file mode 100644 index 0000000..c990f01 --- /dev/null +++ b/scripts/social_bot.py @@ -0,0 +1,203 @@ +"""Morrow Brief: the Telegram bot. + + python scripts/social_bot.py poll --feed data/feed.json --listen 240 + python scripts/social_bot.py set-webhook --url https://.workers.dev + python scripts/social_bot.py delete-webhook + +The bot keeps no state and no list of users. Every fixed reply is pre-rendered in feed.json by +`social_post.py build`; only /find is computed here. `bot/worker.js` gives the same answers from a +Cloudflare Worker (instant, through a webhook); `poll` is the fallback that needs no extra account. +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import time +from pathlib import Path + +import requests + +import social_feed as feed + +TELEGRAM_API = "https://api.telegram.org" +LONG_POLL_SECONDS = 25 +MAX_CLOSING_ROUNDS = 5 +NOT_READY = f'Today\'s edition is not ready yet. Read the latest at {feed.SITE_SHORT}.' + + +def command_of(text: str) -> tuple[str, str]: + """Split '/Find@MorrowBriefBot nvidia chips' into ('/find', 'nvidia chips').""" + head, _, rest = text.strip().partition(" ") + return head.split("@", 1)[0].lower(), rest.strip() + + +def reply_for(feed_data: dict, text: str, private: bool = True) -> str | None: + """The bot's whole behaviour: message text in, HTML reply out. None means stay silent.""" + text = (text or "").strip() + if not text.startswith("/"): + # In a private chat any message gets the menu. In a group the bot only answers commands. + return (feed_data.get("bot") or {}).get("/help", NOT_READY) if private and text else None + replies = feed_data.get("bot") or {} + if not replies: + return NOT_READY + command, argument = command_of(text) + if command == "/find": + return feed.search_reply(feed_data.get("stories") or [], argument) + if command in replies: + return replies[command] + if command in feed.SECTION_COMMANDS.values() or command == "/tape": + return "Nothing under that heading in today's edition. Try /today." + return replies["/help"] + + +def scrub(text: str, token: str) -> str: + if token: + text = text.replace(token, "***") + return re.sub(r"bot\d+:[A-Za-z0-9_-]+", "bot***", text)[:300] + + +def call(http, token: str, method: str, payload: dict, timeout: int = 40) -> tuple[int, dict]: + response = http.post(f"{TELEGRAM_API}/bot{token}/{method}", json=payload, timeout=timeout) + try: + body = response.json() + except ValueError: + body = {} + return response.status_code, body + + +def answer(http, token: str, feed_data: dict, update: dict) -> str | None: + """Reply to one update. Returns a log line, or None when there was nothing to answer.""" + message = update.get("message") or {} + chat = message.get("chat") or {} + reply = reply_for(feed_data, message.get("text") or "", chat.get("type") == "private") + if reply is None or "id" not in chat: + return None + status, body = call(http, token, "sendMessage", { + "chat_id": chat["id"], + "text": reply, + "parse_mode": "HTML", + "link_preview_options": {"is_disabled": True}, + }) + command = command_of(message.get("text") or "")[0] or "text" + if status == 200 and body.get("ok"): + return f"answered {command}" + return f"could not answer {command}: {scrub(str(body.get('description') or status), token)}" + + +def poll(http, token: str, feed_data: dict, listen_seconds: int, clock=time.monotonic) -> list[str]: + """Answer pending messages for about listen_seconds, then acknowledge them so they never repeat.""" + log: list[str] = [] + offset: int | None = None + closing_rounds = 0 + deadline = clock() + listen_seconds + while True: + remaining = deadline - clock() + wait = max(0, min(LONG_POLL_SECONDS, int(remaining))) + payload = {"timeout": wait, "allowed_updates": ["message"]} + if offset is not None: + payload["offset"] = offset + try: + status, body = call(http, token, "getUpdates", payload, timeout=wait + 20) + except requests.exceptions.RequestException as error: + log.append(f"Telegram unreachable ({type(error).__name__}), stopping") + return log + if status == 409: + log.append("a webhook is set, so the Worker answers and polling is not needed") + return log + if status != 200 or not body.get("ok"): + log.append("getUpdates failed: " + scrub(str(body.get("description") or status), token)) + return log + updates = body.get("result") or [] + for update in updates: + # Move the offset first: an update that cannot be answered is skipped, never retried forever. + offset = update["update_id"] + 1 + try: + line = answer(http, token, feed_data, update) + except requests.exceptions.RequestException as error: + line = f"could not answer: {type(error).__name__}" + if line: + log.append(line) + if remaining <= 0: + # Telegram forgets an update only when a later call carries a higher offset. So the run ends + # on a call that returned nothing new; a few extra rounds are allowed to drain late arrivals. + if not updates: + return log + if closing_rounds >= MAX_CLOSING_ROUNDS: + # Still busy: acknowledge what was answered and leave the rest for the next run. + # Whatever this call returns has not been answered, so it is not acknowledged. + try: + call(http, token, "getUpdates", {"timeout": 0, "limit": 1, "offset": offset, + "allowed_updates": ["message"]}) + except requests.exceptions.RequestException: + pass + return log + closing_rounds += 1 + + +def command_poll(args) -> int: + token = (os.environ.get("TELEGRAM_BOT_TOKEN") or "").strip() + if not token: + print("TELEGRAM_BOT_TOKEN is not set, nothing to do") + return 0 + path = Path(args.feed) + feed_data = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} + lines = poll(requests, token, feed_data, args.listen) + answered = sum(line.startswith("answered") for line in lines) + for line in [line for line in lines if not line.startswith("answered")]: + print(line) + print(f"{answered} messages answered") + return 0 + + +def command_set_webhook(args) -> int: + token = (os.environ.get("TELEGRAM_BOT_TOKEN") or "").strip() + secret = (os.environ.get("TELEGRAM_WEBHOOK_SECRET") or "").strip() + if not token or not secret: + print("Set TELEGRAM_BOT_TOKEN and TELEGRAM_WEBHOOK_SECRET first.") + return 1 + if not args.url.startswith("https://"): + print("The webhook URL must start with https://") + return 1 + status, body = call(requests, token, "setWebhook", { + "url": args.url, + "secret_token": secret, + "allowed_updates": ["message"], + "drop_pending_updates": True, + }) + print("webhook set" if status == 200 and body.get("ok") + else "setWebhook failed: " + scrub(str(body.get("description") or status), token)) + return 0 if body.get("ok") else 1 + + +def command_delete_webhook(_args) -> int: + token = (os.environ.get("TELEGRAM_BOT_TOKEN") or "").strip() + if not token: + print("Set TELEGRAM_BOT_TOKEN first.") + return 1 + status, body = call(requests, token, "deleteWebhook", {"drop_pending_updates": False}) + print("webhook removed" if status == 200 and body.get("ok") + else "deleteWebhook failed: " + scrub(str(body.get("description") or status), token)) + return 0 if body.get("ok") else 1 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + commands = parser.add_subparsers(dest="command", required=True) + polling = commands.add_parser("poll") + polling.add_argument("--feed", default="data/feed.json") + polling.add_argument("--listen", type=int, default=240, help="seconds to keep answering") + polling.set_defaults(run=command_poll) + hook = commands.add_parser("set-webhook") + hook.add_argument("--url", required=True) + hook.set_defaults(run=command_set_webhook) + unhook = commands.add_parser("delete-webhook") + unhook.set_defaults(run=command_delete_webhook) + args = parser.parse_args(argv) + return args.run(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/social_cards.py b/scripts/social_cards.py new file mode 100644 index 0000000..2ed43fa --- /dev/null +++ b/scripts/social_cards.py @@ -0,0 +1,408 @@ +"""Morrow Brief: typographic cards (1080 x 1350) for the social posts. + +Text only, in the Morrow palette, with the bundled Source Serif 4 and Source Sans 3 fonts so a card +looks the same on the Linux runner as on a desk. No publisher photos are used. +""" +from __future__ import annotations + +import datetime as dt +from functools import lru_cache +from pathlib import Path + +from PIL import Image, ImageDraw, ImageFont + +import social_feed as feed + +WIDTH, HEIGHT = 1080, 1350 +MARGIN = 88 +FONT_DIR = Path(__file__).resolve().parent.parent / "assets" / "fonts" +FONT_FILES = { + "serif_bold": "SourceSerif4-Semibold.ttf", + "serif": "SourceSerif4-Regular.ttf", + "serif_italic": "SourceSerif4-It.ttf", + "sans": "SourceSans3-Regular.ttf", + "sans_bold": "SourceSans3-Semibold.ttf", +} +PALETTE = { + "paper": "#f7f6f2", + "ink": "#14161a", + "ink2": "#474c55", + "ink3": "#646973", + "accent": "#b4232c", + "hairline": "#d9d5cc", + "up": "#0b7a46", + "down": "#c32a1f", +} +# Every colour that carries text or data, all drawn on the paper colour. The tests check each pair. +TEXT_COLOURS = ("ink", "ink2", "ink3", "accent", "up", "down") +HEADLINE_SIZES = (104, 96, 88, 80, 74, 68, 62, 56, 52) +HEADLINE_LEADING = 1.1 +HEADLINE_LINES = 4 # a story headline is set smaller before it runs to a fifth line +HEADLINE_FLOOR = 62 # ... but never smaller than this just to save a line +DEK_SIZES = (46, 42, 38) # the summary is set as large as its whole sentences allow +DEK_LEADING = 1.36 +DEK_GAP = 34 # between the headline and the summary +DEK_BUDGET = 260 # characters of summary a card carries at most +ACCENT_BAR = 20 # the full-bleed bar at the top edge: what marks a card as Morrow Brief at grid size +COVER_LEAD_SIZES = (88, 80, 74, 68, 62, 56, 50, 46) +COVER_ROW_SIZES = (38, 36, 34, 32, 30) +COVER_ROW_LEADING = 1.16 +COVER_ROW_PAD = 14 # between a rule and the row's text, above and below +COVER_ROW_META = 32 # the section and outlet line under a row's headline +COVER_ROW_META_SIZE = 22 +COVER_LEAD_RATIO = 1.6 # the lead stays at least this much larger than the rows under it + + +def rgb(name: str) -> tuple[int, int, int]: + value = PALETTE[name].lstrip("#") + return tuple(int(value[index:index + 2], 16) for index in (0, 2, 4)) + + +def _channel(value: int) -> float: + scaled = value / 255 + return scaled / 12.92 if scaled <= 0.03928 else ((scaled + 0.055) / 1.055) ** 2.4 + + +def contrast_ratio(foreground: str, background: str = "paper") -> float: + """WCAG contrast ratio between two palette colours.""" + def luminance(name: str) -> float: + red, green, blue = (_channel(value) for value in rgb(name)) + return 0.2126 * red + 0.7152 * green + 0.0722 * blue + + lighter, darker = sorted((luminance(foreground), luminance(background)), reverse=True) + return (lighter + 0.05) / (darker + 0.05) + + +@lru_cache(maxsize=None) +def font(role: str, size: int) -> ImageFont.FreeTypeFont: + return ImageFont.truetype(str(FONT_DIR / FONT_FILES[role]), size) + + +def text_width(text: str, face: ImageFont.FreeTypeFont, tracking: float = 0) -> float: + return face.getlength(text) + tracking * max(len(text) - 1, 0) + + +def wrap(text: str, face: ImageFont.FreeTypeFont, max_width: float) -> list[str]: + lines: list[str] = [] + current = "" + for word in text.split(): + candidate = f"{current} {word}".strip() + if current and face.getlength(candidate) > max_width: + lines.append(current) + current = word + else: + current = candidate + if current: + lines.append(current) + return lines + + +def clamp_lines(lines: list[str], max_lines: int, face: ImageFont.FreeTypeFont, max_width: float) -> list[str]: + """Keep at most max_lines, ending the last kept line with an ellipsis if text was dropped.""" + if len(lines) <= max_lines: + return lines + kept = lines[:max_lines] + last = kept[-1].rstrip(" ,;:-") + while last and face.getlength(last + "…") > max_width: + last = last.rsplit(" ", 1)[0] if " " in last else last[:-1] + kept[-1] = last.rstrip(" ,;:-") + "…" + return kept + + +def fit_line(text: str, face: ImageFont.FreeTypeFont, max_width: float, tracking: float = 0) -> str: + """Shorten a single line until it fits, ending with an ellipsis. For outlet-supplied names and labels.""" + if text_width(text, face, tracking) <= max_width: + return text + while text and text_width(text.rstrip(" ,;:-·") + "…", face, tracking) > max_width: + text = text.rsplit(" ", 1)[0] if " " in text else text[:-1] + return text.rstrip(" ,;:-·") + "…" + + +class Canvas: + """A page with a small text API. Records what it draws so the tests can check the layout.""" + + def __init__(self) -> None: + self.image = Image.new("RGB", (WIDTH, HEIGHT), rgb("paper")) + self.draw = ImageDraw.Draw(self.image) + self.boxes: list[dict] = [] + + def text(self, xy, text, role, size, colour, tracking: float = 0, anchor: str = "left", name: str = "") -> float: + """Draw one line with its top at xy[1]. Returns the line's width.""" + face = font(role, size) + width = text_width(text, face, tracking) + left = xy[0] if anchor == "left" else xy[0] - width + if tracking: + cursor = left + for char in text: + self.draw.text((cursor, xy[1]), char, font=face, fill=rgb(colour)) + cursor += face.getlength(char) + tracking + else: + self.draw.text((left, xy[1]), text, font=face, fill=rgb(colour)) + self.boxes.append({ + "name": name or text[:24], + "left": left, + "right": left + width, + "top": xy[1], + "bottom": xy[1] + size * 1.2, + "colour": colour, + }) + return width + + def block(self, xy, lines, role, size, colour, leading: float, name: str = "") -> float: + """Draw wrapped lines. Returns the y just below the block.""" + y = xy[1] + for line in lines: + self.text((xy[0], y), line, role, size, colour, name=name) + y += size * leading + return y + + def rule(self, y: float, colour: str = "hairline", weight: int = 2, left: float = MARGIN, right: float = WIDTH - MARGIN): + self.draw.rectangle([left, y, right, y + weight - 1], fill=rgb(colour)) + + def triangle(self, x: float, y: float, size: float, direction: str, colour: str) -> None: + if direction == "up": + points = [(x, y + size), (x + size, y + size), (x + size / 2, y)] + elif direction == "down": + points = [(x, y), (x + size, y), (x + size / 2, y + size)] + else: + points = [(x, y + size * 0.4), (x + size, y + size * 0.4), (x + size, y + size * 0.6), (x, y + size * 0.6)] + self.draw.polygon(points, fill=rgb(colour)) + + def metrics(self, **extra) -> dict: + return { + "size": self.image.size, + "colours": sorted({box["colour"] for box in self.boxes}), + "min_left": min(box["left"] for box in self.boxes), + "max_right": max(box["right"] for box in self.boxes), + "max_bottom": max(box["bottom"] for box in self.boxes), + **extra, + } + + def save(self, path: Path) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + self.image.save(path, format="PNG", optimize=True) + + +def _date_label(date_iso: str) -> str: + day = dt.date.fromisoformat(date_iso) + return f"{day:%a} {day.day} {day:%b} {day.year}".upper() + + +def _masthead(canvas: Canvas, date_iso: str) -> float: + """Accent bar on the top edge, wordmark left, date right, a double rule under both. Returns the y below it.""" + canvas.draw.rectangle([0, 0, WIDTH, ACCENT_BAR - 1], fill=rgb("accent")) + canvas.text((MARGIN, 84), feed.BRAND, "serif_bold", 50, "ink", name="wordmark") + canvas.text((WIDTH - MARGIN, 104), _date_label(date_iso), "sans_bold", 24, "ink3", tracking=2.5, + anchor="right", name="date") + canvas.rule(168, "ink", 5) + canvas.rule(180, "ink", 1) + return 181 + + +def _footer(canvas: Canvas, right_text: str) -> None: + canvas.rule(HEIGHT - 150, "hairline", 2) + site_width = canvas.text((MARGIN, HEIGHT - 122), feed.SITE_SHORT, "sans_bold", 27, "ink2", name="site") + right_text = fit_line(right_text, font("sans", 27), WIDTH - 2 * MARGIN - site_width - 48) + canvas.text((WIDTH - MARGIN, HEIGHT - 122), right_text, "sans", 27, "ink3", anchor="right", name="footer_right") + + +def _fit_headline(title: str, max_width: float, max_height: float, sizes: tuple[int, ...] = HEADLINE_SIZES, + prefer_lines: int = 0, floor: int = 0) -> tuple[int, list[str]]: + """The largest size at which the whole headline fits. With prefer_lines, a size no smaller than + `floor` that keeps the headline to that many lines wins over a larger one that needs more.""" + fitting = [] + for size in sizes: + face = font("serif_bold", size) + lines = wrap(title, face, max_width) + if len(lines) * size * HEADLINE_LEADING <= max_height and all(face.getlength(line) <= max_width for line in lines): + fitting.append((size, lines)) + if fitting: + compact = [item for item in fitting if len(item[1]) <= prefer_lines and item[0] >= floor] + return compact[0] if compact else fitting[0] + size = sizes[-1] + face = font("serif_bold", size) + max_lines = max(int(max_height // (size * HEADLINE_LEADING)), 1) + return size, clamp_lines(wrap(title, face, max_width), max_lines, face, max_width) + + +def _coverage_block(canvas: Canvas, story: dict, top: float) -> None: + """The card's signature: how many outlets reported the story, and which.""" + inner_width = WIDTH - 2 * MARGIN + names = feed.outlet_names(story) + if story["coverage"] <= 1: + canvas.text((MARGIN, top + 8), "SOURCE", "sans_bold", 24, "ink3", tracking=3, name="coverage_label") + source = fit_line(story["source"], font("serif_bold", 58), inner_width) + canvas.text((MARGIN, top + 46), source, "serif_bold", 58, "ink", name="coverage_names") + return + numeral = str(story["coverage"]) + numeral_width = canvas.text((MARGIN, top - 26), numeral, "serif_bold", 150, "accent", name="coverage_numeral") + text_left = MARGIN + numeral_width + 34 + canvas.text((text_left, top + 8), "OUTLETS REPORTED THIS", "sans_bold", 26, "ink3", tracking=3, + name="coverage_label") + face = font("sans", 34) + listing = ", ".join(names) if len(names) > 1 else f"First: {story['source']}" + hidden = story["coverage"] - len(names) + if len(names) > 1 and hidden > 0: + listing += f" +{hidden}" + lines = clamp_lines(wrap(listing, face, WIDTH - MARGIN - text_left), 2, face, WIDTH - MARGIN - text_left) + canvas.block((text_left, top + 52), lines, "sans", 34, "ink2", 1.25, name="coverage_names") + assert inner_width > 0 + + +def _fit_summary(parts: list[str], max_width: float, max_height: float) -> tuple[int, list[str]]: + """Whole sentences only: as many as fit, then the largest size they fit at. No sentence is ever cut.""" + for count in range(len(parts), 0, -1): + text = " ".join(parts[:count]) + for size in DEK_SIZES: + face = font("serif", size) + lines = wrap(text, face, max_width) + fits_height = len(lines) * size * DEK_LEADING <= max_height + if fits_height and all(face.getlength(line) <= max_width for line in lines): + return size, lines + return DEK_SIZES[-1], [] + + +def render_story_card(story: dict, date_iso: str, path: Path | None = None) -> dict: + """One story: kicker, headline, summary, and the outlet count as the signature element.""" + canvas = Canvas() + inner_width = WIDTH - 2 * MARGIN + y = _masthead(canvas, date_iso) + 54 + kicker = " · ".join(part for part in (story["section"], story["tag"]) if part).upper() + if kicker: + kicker = fit_line(kicker, font("sans_bold", 28), inner_width, 3) + canvas.text((MARGIN, y), kicker, "sans_bold", 28, "accent", tracking=3, name="kicker") + y += 62 + + coverage_top = HEIGHT - 150 - 178 + content_limit = coverage_top - 60 + # The same rule as the captions: whole sentences, and an outlet's cut-off fragment is left out. + parts = feed.sentences(feed.fit_dek(story["dek"], DEK_BUDGET, cut=False)) + smallest = font("serif", DEK_SIZES[-1]) + reserve = 0.0 + if parts: + reserve = DEK_GAP + len(wrap(parts[0], smallest, inner_width)) * DEK_SIZES[-1] * DEK_LEADING + size, lines = _fit_headline(story["title"], inner_width, content_limit - y - reserve, + prefer_lines=HEADLINE_LINES, floor=HEADLINE_FLOOR) + if not parts or size < HEADLINE_FLOOR: + # A long headline gets the room before the summary does, and a card with no summary gives + # the headline one more line before it is set smaller. + size, lines = _fit_headline(story["title"], inner_width, content_limit - y, + prefer_lines=HEADLINE_LINES + (0 if parts else 1), floor=HEADLINE_FLOOR) + y = canvas.block((MARGIN, y), lines, "serif_bold", size, "ink", HEADLINE_LEADING, name="headline") + dek_size, dek_lines = _fit_summary(parts, inner_width, content_limit - y - DEK_GAP) + if dek_lines: + y = canvas.block((MARGIN, y + DEK_GAP), dek_lines, "serif", dek_size, "ink2", DEK_LEADING, name="dek") + content_bottom = y + + canvas.rule(coverage_top - 34, "hairline", 2) + _coverage_block(canvas, story, coverage_top) + _footer(canvas, f"Headline: {story['source']}") + if path: + canvas.save(path) + return canvas.metrics( + headline_size=size, + headline_lines=len(lines), + dek_size=dek_size, + dek_lines=len(dek_lines), + dek_text=" ".join(dek_lines), + overlaps=content_bottom > coverage_top - 34, + ) + + +def render_cover_card(stories: list[dict], markets: list[dict], date_iso: str, outlet_count: int, + path: Path | None = None) -> dict: + """The digest cover: the lead story set large, the rest as a numbered list, the market tape below.""" + canvas = Canvas() + inner_width = WIDTH - 2 * MARGIN + lead, rest = stories[0], stories[1:] + tape_height = 132 if markets else 0 + list_bottom = HEIGHT - 150 - tape_height - 16 + text_left = MARGIN + 64 + row_width = WIDTH - MARGIN - text_left + + # The lead: kicker, then the headline as large as the space above the list allows. + y = _masthead(canvas, date_iso) + 50 + outlets = f"{lead['coverage']} OUTLETS" if lead["coverage"] > 1 else lead["source"].upper() + kicker = " · ".join(part for part in ("THE LEAD", lead["section"].upper(), outlets) if part) + kicker = fit_line(kicker, font("sans_bold", 26), inner_width, 3) + canvas.text((MARGIN, y), kicker, "sans_bold", 26, "accent", tracking=3, name="lead_kicker") + y += 58 + + # Each row is as tall as its own headline needs. Nothing is cut if any size avoids it. After that + # the rows are set as large as they can be while the lead stays clearly larger (COVER_LEAD_RATIO); + # when no size keeps that ratio, the size that leaves the lead largest wins. + lead_title = " ".join(lead["title"].split()) + best = None + for size in COVER_ROW_SIZES: + face = font("serif_bold", size) + rows = [] + for story in rest: + lines = clamp_lines(wrap(story["title"], face, row_width), 2, face, row_width) + height = 2 * COVER_ROW_PAD + len(lines) * size * COVER_ROW_LEADING + COVER_ROW_META + rows.append((story, lines, height)) + top = list_bottom - sum(height for _, _, height in rows) + fitted_size, fitted_lines = _fit_headline(lead["title"], inner_width, top - 30 - y, COVER_LEAD_SIZES) + rows_whole = all(" ".join(lines) == " ".join(story["title"].split()) for story, lines, _ in rows) + balanced = fitted_size >= COVER_LEAD_RATIO * size + rank = (" ".join(fitted_lines) == lead_title, rows_whole, balanced, size if balanced else fitted_size) + if best is None or rank > best[0]: + best = (rank, size, rows, top, fitted_size, fitted_lines) + (lead_whole, rows_whole, _, _), title_size, rows, rows_top, lead_size, lead_lines = best + clipped = not (lead_whole and rows_whole) + + lead_bottom = canvas.block((MARGIN, y), lead_lines, "serif_bold", lead_size, "ink", HEADLINE_LEADING, + name="lead_title") + overlaps = lead_bottom > rows_top + + y = rows_top + for number, (story, lines, height) in enumerate(rows, 2): + canvas.rule(y, "hairline", 2) + row_top = y + COVER_ROW_PAD + canvas.text((MARGIN, row_top - 6), str(number), "serif_bold", 48, "accent", name=f"numeral_{number}") + bottom = canvas.block((text_left, row_top), lines, "serif_bold", title_size, "ink", COVER_ROW_LEADING, + name=f"title_{number}") + outlets = f"{story['coverage']} OUTLETS" if story["coverage"] > 1 else story["source"].upper() + meta = " · ".join(part for part in (story["section"].upper(), outlets) if part) + meta = fit_line(meta, font("sans_bold", COVER_ROW_META_SIZE), row_width, 2) + canvas.text((text_left, bottom + 4), meta, "sans_bold", COVER_ROW_META_SIZE, "ink3", tracking=2, + name=f"meta_{number}") + y += height + + if markets: + canvas.rule(y, "ink", 2) + shown = markets[:4] + column = inner_width / len(shown) + for index, quote in enumerate(shown): + left = MARGIN + index * column + colour = quote["dir"] if quote["dir"] in ("up", "down") else "ink2" + name = fit_line(quote["name"].upper(), font("sans_bold", 22), column - 20, 1.5) + canvas.text((left, y + 22), name, "sans_bold", 22, "ink3", tracking=1.5, + name=f"quote_name_{index}") + canvas.triangle(left, y + 70, 22, quote["dir"], colour) + canvas.text((left + 32, y + 56), f"{quote['pct']:.2f}%", "sans_bold", 40, colour, name=f"quote_pct_{index}") + + count_word = {3: "Three", 4: "Four", 5: "Five", 6: "Six"}.get(len(stories), str(len(stories))) + _footer(canvas, f"{count_word} stories, read across {outlet_count} outlets") + if path: + canvas.save(path) + return canvas.metrics(rows=len(stories), lead_size=lead_size, lead_lines=len(lead_lines), + title_size=title_size, clipped=clipped, overlaps=overlaps) + + +def render_post_cards(post: dict, edition: dict, out_dir: Path) -> list[Path]: + """Render the cards a post needs. A card that already exists is left alone, so it never changes.""" + by_id = {story["id"]: story for story in edition["stories"]} + stories = [by_id[story_id] for story_id in post["story_ids"] if story_id in by_id] + written: list[Path] = [] + for relative in post["cards"]: + path = out_dir / relative + if path.exists(): + continue + if path.stem == "digest": + render_cover_card(stories, edition["markets"], post["date"], feed.outlet_total(edition["stories"]), path) + elif path.stem in by_id: + render_story_card(by_id[path.stem], post["date"], path) + else: + continue + written.append(path) + return written diff --git a/scripts/social_feed.py b/scripts/social_feed.py new file mode 100644 index 0000000..841de12 --- /dev/null +++ b/scripts/social_feed.py @@ -0,0 +1,756 @@ +"""Morrow Brief: turn the published edition (index.html) into social posts. + +Pure functions only: parse the page, pick the stories, write the text for each platform and the +Telegram bot's replies. No network, no files. `social_post.py` schedules and delivers the result. +""" +from __future__ import annotations + +import datetime as dt +import hashlib +import html as htmllib +import re +import unicodedata + +from bs4 import BeautifulSoup + +BRAND = "Morrow Brief" +SITE_URL = "https://juw12.github.io/morrow/" +SITE_SHORT = "juw12.github.io/morrow" +# Cards, state and the bot's feed live on their own branch, so the generator's pushes to main never collide. +DATA_BRANCH = "social-data" +DATA_URL = f"https://raw.githubusercontent.com/juw12/morrow/{DATA_BRANCH}/" +DIGEST_SIZE = 5 +# A headline longer than this is cut at a word; a summary shorter than MIN_DEK is dropped, not shown as a stub. +MAX_TITLE = 300 +MIN_DEK = 40 +DIGEST_DEK = 240 # the longest summary a digest caption gives one story +MIN_THREADS_STORIES = 3 # below this the Threads digest shortens headlines rather than the list +MAX_THREADS_URL = 160 +MAX_PER_SECTION = 2 +ALERT_MIN_OUTLETS = 3 +LIMITS = { + "x": 280, + "threads": 500, + "telegram_caption": 1024, + "telegram_message": 4096, + "instagram": 2200, +} +TAGLINE = "Tech and markets, read across the outlets. One brief a day, every story sourced." +DISCLOSURE = ( + "Automated selection from published reporting. Headlines belong to their outlets; " + "every story names its source." +) +SECTION_HASHTAGS = { + "ai": ["AI", "Chips"], + "big_tech": ["BigTech"], + "robotics": ["Robotics"], + "space": ["Space"], + "technology": ["Tech"], + "markets_news": ["Markets"], +} +SECTION_COMMANDS = { + "ai": "/ai", + "big_tech": "/bigtech", + "robotics": "/robotics", + "space": "/space", + "technology": "/tech", + "markets_news": "/markets", +} +URL_PATTERN = re.compile(r"https?://\S+") +SENTENCE_BREAK = re.compile(r"[.!?][\"'\u201d\u2019)]*\s+") +# A full stop after one of these does not end a sentence. +ABBREVIATIONS = frozenset( + "Mr Mrs Ms Dr Prof Inc Corp Co Ltd vs St Jr Sr Gen Sen Rep Gov No Fig approx est " + "Jan Feb Mar Apr Jun Jul Aug Sep Sept Oct Nov Dec".split() +) +EMOJI_PATTERN = re.compile( + "[\U0001F000-\U0001FAFF\U00002600-\U000027BF\U0001F1E6-\U0001F1FF\uFE0F\u200D]" +) + + +def clean(text: str | None) -> str: + """Collapse whitespace and drop emoji, so an outlet's decorated headline cannot break the no-emoji rule.""" + return re.sub(r"\s+", " ", EMOJI_PATTERN.sub(" ", text or "")).strip() + + +def story_id(url: str) -> str: + return hashlib.sha1(url.encode("utf-8")).hexdigest()[:10] + + +def slugify(text: str) -> str: + return re.sub(r"[^a-z0-9]+", "_", text.lower()).strip("_") + + +# --------------------------------------------------------------------------- parsing + + +def _edition_date(soup: BeautifulSoup) -> dt.date | None: + title = clean(soup.title.string if soup.title else "") + match = re.search(r"(\w+) (\d{1,2}), (\d{4})", title) + if not match: + return None + try: + return dt.datetime.strptime(" ".join(match.groups()), "%B %d %Y").date() + except ValueError: + return None + + +def _section_titles(soup: BeautifulSoup) -> dict[str, str]: + """Map a section's display title to its id, read from the page's own section blocks.""" + titles = {} + for block in soup.select('section[id^="s-"]'): + heading = block.select_one(".block-head h2, h2, h3") + if heading: + titles[clean(heading.get_text(" "))] = block["id"][2:] + return titles + + +def _parse_story(article, section_titles: dict[str, str]) -> dict | None: + link = article.select_one("a.story-link[href]") + heading = article.select_one("h2, h3, h4") + if not link or not heading: + return None + url = link["href"].strip() + title = clean(heading.get_text(" ")) + if not url.startswith("http") or not title: + return None + kicker = clean(article.select_one(".kicker").get_text(" ")) if article.select_one(".kicker") else "" + section, _, tag = kicker.partition(" · ") + block = article.find_parent("section", id=re.compile(r"^s-")) + section_id = block["id"][2:] if block else section_titles.get(section, slugify(section)) + if not section: + # Row stories carry no kicker; their section is the block they sit in. + section = next((name for name, ident in section_titles.items() if ident == section_id), "") + coverage_tag = article.select_one(".cov") + coverage_match = re.search(r"\d+", (coverage_tag.get("title") or coverage_tag.get_text()) if coverage_tag else "") + related = [] + for item in article.select("details.related li a[href]"): + outlet = item.select_one("b") + outlet_name = clean(outlet.get_text(" ")) if outlet else "" + if outlet: + outlet.extract() + related.append({"source": outlet_name, "title": clean(item.get_text(" ")), "url": item["href"].strip()}) + time_tag = article.select_one("time[datetime]") + source_tag = article.select_one(".src") + dek_tag = article.select_one(".dek") + return { + "id": story_id(url), + "title": title, + "url": url, + "source": clean(source_tag.get_text(" ")) if source_tag else "", + "section": section, + "section_id": section_id, + "tag": tag, + "dek": clean(dek_tag.get_text(" ")) if dek_tag else "", + "published_iso": time_tag["datetime"] if time_tag else "", + "coverage": max(int(coverage_match.group()) if coverage_match else 1, 1 + len(related)), + "related": related, + "is_lead": "lead" in (article.get("class") or []), + } + + +def _parse_markets(soup: BeautifulSoup) -> list[dict]: + quotes = [] + for card in soup.select(".qcard"): + name = card.select_one(".qname") + pct = card.select_one(".qpct") + price = card.select_one(".qprice") + number = re.search(r"-?\d+(?:\.\d+)?", pct.get_text() if pct else "") + if not name or not number: + continue + classes = card.get("class") or [] + direction = "up" if "up" in classes else "down" if "down" in classes else "flat" + quotes.append({ + "name": clean(name.get_text(" ")), + "pct": abs(float(number.group())), + "dir": direction, + "price": clean(price.get_text(" ")) if price else "", + }) + return quotes + + +def parse_edition(page_html: str) -> dict: + """Read the published page into {date, updated, stories, markets}. Stories keep page order.""" + soup = BeautifulSoup(page_html, "html.parser") + section_titles = _section_titles(soup) + stories: list[dict] = [] + seen: dict[str, dict] = {} + for article in soup.select("article.story"): + story = _parse_story(article, section_titles) + if not story: + continue + if story["id"] in seen: + first = seen[story["id"]] + first["coverage"] = max(first["coverage"], story["coverage"]) + first["related"] = first["related"] or story["related"] + continue + story["order"] = len(stories) + seen[story["id"]] = story + stories.append(story) + edition_date = _edition_date(soup) + header = soup.select_one("header") + updated = re.search(r"Updated (\d{1,2}:\d{2})", header.get_text(" ") if header else "") + return { + "date": edition_date.isoformat() if edition_date else "", + "updated": updated.group(1) if updated else "", + "stories": stories, + "markets": _parse_markets(soup), + } + + +# --------------------------------------------------------------------------- selection + + +def ranked(stories: list[dict]) -> list[dict]: + """Lead first, then the most widely reported, then the page's own order.""" + return sorted(stories, key=lambda story: (not story["is_lead"], -story["coverage"], story["order"])) + + +def pick_digest(stories: list[dict], size: int = DIGEST_SIZE, per_section: int = MAX_PER_SECTION) -> list[dict]: + picked: list[dict] = [] + per_section_count: dict[str, int] = {} + for story in ranked(stories): + if len(picked) == size: + break + if per_section_count.get(story["section_id"], 0) >= per_section: + continue + per_section_count[story["section_id"]] = per_section_count.get(story["section_id"], 0) + 1 + picked.append(story) + return picked + + +def alert_candidates(stories: list[dict], used_ids: set[str]) -> list[dict]: + return [ + story for story in ranked(stories) + if story["coverage"] >= ALERT_MIN_OUTLETS and story["id"] not in used_ids + ] + + +def outlet_names(story: dict) -> list[str]: + names: list[str] = [] + for name in [story["source"]] + [item["source"] for item in story["related"]]: + if name and name not in names: + names.append(name) + return names + + +def outlet_total(stories: list[dict]) -> int: + return len({name for story in stories for name in outlet_names(story)}) + + +# --------------------------------------------------------------------------- text helpers + + +def x_length(text: str) -> int: + """Length the way X counts it: every link is 23, most Latin and punctuation 1, the rest 2.""" + total = 0 + for char in unicodedata.normalize("NFC", URL_PATTERN.sub("x" * 23, text)): + code = ord(char) + light = ( + code <= 0x10FF + or 0x2000 <= code <= 0x200D + or 0x2010 <= code <= 0x201F + or 0x2032 <= code <= 0x2037 + ) + total += 1 if light else 2 + return total + + +def visible_length(telegram_html: str) -> int: + """Telegram counts a caption after entities are parsed, so tags and hrefs do not count.""" + return len(htmllib.unescape(re.sub(r"<[^>]+>", "", telegram_html))) + + +def truncate(text: str, limit: int, measure=len) -> str: + """Cut at a word boundary and end with an ellipsis; never returns more than `limit`.""" + if measure(text) <= limit: + return text + words = text.split(" ") + while len(words) > 1: + words.pop() + candidate = " ".join(words).rstrip(" ,;:-—–") + "…" + if measure(candidate) <= limit: + return candidate + # A single word that does not fit (a long URL-like token): cut inside it. + word = words[0] + while word: + word = word[:-1] + if word and measure(word + "…") <= limit: + return word + "…" + return "" + + +def fit_title(build, title: str, limit: int, measure=len) -> str: + """Return build(title), shortening the title until the whole text fits the limit.""" + overhead = measure(build("")) + return build(truncate(title, max(limit - overhead, 0), measure)) + + +def sentences(text: str) -> list[str]: + """Split a summary into sentences without breaking on 'Nov. 13', 'U.S. Navy' or 'Inc. said'.""" + parts: list[str] = [] + start = 0 + for match in SENTENCE_BREAK.finditer(text): + word = text[start:match.start()].split(" ")[-1].lstrip("\"'\u201c\u2018(") + following = text[match.end():match.end() + 1] + if text[match.start()] == "." and (word in ABBREVIATIONS or len(word) <= 1 or "." in word): + continue + if not (following.isupper() or following in "\"'\u201c\u2018("): + continue + parts.append(text[start:match.end()].strip()) + start = match.end() + tail = text[start:].strip() + if tail: + parts.append(tail) + return parts + + +def fit_dek(dek: str, room: int, measure=len, cut: bool = True) -> str: + """The summary as whole sentences that fit `room`. + + Outlet feeds often end a summary mid-sentence with an ellipsis; that fragment is dropped when a + whole sentence stands before it. With cut=False nothing is ever shortened inside a sentence: the + summary is left out instead. With cut=True the last resort is a word-boundary cut with an ellipsis. + """ + dek = dek.strip() + if not dek or room < MIN_DEK: + return "" + parts = sentences(dek) + if parts and parts[-1].endswith(("\u2026", "...")): + parts = parts[:-1] + kept = "" + for part in parts: + candidate = f"{kept} {part}".strip() + if measure(candidate) > room: + break + kept = candidate + if kept and (kept == dek or len(kept) >= MIN_DEK): + return kept + if not cut: + return "" + return truncate(dek, room, measure) + + +def esc(text: str) -> str: + return htmllib.escape(text, quote=False) + + +def link(url: str, label: str) -> str: + return f'{esc(label)}' + + +def short_date(date_iso: str) -> str: + day = dt.date.fromisoformat(date_iso) + return f"{day:%a} {day.day} {day:%b}" + + +def long_date(date_iso: str) -> str: + day = dt.date.fromisoformat(date_iso) + return f"{day:%A} {day.day} {day:%B} {day.year}" + + +def outlets_phrase(story: dict, max_names: int = 4) -> str: + """'5 outlets: Space.com, Spaceflight Now +3', '3 outlets, first: CNBC' or 'via CNBC'. + + The page lists the other outlets only for its lead story. Elsewhere it gives the count alone, + so the count is kept and only the outlet actually linked is named. + """ + names = outlet_names(story) + if story["coverage"] <= 1: + return f"via {story['source']}" + if len(names) <= 1: + return f"{story['coverage']} outlets, first: {story['source']}" + shown = names[:max_names] + hidden = story["coverage"] - len(shown) + return f"{story['coverage']} outlets: {', '.join(shown)}" + (f" +{hidden}" if hidden > 0 else "") + + +def reported_by(story: dict, max_names: int = 4) -> str: + """The same fact as outlets_phrase, as a full phrase for captions and cards.""" + phrase = outlets_phrase(story, max_names) + if phrase.startswith("via "): + return "Reported by " + phrase[4:] + return "Reported by " + phrase + + +def section_line(story: dict, max_names: int = 2) -> str: + return " · ".join(part for part in (story["section"], outlets_phrase(story, max_names)) if part) + + +def outlet_links(story: dict) -> str: + """Every outlet's own article, so the reader picks the source.""" + links = [link(story["url"], story["source"])] + seen = {story["source"]} + for item in story["related"]: + if item["source"] and item["source"] not in seen: + seen.add(item["source"]) + links.append(link(item["url"], item["source"])) + return " · ".join(links) + + +def markets_line(markets: list[dict], count: int = 4) -> str: + arrows = {"up": "▲", "down": "▼", "flat": "="} + parts = [f"{quote['name']} {arrows[quote['dir']]}{quote['pct']:.2f}%" for quote in markets[:count]] + return " · ".join(parts) + + +def hashtags(stories: list[dict], limit: int = 5) -> str: + tags: list[str] = [] + for story in stories: + candidates = SECTION_HASHTAGS.get(story["section_id"], []) + [re.sub(r"[^A-Za-z0-9]", "", story["tag"])] + for tag in candidates: + if tag and tag not in tags: + tags.append(tag) + return " ".join(f"#{tag}" for tag in tags[:limit]) + + +def has_emoji(text: str) -> bool: + return bool(EMOJI_PATTERN.search(text)) + + +def post_problems(post: dict) -> list[str]: + """Every rule a post must pass before it is frozen. An empty list means it is safe to send.""" + problems: list[str] = [] + checks = [("telegram", visible_length(post["telegram"]), LIMITS["telegram_caption"]), + ("threads", len(post["threads"]), LIMITS["threads"]), + ("instagram", len(post["instagram"]), LIMITS["instagram"])] + checks += [(f"x {number}", x_length(text), LIMITS["x"]) for number, text in enumerate(post["x"], 1)] + for name, length, limit in checks: + if length > limit: + problems.append(f"{name} text is {length}, the limit is {limit}") + if length == 0: + problems.append(f"{name} text is empty") + texts = [post["telegram"], post["threads"], post["instagram"], *post["x"], *post["alts"]] + if any(has_emoji(text) for text in texts): + problems.append("emoji in the text") + if " str: + title = truncate(story["title"], MAX_TITLE) + + def build(dek: str) -> str: + lines = [f"{esc(title)}"] + if dek: + lines += ["", esc(dek)] + if len(outlet_names(story)) > 1: + lines += ["", f"{story['coverage']} outlets are on this. Pick your source:", outlet_links(story)] + elif story["coverage"] > 1: + lines += ["", f"{story['coverage']} outlets are on this. First: " + outlet_links(story)] + else: + lines += ["", "Source: " + outlet_links(story)] + return "\n".join(lines) + + room = LIMITS["telegram_caption"] - visible_length(build("")) + return build(fit_dek(story["dek"], room - 2)) + + +def _x_story(story: dict, prefix: str, max_names: int) -> str: + """One X post: headline, a summary only as whole sentences, the outlets, the source link.""" + def build(title: str, dek: str = "") -> str: + middle = f"\n\n{dek}" if dek else "" + return f"{prefix}{title}{middle}\n\n{outlets_phrase(story, max_names)}\n{story['url']}" + + title = truncate(story["title"], max(LIMITS["x"] - x_length(build("")), 0), x_length) + room = LIMITS["x"] - x_length(build(title)) - 2 + return build(title, fit_dek(story["dek"], room, x_length, cut=False)) + + +def story_x(story: dict) -> str: + return _x_story(story, "", 3) + + +def story_threads(story: dict) -> str: + # Threads counts a link at full length, so a tracking-laden URL gives way to the edition link. + url = story["url"] if len(story["url"]) <= MAX_THREADS_URL else SITE_URL + + def build(title: str, dek: str = "") -> str: + middle = f"\n\n{dek}" if dek else "" + return f"{title}{middle}\n\n{outlets_phrase(story)}\n{url}" + + title = truncate(story["title"], max(LIMITS["threads"] - len(build("")), 0)) + room = LIMITS["threads"] - len(build(title)) - 2 + return build(title, fit_dek(story["dek"], room)) + + +def story_instagram(story: dict) -> str: + def build(dek: str) -> str: + lines = [truncate(story["title"], MAX_TITLE), ""] + if dek: + lines += [dek, ""] + lines += [ + reported_by(story, 6), + "", + f"Every story links to its source: {SITE_SHORT} (link in bio).", + DISCLOSURE, + "", + hashtags([story]), + ] + return "\n".join(lines) + + room = LIMITS["instagram"] - len(build("")) - 2 + return build(fit_dek(story["dek"], room)) + + +def story_alt(story: dict) -> str: + return truncate(f"{BRAND} card. {story['title']}. {reported_by(story, 6)}.", 400) + + +def story_post(story: dict, date_iso: str) -> dict: + return { + "id": f"story-{story['id']}", + "kind": "alert", + "date": date_iso, + "story_ids": [story["id"]], + "cards": [f"cards/{date_iso}/{story['id']}.png"], + "alts": [story_alt(story)], + "telegram": story_telegram(story), + "x": [story_x(story)], + "threads": story_threads(story), + "instagram": story_instagram(story), + } + + +# --------------------------------------------------------------------------- digest post + + +def digest_telegram(stories: list[dict], markets: list[dict], date_iso: str) -> str: + """The caption under the cover card. The lead carries a link to every outlet the page names.""" + limit = LIMITS["telegram_caption"] + + def build(title_budget: int, lead_links: bool = False, lead_dek: str = "") -> str: + lines = [f"{BRAND} · {short_date(date_iso)}", ""] + for number, story in enumerate(stories, 1): + lines.append(f"{number}. {link(story['url'], truncate(story['title'], title_budget))}") + if number == 1 and lead_dek: + lines.append(esc(lead_dek)) + if number == 1 and lead_links and len(outlet_names(story)) > 1: + hidden = story["coverage"] - len(outlet_names(story)) + more = f" +{hidden}" if hidden > 0 else "" + lines.append(f"{esc(story['section'])} · {story['coverage']} outlets: {outlet_links(story)}{more}") + else: + lines.append(esc(section_line(story))) + lines.append("") + if markets: + lines += [esc("Markets: " + markets_line(markets)), ""] + lines.append(link(SITE_URL, "Read the full edition")) + return "\n".join(lines) + + # First choice: full headlines and the lead's outlets as links, plus its summary if whole + # sentences fit. Then the plain list, shortening headlines only when nothing else is left. + if stories and visible_length(build(140, lead_links=True)) <= limit: + room = limit - visible_length(build(140, lead_links=True)) - 1 + return build(140, lead_links=True, lead_dek=fit_dek(stories[0]["dek"], room, cut=False)) + for title_budget in range(140, 30, -10): + text = build(title_budget) + if visible_length(text) <= limit: + return text + return build(30) + + +def digest_x(stories: list[dict], date_iso: str) -> list[str]: + """One thread: each post is a story with its summary, outlets and link. The first carries the masthead.""" + total = len(stories) + thread = [] + for number, story in enumerate(stories, 1): + head = f"{BRAND} · {short_date(date_iso)}\n\n" if number == 1 else "" + thread.append(_x_story(story, f"{head}{number}/{total} ", 2)) + return thread + + +def digest_threads(stories: list[dict], date_iso: str) -> str: + """One post. Headlines are printed whole; stories that do not fit are named by outlet instead.""" + def line(number: int, story: dict, title: str) -> str: + extra = f" +{story['coverage'] - 1}" if story["coverage"] > 1 else "" + return f"{number}. {title} ({story['source']}{extra})" + + def build(shown: int) -> str: + lines = [f"{BRAND} · {short_date(date_iso)}", ""] + lines += [line(number, story, story["title"]) for number, story in enumerate(stories[:shown], 1)] + rest = stories[shown:] + if rest: + outlets = list(dict.fromkeys(story["source"] for story in rest)) + footer = f"+{len(rest)} more from {', '.join(outlets)}. Full edition, every story sourced: {SITE_URL}" + else: + footer = f"Full edition, every story linked to its source: {SITE_URL}" + return "\n".join(lines + ["", footer]) + + for shown in range(len(stories), min(MIN_THREADS_STORIES, len(stories)) - 1, -1): + if len(build(shown)) <= LIMITS["threads"]: + return build(shown) + + # Headlines too long to print even three whole: keep every story and shorten the headlines. + def build_short(title_budget: int) -> str: + lines = [f"{BRAND} · {short_date(date_iso)}", ""] + lines += [line(number, story, truncate(story["title"], title_budget)) + for number, story in enumerate(stories, 1)] + return "\n".join(lines + ["", f"Full edition, every story sourced: {SITE_URL}"]) + + for title_budget in range(110, 20, -5): + if len(build_short(title_budget)) <= LIMITS["threads"]: + return build_short(title_budget) + return build_short(20) + + +def digest_instagram(stories: list[dict], markets: list[dict], date_iso: str) -> str: + def build(title_budget: int, with_summaries: bool) -> str: + lines = [f"{BRAND} · {long_date(date_iso)}", ""] + for number, story in enumerate(stories, 1): + lines.append(f"{number}. {truncate(story['title'], title_budget)}") + summary = fit_dek(story["dek"], DIGEST_DEK, cut=False) if with_summaries else "" + if summary: + lines.append(f" {summary}") + lines += [f" {reported_by(story, 4)}", ""] + if markets: + lines += ["Markets: " + markets_line(markets), ""] + lines += [ + f"Every story links to its source: {SITE_SHORT} (link in bio).", + DISCLOSURE, + "", + hashtags(stories), + ] + return "\n".join(lines) + + if len(build(MAX_TITLE, True)) <= LIMITS["instagram"]: + return build(MAX_TITLE, True) + # Shorten the headlines, never the list: every story keeps its line and its outlets. + for title_budget in range(MAX_TITLE, 40, -20): + text = build(title_budget, False) + if len(text) <= LIMITS["instagram"]: + return text + return build(40, False) + + +def digest_post(stories: list[dict], markets: list[dict], date_iso: str) -> dict: + return { + "id": f"digest-{date_iso}", + "kind": "digest", + "date": date_iso, + "story_ids": [story["id"] for story in stories], + "cards": [f"cards/{date_iso}/digest.png"] + [f"cards/{date_iso}/{story['id']}.png" for story in stories], + "alts": [truncate(f"{BRAND} for {long_date(date_iso)}: " + "; ".join(s["title"] for s in stories), 400)] + + [story_alt(story) for story in stories], + "telegram": digest_telegram(stories, markets, date_iso), + "x": digest_x(stories, date_iso), + "threads": digest_threads(stories, date_iso), + "instagram": digest_instagram(stories, markets, date_iso), + } + + +# --------------------------------------------------------------------------- Telegram bot replies + + +def _story_line(story: dict) -> str: + extra = f" +{story['coverage'] - 1}" if story["coverage"] > 1 else "" + return f"• {link(story['url'], truncate(story['title'], 120))} ({esc(story['source'])}{extra})" + + +def _list_reply(heading: str, stories: list[dict], footer: str = "") -> str: + lines = [f"{esc(heading)}", ""] + [_story_line(story) for story in stories] + if footer: + lines += ["", footer] + text = "\n".join(lines) + while visible_length(text) > LIMITS["telegram_message"] and stories: + stories = stories[:-1] + text = "\n".join([f"{esc(heading)}", ""] + [_story_line(story) for story in stories]) + return text + + +def bot_replies(edition: dict) -> dict[str, str]: + """Every fixed command's reply, pre-rendered, so the bot itself needs no logic or state.""" + stories = edition["stories"] + date_label = short_date(edition["date"]) if edition["date"] else "today" + section_names: dict[str, str] = {} + for story in stories: + section_names.setdefault(story["section_id"], story["section"]) + commands = [f"{command} {section_names[section]}" for section, command in SECTION_COMMANDS.items() + if section in section_names] + help_text = "\n".join([ + f"{BRAND}", + TAGLINE, + "", + "/today the five stories that matter", + *commands, + "/tape index moves", + "/sources outlets in today's edition", + "/find word search today's stories", + "/about how this works", + ]) + replies = { + "/start": help_text, + "/help": help_text, + "/today": _list_reply( + f"{BRAND} · {date_label}", pick_digest(stories), + esc("Markets: " + markets_line(edition["markets"])) if edition["markets"] else "", + ), + "/about": ( + f"{BRAND} is built from Morrow, a daily paper that reads hundreds of headlines and merges " + "coverage of the same story across outlets. Selection is automated: the most widely reported stories " + "come first. Nothing is rewritten; headlines belong to their outlets and every story links to them. " + f"Not financial advice.\n\n{link(SITE_URL, 'Full edition')}" + ), + } + for section, command in SECTION_COMMANDS.items(): + section_stories = [story for story in ranked(stories) if story["section_id"] == section][:6] + if section_stories: + replies[command] = _list_reply(f"{section_names[section]} · {date_label}", section_stories) + if edition["markets"]: + tape = [f"Tape · {date_label}", ""] + for quote in edition["markets"]: + arrow = {"up": "▲", "down": "▼", "flat": "="}[quote["dir"]] + tape.append(esc(f"{quote['name']} {quote['price']} {arrow}{quote['pct']:.2f}%")) + tape += ["", "Not financial advice."] + replies["/tape"] = "\n".join(tape) + names = sorted({name for story in stories for name in outlet_names(story)}, key=str.lower) + replies["/sources"] = ( + f"{len(stories)} stories from {len(names)} outlets · {date_label}\n\n" + esc(", ".join(names)) + ) + return replies + + +def search_reply(stories: list[dict], query: str) -> str: + words = [word for word in re.findall(r"[a-z0-9]+", query.lower()) if len(word) > 1] + if not words: + return "Send /find and a word, for example: /find nvidia" + hits = [] + for story in ranked(stories): + haystack = " ".join([story["title"], story["dek"], story["tag"], story["source"]]).lower() + if all(word in haystack for word in words): + hits.append(story) + if not hits: + return f"Nothing on “{esc(query.strip())}” in today's edition. Try /today." + noun = "story" if len(hits) == 1 else "stories" + return _list_reply(f"{len(hits)} {noun} on “{query.strip()}”", hits[:6]) + + +# --------------------------------------------------------------------------- public feed + + +def public_feed(edition: dict, posts: list[dict]) -> dict: + """What gets published as feed.json on the data branch: today's stories, posts and the bot's replies.""" + return { + "brand": BRAND, + "site": SITE_URL, + "date": edition["date"], + "updated": edition["updated"], + "stats": {"stories": len(edition["stories"]), "outlets": outlet_total(edition["stories"])}, + "markets": edition["markets"], + "stories": [ + {key: story[key] for key in ( + "id", "title", "url", "source", "section", "section_id", "tag", "dek", + "published_iso", "coverage", "related", "is_lead", "order", + )} + for story in edition["stories"] + ], + "posts": posts, + "bot": bot_replies(edition) if edition["date"] else {}, + } diff --git a/scripts/social_post.py b/scripts/social_post.py new file mode 100644 index 0000000..316992b --- /dev/null +++ b/scripts/social_post.py @@ -0,0 +1,647 @@ +"""Morrow Brief: decide what to post, freeze it, and deliver it. + + python scripts/social_post.py build --page index.html --out data --run + python scripts/social_post.py deliver --out data --run --image-base https://raw.githubusercontent.com/// + python scripts/social_post.py preview --page index.html --out preview + python scripts/social_post.py channels + +`build` reads the published page, picks at most one new post (the day's digest, or one alert), renders +its cards and freezes it in state.json. `deliver` sends every frozen post that is still pending: +Telegram through the Bot API, Instagram, Threads and X through Buffer. With no credentials nothing is +sent and the targets are recorded as skipped, so the workflow is safe to merge before any account exists. + +The workflow saves state.json between the two commands. `build --run` marks every target it is about to +send as `sending` for that run, so a run that dies after sending but before saving the result can never +cause a second send: the next run finds the stale mark and records the target as unconfirmed. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +import re +import shutil +import sys +import time +from pathlib import Path + +import requests + +import social_cards as cards +import social_feed as feed + +WIB = dt.timezone(dt.timedelta(hours=7), "WIB") +WINDOW_START_HOUR = 6 +WINDOW_END_HOUR = 22 +MAX_ALERTS_PER_DAY = 4 +ALERT_GAP = dt.timedelta(minutes=90) +MIN_DIGEST_STORIES = 3 +MAX_ATTEMPTS = 3 +MAX_POST_AGE = dt.timedelta(hours=3) +KEEP_DAYS = 7 +TARGETS = ("telegram", "instagram", "threads", "x") +BUFFER_SERVICES = {"instagram": "instagram", "threads": "threads", "twitter": "x", "x": "x"} +BUFFER_API = "https://api.buffer.com" +TELEGRAM_API = "https://api.telegram.org" +WAITING_STATUSES = ("pending", "failed") +STALE_CLAIM = "an earlier run started sending and never recorded the result" +CREATE_POST = """ +mutation CreatePost($input: CreatePostInput!) { + createPost(input: $input) { + __typename + ... on PostActionSuccess { post { id status } } + ... on MutationError { message } + } +} +""" + + +class Retryable(Exception): + """The post was definitely not published. Safe to try again on a later run.""" + + +class Unconfirmed(Exception): + """The request may have gone through. Trying again could publish twice, so it is not retried.""" + + +class NotConfigured(Exception): + """No credential or no allowed channel for this target.""" + + +# --------------------------------------------------------------------------- state + + +def empty_state() -> dict: + return {"version": 1, "days": {}, "posts": {}} + + +def load_state(path: Path) -> dict: + if not path.exists(): + return empty_state() + try: + state = json.loads(path.read_text(encoding="utf-8")) + except ValueError: + # A corrupt state must not cause a second digest: stop instead of starting from nothing. + raise SystemExit(f"{path} is not valid JSON. Fix or delete it by hand; refusing to post blind.") + for key, value in empty_state().items(): + state.setdefault(key, value) + return state + + +def save_state(path: Path, state: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(state, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", newline="\n") + + +def day_record(state: dict, date_iso: str) -> dict: + return state["days"].setdefault(date_iso, {"digest": None, "alerts": [], "used": [], "last_post_at": None}) + + +def prune(state: dict, out_dir: Path, today: dt.date) -> None: + """Drop state and card folders older than KEEP_DAYS, so the repo does not grow without limit.""" + cutoff = today - dt.timedelta(days=KEEP_DAYS) + + def is_old(date_iso: str) -> bool: + try: + return dt.date.fromisoformat(date_iso) < cutoff + except ValueError: + return False + + for date_iso in [key for key in state["days"] if is_old(key)]: + del state["days"][date_iso] + for post_id in [key for key, post in state["posts"].items() if is_old(post.get("date", ""))]: + del state["posts"][post_id] + cards_dir = out_dir / "cards" + if cards_dir.is_dir(): + for folder in cards_dir.iterdir(): + if folder.is_dir() and is_old(folder.name): + shutil.rmtree(folder) + + +# --------------------------------------------------------------------------- planning + + +def parse_now(value: str | None) -> dt.datetime: + if not value: + return dt.datetime.now(dt.timezone.utc) + moment = dt.datetime.fromisoformat(value) + return moment if moment.tzinfo else moment.replace(tzinfo=WIB) + + +def plan(edition: dict, state: dict, now: dt.datetime) -> tuple[dict | None, str]: + """Return (the one post to freeze now, or None) and the reason, as plain words for the log.""" + local = now.astimezone(WIB) + if not edition["date"]: + return None, "the page has no readable date" + if edition["date"] != local.date().isoformat(): + return None, f"the page is dated {edition['date']} but today is {local.date()} (stale page, nothing posted)" + if not WINDOW_START_HOUR <= local.hour < WINDOW_END_HOUR: + return None, f"{local:%H:%M} WIB is outside the {WINDOW_START_HOUR:02d}:00-{WINDOW_END_HOUR:02d}:00 window" + record = day_record(state, edition["date"]) + if not record["digest"]: + stories = feed.pick_digest(edition["stories"]) + if len(stories) < MIN_DIGEST_STORIES: + return None, f"only {len(stories)} stories on the page, a digest needs {MIN_DIGEST_STORIES}" + return feed.digest_post(stories, edition["markets"], edition["date"]), "first run of the day in the window" + if len(record["alerts"]) >= MAX_ALERTS_PER_DAY: + return None, f"the day's {MAX_ALERTS_PER_DAY} alerts are used" + if record["last_post_at"]: + since = now - dt.datetime.fromisoformat(record["last_post_at"]) + if since < ALERT_GAP: + return None, f"last post was {int(since.total_seconds() // 60)} min ago, alerts wait {ALERT_GAP.seconds // 60}" + candidates = feed.alert_candidates(edition["stories"], set(record["used"])) + if not candidates: + return None, f"no unused story is reported by {feed.ALERT_MIN_OUTLETS} or more outlets" + story = candidates[0] + return feed.story_post(story, edition["date"]), f"{story['coverage']} outlets report it and it has not been posted" + + +def freeze(state: dict, post: dict, now: dt.datetime) -> dict: + """Record the post exactly as it will be sent. A frozen post is never rebuilt or re-selected.""" + if post["id"] in state["posts"]: + return state["posts"][post["id"]] + record = day_record(state, post["date"]) + overlap = set(post["story_ids"]) & set(record["used"]) + if overlap: + raise ValueError(f"story already posted today: {sorted(overlap)}") + frozen = dict(post) + frozen["created_at"] = now.astimezone(dt.timezone.utc).isoformat(timespec="seconds") + frozen["targets"] = {target: {"status": "pending", "attempts": 0} for target in TARGETS} + state["posts"][post["id"]] = frozen + record["used"] += post["story_ids"] + record["last_post_at"] = frozen["created_at"] + if post["kind"] == "digest": + record["digest"] = post["id"] + else: + record["alerts"].append(post["id"]) + return frozen + + +# --------------------------------------------------------------------------- payloads + + +def image_assets(post: dict, image_base: str, count: int | None = None) -> list[dict]: + pairs = list(zip(post["cards"], post["alts"]))[:count] + return [{"image": {"url": image_base + card, "metadata": {"altText": alt}}} for card, alt in pairs] + + +def buffer_input(post: dict, target: str, channel_id: str, image_base: str) -> dict: + """The CreatePostInput for one target. Shapes follow Buffer's introspected schema (docs/SOCIAL.md).""" + payload = { + "channelId": channel_id, + "mode": "shareNow", + "schedulingType": "automatic", + "needsApproval": False, + "source": "morrow-brief", + } + if target == "instagram": + assets = image_assets(post, image_base, 10) + payload["text"] = post["instagram"] + payload["assets"] = assets + payload["metadata"] = { + "instagram": {"type": "carousel" if len(assets) > 1 else "post", "shouldShareToFeed": True} + } + elif target == "threads": + payload["text"] = post["threads"] + payload["assets"] = image_assets(post, image_base, 1) + elif target == "x": + cover = image_assets(post, image_base, 1) + payload["text"] = post["x"][0] + payload["assets"] = cover + if len(post["x"]) > 1: + payload["metadata"] = { + "twitter": { + "thread": [ + {"text": text, "assets": cover if index == 0 else []} + for index, text in enumerate(post["x"]) + ] + } + } + else: + raise ValueError(f"not a Buffer target: {target}") + return payload + + +def telegram_fields(post: dict, chat_id: str) -> dict: + return { + "chat_id": chat_id, + "caption": post["telegram"], + "parse_mode": "HTML", + # One notification a day: the digest pings, alerts arrive silently. + "disable_notification": "false" if post["kind"] == "digest" else "true", + } + + +# --------------------------------------------------------------------------- delivery + + +def scrub(text: str, secrets: list[str]) -> str: + """Never let a credential reach a log, the state file or an error message.""" + for secret in secrets: + if secret: + text = text.replace(secret, "***") + return re.sub(r"bot\d+:[A-Za-z0-9_-]+", "bot***", text)[:300] + + +def buffer_graphql(http, api_key: str, query: str, variables: dict, is_mutation: bool = False) -> dict: + try: + response = http.post( + BUFFER_API, + headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}, + json={"query": query, "variables": variables}, + timeout=45, + ) + except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError) as error: + raise Retryable(f"Buffer unreachable ({type(error).__name__})") from error + except requests.exceptions.RequestException as error: + if is_mutation: + raise Unconfirmed(f"no answer from Buffer ({type(error).__name__})") from error + raise Retryable(f"no answer from Buffer ({type(error).__name__})") from error + if response.status_code == 429: + raise Retryable("Buffer rate limit (HTTP 429)") + if response.status_code in (401, 403): + raise Retryable(f"Buffer refused the API key (HTTP {response.status_code})") + if response.status_code >= 500: + if is_mutation: + raise Unconfirmed(f"Buffer server error (HTTP {response.status_code})") + raise Retryable(f"Buffer server error (HTTP {response.status_code})") + try: + body = response.json() + except ValueError as error: + raise Retryable(f"Buffer answered HTTP {response.status_code} without JSON") from error + if body.get("errors"): + raise Retryable("Buffer rejected the request: " + str(body["errors"][0].get("message", "unknown error"))) + return body.get("data") or {} + + +def buffer_channels(http, api_key: str) -> list[dict]: + data = buffer_graphql(http, api_key, "query { account { organizations { id name } } }", {}) + found = [] + for organization in ((data.get("account") or {}).get("organizations") or []): + result = buffer_graphql( + http, api_key, + "query Channels($input: ChannelsInput!) { channels(input: $input) " + "{ id name displayName service isDisconnected } }", + {"input": {"organizationId": organization["id"]}}, + ) + for channel in result.get("channels") or []: + found.append({ + "id": channel["id"], + "name": channel.get("displayName") or channel.get("name") or "", + "service": (channel.get("service") or "").lower(), + "disconnected": bool(channel.get("isDisconnected")), + }) + return found + + +def allowed_channels(channels: list[dict], allow: set[str]) -> dict[str, str]: + """Map target -> channel id, using only ids the owner listed. An unlisted channel is never used.""" + chosen: dict[str, str] = {} + for channel in channels: + target = BUFFER_SERVICES.get(channel["service"]) + if target and channel["id"] in allow and not channel["disconnected"]: + chosen.setdefault(target, channel["id"]) + return chosen + + +def send_buffer(http, api_key: str, payload: dict) -> str: + data = buffer_graphql(http, api_key, CREATE_POST, {"input": payload}, is_mutation=True) + result = data.get("createPost") or {} + post = result.get("post") + if post and post.get("id"): + return f"buffer:{post['id']}" + raise Retryable("Buffer did not create the post: " + str(result.get("message") or result.get("__typename") or "no detail")) + + +def send_telegram(http, token: str, fields: dict, card_path: Path, sleep=time.sleep) -> str: + if not card_path.exists(): + raise Retryable(f"card file missing: {card_path.name}") + for attempt in range(2): + try: + with card_path.open("rb") as handle: + response = http.post( + f"{TELEGRAM_API}/bot{token}/sendPhoto", + data=fields, + files={"photo": (card_path.name, handle, "image/png")}, + timeout=60, + ) + except (requests.exceptions.ConnectTimeout, requests.exceptions.ConnectionError) as error: + raise Retryable(f"Telegram unreachable ({type(error).__name__})") from error + except requests.exceptions.RequestException as error: + raise Unconfirmed(f"no answer from Telegram ({type(error).__name__})") from error + try: + body = response.json() + except ValueError: + body = {} + if response.status_code == 200 and body.get("ok"): + return f"telegram:{body['result'].get('message_id')}" + if response.status_code == 429: + wait = int((body.get("parameters") or {}).get("retry_after", 60)) + if attempt == 0 and wait <= 30: + sleep(wait) + continue + raise Retryable(f"Telegram rate limit, retry after {wait}s") + if response.status_code >= 500: + raise Unconfirmed(f"Telegram server error (HTTP {response.status_code})") + raise Retryable(f"Telegram refused: {body.get('description') or response.status_code}") + raise Retryable("Telegram rate limit") + + +def find_image_base(http, candidates: list[str], probe: str, wait_seconds: int = 0, sleep=time.sleep) -> str | None: + """Return the first base URL that serves the probe card as an image. Only the first one is waited for.""" + for index, base in enumerate(candidates): + deadline = time.monotonic() + (wait_seconds if index == 0 else 0) + while True: + try: + response = http.get(base + probe, timeout=20, stream=True) + if response.status_code == 200 and str(response.headers.get("Content-Type", "")).startswith("image/"): + return base + except requests.exceptions.RequestException: + pass + if time.monotonic() >= deadline: + break + sleep(15) + return None + + +def claim(state: dict, run_id: str) -> int: + """Mark every target that is waiting to be sent as `sending`, owned by this run. + + The workflow saves the state after this and before `deliver`. If the run then dies between a send and + the final save, the mark is still there for the next run, which records the target as unconfirmed + and does not send it again. + """ + claimed = 0 + for post in state["posts"].values(): + for target in TARGETS: + entry = post["targets"][target] + if entry["status"] in WAITING_STATUSES: + entry["status"] = "sending" + entry["claim"] = run_id + claimed += 1 + return claimed + + +def record_outcome(entry: dict, outcome: str, detail: str = "") -> None: + entry["status"] = outcome + entry.pop("claim", None) + if detail: + entry["detail"] = detail + else: + entry.pop("detail", None) + + +def deliver(state: dict, out_dir: Path, env: dict, now: dt.datetime, http=requests, + image_bases: list[str] | None = None, image_wait: int = 0, sleep=time.sleep, + run_id: str | None = None) -> list[str]: + """Send every waiting target of every frozen post, and every target this run claimed. Returns the log lines.""" + token = (env.get("TELEGRAM_BOT_TOKEN") or "").strip() + chat_id = (env.get("TELEGRAM_CHAT_ID") or "").strip() + api_key = (env.get("BUFFER_API_KEY") or "").strip() + allow = {item for item in re.split(r"[\s,]+", env.get("MORROW_BUFFER_CHANNELS") or "") if item} + secrets = [token, api_key] + log: list[str] = [] + channels: dict[str, str] | None = None + channels_error: Exception | None = None + image_base: str | None = None + image_base_checked = False + + for post in state["posts"].values(): + waiting = [] + for name in TARGETS: + entry = post["targets"][name] + if entry["status"] in WAITING_STATUSES: + waiting.append(name) + elif entry["status"] == "sending": + if run_id and entry.get("claim") == run_id: + waiting.append(name) + else: + # Another run claimed this and never saved a result. It may have been sent. + record_outcome(entry, "unconfirmed", STALE_CLAIM) + log.append(f"{post['id']} {name}: unconfirmed, will not retry ({STALE_CLAIM})") + if not waiting: + continue + age = now - dt.datetime.fromisoformat(post["created_at"]) + for target in waiting: + entry = post["targets"][target] + if age > MAX_POST_AGE: + record_outcome(entry, "expired", "older than 3 hours when delivery was tried") + log.append(f"{post['id']} {target}: expired, not sent") + continue + try: + if target == "telegram": + if not token or not chat_id: + raise NotConfigured("TELEGRAM_BOT_TOKEN or TELEGRAM_CHAT_ID is not set") + reference = send_telegram(http, token, telegram_fields(post, chat_id), out_dir / post["cards"][0], sleep) + else: + if not api_key: + raise NotConfigured("BUFFER_API_KEY is not set") + if not allow: + raise NotConfigured("MORROW_BUFFER_CHANNELS is empty, so no channel is allowed") + if channels is None and channels_error is None: + try: + channels = allowed_channels(buffer_channels(http, api_key), allow) + except Retryable as error: + channels_error = error + if channels_error: + raise channels_error + if target not in channels: + raise NotConfigured(f"no allowed, connected Buffer channel for {target}") + if not image_base_checked: + image_base = find_image_base(http, image_bases or [], post["cards"][0], image_wait, sleep) + image_base_checked = True + if not image_base: + raise Retryable("the cards are not reachable at a public URL yet") + reference = send_buffer(http, api_key, buffer_input(post, target, channels[target], image_base)) + except NotConfigured as error: + record_outcome(entry, "skipped", str(error)) + log.append(f"{post['id']} {target}: skipped ({error})") + except Unconfirmed as error: + entry["attempts"] += 1 + record_outcome(entry, "unconfirmed", scrub(str(error), secrets)) + log.append(f"{post['id']} {target}: unconfirmed, will not retry ({entry['detail']})") + except Retryable as error: + entry["attempts"] += 1 + outcome = "gave_up" if entry["attempts"] >= MAX_ATTEMPTS else "failed" + record_outcome(entry, outcome, scrub(str(error), secrets)) + log.append(f"{post['id']} {target}: {outcome} after attempt {entry['attempts']} ({entry['detail']})") + else: + entry["attempts"] += 1 + entry["ref"] = reference + entry["at"] = now.astimezone(dt.timezone.utc).isoformat(timespec="seconds") + record_outcome(entry, "delivered") + log.append(f"{post['id']} {target}: delivered ({reference})") + return log + + +# --------------------------------------------------------------------------- commands + + +def report(lines: list[str]) -> None: + for line in lines: + print(line) + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as handle: + handle.write("\n".join(f"- {line}" for line in lines) + "\n") + + +def write_if_changed(path: Path, text: str) -> bool: + if path.exists() and path.read_text(encoding="utf-8") == text: + return False + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8", newline="\n") + return True + + +def public_post(post: dict) -> dict: + return {key: value for key, value in post.items() if key != "targets"} + + +def command_build(args) -> int: + out_dir = Path(args.out) + now = parse_now(args.now) + edition = feed.parse_edition(Path(args.page).read_text(encoding="utf-8")) + state = load_state(out_dir / "state.json") + post, reason = plan(edition, state, now) + lines = [f"edition {edition['date'] or 'unknown'} updated {edition['updated'] or '?'}: " + f"{len(edition['stories'])} stories from {feed.outlet_total(edition['stories'])} outlets"] + problems = feed.post_problems(post) if post else [] + if problems: + # A post that breaks a platform rule is never frozen, so it can never be sent. + lines.append(f"not posted, {post['id']} failed its checks: " + "; ".join(problems)) + elif post: + cards.render_post_cards(post, edition, out_dir) + freeze(state, post, now) + lines.append(f"frozen {post['id']}: {reason}") + else: + lines.append(f"no new post: {reason}") + prune(state, out_dir, now.astimezone(WIB).date()) + if args.run_id: + claimed = claim(state, args.run_id) + if claimed: + lines.append(f"{claimed} targets claimed by run {args.run_id}") + save_state(out_dir / "state.json", state) + todays = [public_post(item) for item in state["posts"].values() if item["date"] == edition["date"]] + feed_text = json.dumps(feed.public_feed(edition, todays), indent=1, ensure_ascii=False) + "\n" + if edition["date"] and write_if_changed(out_dir / "feed.json", feed_text): + lines.append("feed.json updated") + report(lines) + return 0 + + +def command_deliver(args) -> int: + out_dir = Path(args.out) + state = load_state(out_dir / "state.json") + lines = deliver(state, out_dir, dict(os.environ), parse_now(args.now), + image_bases=args.image_base or [], image_wait=args.image_wait, run_id=args.run_id) + save_state(out_dir / "state.json", state) + report(lines or ["nothing to deliver"]) + return 0 + + +def command_channels(_args) -> int: + api_key = (os.environ.get("BUFFER_API_KEY") or "").strip() + if not api_key: + print("Set BUFFER_API_KEY first.") + return 1 + try: + found = buffer_channels(requests, api_key) + except Retryable as error: + print(scrub(str(error), [api_key])) + return 1 + for channel in found: + state = "disconnected" if channel["disconnected"] else "connected" + usable = "usable" if channel["service"] in BUFFER_SERVICES else "not used by Morrow Brief" + print(f"{channel['id']} {channel['service']:<10} {channel['name']} ({state}, {usable})") + print("Put the ids of the Morrow Brief channels, comma separated, in the repo variable MORROW_BUFFER_CHANNELS.") + return 0 + + +def sample_alert(edition: dict, digest: dict) -> tuple[dict | None, str]: + used = set(digest["story_ids"]) + candidates = feed.alert_candidates(edition["stories"], used) + if candidates: + return feed.story_post(candidates[0], edition["date"]), "a real alert candidate on this page" + rest = [story for story in feed.ranked(edition["stories"]) if story["id"] not in used] + if not rest: + return None, "" + return feed.story_post(rest[0], edition["date"]), ( + "format sample only: no story outside the digest is reported by " + f"{feed.ALERT_MIN_OUTLETS} or more outlets on this page, so no alert would be sent" + ) + + +def command_preview(args) -> int: + """Render what would be posted from a page, without state, schedule or network. Used on pull requests.""" + out_dir = Path(args.out) + edition = feed.parse_edition(Path(args.page).read_text(encoding="utf-8")) + stories = feed.pick_digest(edition["stories"]) + if len(stories) < MIN_DIGEST_STORIES: + print("not enough stories on the page for a digest") + return 1 + digest = feed.digest_post(stories, edition["markets"], edition["date"]) + posts = [(digest, "the day's digest")] + alert, note = sample_alert(edition, digest) + if alert: + posts.append((alert, note)) + image_base = feed.DATA_URL + lines = [f"# {feed.BRAND} preview for {edition['date']}", ""] + payloads = {} + for post, note in posts: + cards.render_post_cards(post, edition, out_dir) + lines += [f"## {post['id']} ({note})", "", "Cards: " + ", ".join(post["cards"]), ""] + lines += [f"### Telegram caption ({feed.visible_length(post['telegram'])} of {feed.LIMITS['telegram_caption']})", + "", "```html", post["telegram"], "```", ""] + for number, text in enumerate(post["x"], 1): + lines += [f"### X {number}/{len(post['x'])} ({feed.x_length(text)} of {feed.LIMITS['x']})", + "", "```text", text, "```", ""] + lines += [f"### Threads ({len(post['threads'])} of {feed.LIMITS['threads']})", + "", "```text", post["threads"], "```", ""] + lines += [f"### Instagram ({len(post['instagram'])} of {feed.LIMITS['instagram']})", + "", "```text", post["instagram"], "```", ""] + payloads[post["id"]] = { + "telegram": telegram_fields(post, ""), + **{target: buffer_input(post, target, f"<{target}-channel-id>", image_base) + for target in ("instagram", "threads", "x")}, + } + (out_dir / "preview.md").write_text("\n".join(lines), encoding="utf-8", newline="\n") + (out_dir / "payloads.json").write_text( + json.dumps(payloads, indent=2, ensure_ascii=False) + "\n", encoding="utf-8", newline="\n") + (out_dir / "feed.json").write_text( + json.dumps(feed.public_feed(edition, [post for post, _ in posts]), indent=1, ensure_ascii=False) + "\n", + encoding="utf-8", newline="\n") + print(f"preview written to {out_dir}: {len(posts)} posts, {sum(len(post['cards']) for post, _ in posts)} card slots") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + commands = parser.add_subparsers(dest="command", required=True) + build = commands.add_parser("build") + build.add_argument("--page", default="index.html") + build.add_argument("--out", default="data") + build.add_argument("--now", help="ISO time, for tests") + build.add_argument("--run", dest="run_id", help="id of this workflow run; claims the waiting targets before the state is saved") + build.set_defaults(run=command_build) + send = commands.add_parser("deliver") + send.add_argument("--out", default="data") + send.add_argument("--now", help="ISO time, for tests") + send.add_argument("--run", dest="run_id", help="the id given to build; targets claimed by any other run are not sent") + send.add_argument("--image-base", action="append", + help="public URL of the data folder, where Buffer fetches the cards; repeat for a fallback") + send.add_argument("--image-wait", type=int, default=90, help="seconds to wait for the first URL to serve a card") + send.set_defaults(run=command_deliver) + listing = commands.add_parser("channels") + listing.set_defaults(run=command_channels) + preview = commands.add_parser("preview") + preview.add_argument("--page", default="index.html") + preview.add_argument("--out", default="preview") + preview.set_defaults(run=command_preview) + args = parser.parse_args(argv) + return args.run(args) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/social_requirements.txt b/scripts/social_requirements.txt new file mode 100644 index 0000000..737c8c7 --- /dev/null +++ b/scripts/social_requirements.txt @@ -0,0 +1,4 @@ +# Only the social page needs these; the waitlist tally runs on the standard library. +beautifulsoup4>=4.12,<5 +pillow>=10.3,<13 +requests>=2.31,<3 diff --git a/scripts/social_save.sh b/scripts/social_save.sh new file mode 100644 index 0000000..4764c82 --- /dev/null +++ b/scripts/social_save.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Save the data folder as one commit on the data branch and print the commit id. +# +# scripts/social_save.sh "" [parent commit] +# +# With no parent the commit has no history, so the branch never grows: every run replaces it. With a +# parent (the commit this same run made a minute earlier) that first commit stays reachable, and the +# card images Buffer was told to fetch from it keep working until the next run. +set -euo pipefail + +message="${1:?commit message}" +parent="${2:-}" +folder="${DATA_DIR:-data}" +branch="${DATA_BRANCH:-social-data}" +remote="${DATA_REMOTE:-origin}" + +git -C "$folder" add -A +tree="$(git -C "$folder" write-tree)" +if [ -n "$parent" ]; then + commit="$(git -C "$folder" commit-tree "$tree" -p "$parent" -m "$message")" +else + commit="$(git -C "$folder" commit-tree "$tree" -m "$message")" +fi + +for attempt in 1 2 3 4 5; do + if git -C "$folder" push --quiet --force "$remote" "$commit:refs/heads/$branch" >&2; then + echo "$commit" + exit 0 + fi + echo "push attempt $attempt failed" >&2 + sleep $((attempt * 5)) +done +echo "could not save the data branch" >&2 +exit 1 diff --git a/tests/social/fixtures/bot_cases.json b/tests/social/fixtures/bot_cases.json new file mode 100644 index 0000000..adce7e7 --- /dev/null +++ b/tests/social/fixtures/bot_cases.json @@ -0,0 +1,679 @@ +{ + "feed": { + "brand": "Morrow Brief", + "site": "https://juw12.github.io/morrow/", + "date": "2026-11-14", + "updated": "14:30", + "stats": { + "stories": 32, + "outlets": 15 + }, + "markets": [ + { + "name": "S&P 500", + "pct": 0.4, + "dir": "down", + "price": "7,561.67" + }, + { + "name": "Nasdaq", + "pct": 0.91, + "dir": "down", + "price": "25,136.27" + }, + { + "name": "Dow Jones", + "pct": 0.1, + "dir": "up", + "price": "52,347.55" + }, + { + "name": "IHSG (Jakarta)", + "pct": 1.78, + "dir": "up", + "price": "5,344.41" + }, + { + "name": "Gold (XAU)", + "pct": 0.03, + "dir": "up", + "price": "3,886.30" + } + ], + "stories": [ + { + "id": "7b936009af", + "title": "SpaceX launches Starship on ambitious 22-hour Flight 15 test, but aborts booster 'chopsticks' catch (video)", + "url": "https://www.space.com/space-exploration/launches-spacecraft/spacex-starship-flight-15-chopsticks-double-catch-super-heavy-ship", + "source": "Space.com", + "section": "Space & Aircraft", + "section_id": "space", + "tag": "SpaceX", + "dek": "STARBASE, Texas — SpaceX launched the 15th test flight of its massive Starship rocket today (Nov. 13) and is preparing the vehicle's upper stage for its first-ever return to the launch site in less than 24 hours. Flight 15 lifted off from…", + "published_iso": "2026-11-13T23:20:03+00:00", + "coverage": 5, + "related": [ + { + "source": "Spaceflight Now", + "title": "Starship in orbit ahead of catch attempt, booster diverts to explosive splashdown", + "url": "https://spaceflightnow.com/2026/11/13/live-coverage-starship-flight-15-attempts-to-recover-full-vehicle-in-double-launch-pad-catch/" + }, + { + "source": "Teslarati", + "title": "SpaceX Starship Flight 15: How to watch the double catch attempt", + "url": "https://www.teslarati.com/spacex-starship-flight-15-how-to-watch-the-double-catch-attempt/" + }, + { + "source": "NASASpaceflight", + "title": "Starship Flight 15: Ship 42 orbital, Booster diverts to ocean", + "url": "https://www.nasaspaceflight.com/2026/11/starship-flight-15-spacex-history/" + }, + { + "source": "Ars Technica", + "title": "Rocket Report: Starship set for historic test flight; a century of Electron", + "url": "https://arstechnica.com/space/2026/11/rocket-report-starship-set-for-historic-test-flight-a-century-of-electron/" + } + ], + "is_lead": true, + "order": 0 + }, + { + "id": "00ccf51f4f", + "title": "Nvidia stock suffers longest weekly losing streak since 2012 as market value sinks by $1.7 trillion", + "url": "https://www.cnbc.com/2026/11/13/nvidia-stock-jensen-affair-ai.html", + "source": "CNBC", + "section": "Big US Tech", + "section_id": "big_tech", + "tag": "Nvidia", + "dek": "Jensen Huang's resignation added to Nvidia's market woes, with the stock already in a downward slide since early October.", + "published_iso": "2026-11-13T22:18:43+00:00", + "coverage": 3, + "related": [], + "is_lead": false, + "order": 1 + }, + { + "id": "951c845619", + "title": "ChatGPT Wallet lets OpenAI’s Dots agents make purchases for you", + "url": "https://www.theverge.com/ai-artificial-intelligence/1033425/openai-chatgpt-wallet", + "source": "The Verge", + "section": "AI & Chips", + "section_id": "ai", + "tag": "OpenAI", + "dek": "After launching its Dots AI agents platform earlier this year, OpenAI is launching a wallet for the agents. ChatGPT Wallet is rolling out…", + "published_iso": "2026-11-13T20:45:10+00:00", + "coverage": 3, + "related": [], + "is_lead": false, + "order": 2 + }, + { + "id": "b777df82a2", + "title": "Analysis: Companies love a ‘rockstar’ CEO. Until they depend on one", + "url": "https://www.cnbc.com/2026/11/14/nvidia-jensen-huang-ceo-succession-risk.html", + "source": "CNBC", + "section": "Big US Tech", + "section_id": "big_tech", + "tag": "Nvidia", + "dek": "Jensen Huang’s exit tests Nvidia’s succession planning and governance, and offers lessons for companies that elevate their CEOs to rockstar…", + "published_iso": "2026-11-14T07:08:45+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 3 + }, + { + "id": "d73553036a", + "title": "Outlook Went Down? Microsoft May Owe You Money", + "url": "https://www.wired.com/story/outlook-went-down-microsoft-may-owe-you-money/", + "source": "Wired", + "section": "Big US Tech", + "section_id": "big_tech", + "tag": "", + "dek": "If your Microsoft business services were offline last week, you’re probably entitled to a partial credit on your next monthly bill.", + "published_iso": "2026-11-14T07:00:00+00:00", + "coverage": 2, + "related": [], + "is_lead": false, + "order": 4 + }, + { + "id": "625517f21d", + "title": "Microsoft’s global outage may be worse than Amazon's or CrowdStrike's, expert says", + "url": "https://arstechnica.com/tech-policy/2026/11/microsoft-blames-failed-server-health-checks-not-hackers-for-global-outage/", + "source": "Ars Technica", + "section": "Big US Tech", + "section_id": "big_tech", + "tag": "", + "dek": "Governments, hospitals, courts scrambled by global Microsoft outage.", + "published_iso": "2026-11-13T15:26:53+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 5 + }, + { + "id": "46f76adf16", + "title": "Starship flight 15 powers to orbit—setting up a possible catch on Saturday", + "url": "https://arstechnica.com/space/2026/11/starship-flight-15-powers-to-orbit-setting-up-a-possible-catch-on-saturday/", + "source": "Ars Technica", + "section": "Big US Tech", + "section_id": "big_tech", + "tag": "", + "dek": "One way or another, the company will get some more data on Saturday.", + "published_iso": "2026-11-14T00:49:13+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 6 + }, + { + "id": "04367a7d0b", + "title": "Amazon has a plan to index the world’s online shopping data", + "url": "https://www.theverge.com/tech/1033697/amazon-world-product-graph-ai-search", + "source": "The Verge", + "section": "Big US Tech", + "section_id": "big_tech", + "tag": "", + "dek": "Amazon has plans to build a vault of data on every product sold anywhere on the internet, even products Amazon doesn't sell, and make the information available through its own site, according to Reuters. The project, called the \"World...", + "published_iso": "2026-11-13T23:49:11+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 7 + }, + { + "id": "ec577d4c46", + "title": "Jensen Huang’s departure blasts big hole in AI startup funding, VCs and founders say", + "url": "https://techcrunch.com/2026/11/13/jensen-huangs-departure-blasts-big-hole-in-ai-startup-funding-vcs-and-founders-say/", + "source": "TechCrunch", + "section": "AI & Chips", + "section_id": "ai", + "tag": "Nvidia", + "dek": "Nvidia poured tens of billions into AI startups under Huang. Without him at the helm, founders and VCs are bracing for smaller checks and…", + "published_iso": "2026-11-13T22:43:00+00:00", + "coverage": 2, + "related": [], + "is_lead": false, + "order": 8 + }, + { + "id": "3dc626ea28", + "title": "Export-banned Nvidia chips are behind China’s most powerful AI models — uh-oh, Jensen!", + "url": "https://techcrunch.com/2026/11/13/export-banned-nvidia-chips-are-behind-chinas-most-powerful-ai-models-uh-oh-jensen/", + "source": "TechCrunch", + "section": "AI & Chips", + "section_id": "ai", + "tag": "", + "dek": "New research shows the most advanced AI models from China were trained on banned Nvidia chips.", + "published_iso": "2026-11-13T18:40:30+00:00", + "coverage": 2, + "related": [], + "is_lead": false, + "order": 9 + }, + { + "id": "71dad9da95", + "title": "Let’s talk about Jensen Huang’s many affairs!", + "url": "https://www.theverge.com/ai-artificial-intelligence/1032339/lets-talk-about-the-nvidia-shitshow", + "source": "The Verge", + "section": "AI & Chips", + "section_id": "ai", + "tag": "", + "dek": "Usually, we'd be offering to answer your questions about a news event, but honestly we don't really have any answers about what the hell is going on over at Nvidia. I can't speak for Nilay but at the rate Jensen Huang's new mistresses are...", + "published_iso": "2026-11-13T19:07:12+00:00", + "coverage": 2, + "related": [], + "is_lead": false, + "order": 10 + }, + { + "id": "0d9fae548b", + "title": "How Microcredentials Help Engineers Build Trustworthy AI", + "url": "https://spectrum.ieee.org/microcredentials-trustworthy-ai", + "source": "IEEE Spectrum", + "section": "AI & Chips", + "section_id": "ai", + "tag": "", + "dek": "Engineering teams are moving at breakneck speed to integrate artificial intelligence, but they are running into a formidable new barrier: compliance issues. Translating high-level AI ethics , governance , and auditing into daily...", + "published_iso": "2026-11-13T19:00:03+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 11 + }, + { + "id": "0967f49901", + "title": "Video Friday: NASA PROMISEs a New Moon Rover", + "url": "https://spectrum.ieee.org/video-friday-moon-rover-promise", + "source": "IEEE Spectrum", + "section": "Robotics", + "section_id": "robotics", + "tag": "Robotics", + "dek": "Video Friday is your weekly selection of awesome robotics videos, collected by your friends at IEEE Spectrum robotics. We also post a…", + "published_iso": "2026-11-13T17:30:03+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 12 + }, + { + "id": "2f9c1e909e", + "title": "Narwal builds robot vacuums to serve regional needs", + "url": "https://www.therobotreport.com/narwal-builds-robot-vacuums-to-serve-regional-needs/", + "source": "The Robot Report", + "section": "Robotics", + "section_id": "robotics", + "tag": "", + "dek": "Narwal executives say they spent a decade in research and development and have tailored their household cleaning robots to local markets.", + "published_iso": "2026-11-13T13:18:05+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 13 + }, + { + "id": "471fb334b4", + "title": "Six AGIBOT G2 Robots Complete 1,730 Tasks in 24-Hour Shanghai Convenience Store Trial", + "url": "https://www.therobotreport.com/six-agibot-g2-robots-complete-1730-tasks-in-24-hour-shanghai-convenience-store-trial/", + "source": "The Robot Report", + "section": "Robotics", + "section_id": "robotics", + "tag": "", + "dek": "Four of the six robots pass a performance assessment as the trial puts customer service, retail operations and embodied AI capabilities to the test in a working store.", + "published_iso": "2026-11-13T16:32:58+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 14 + }, + { + "id": "3eacf60382", + "title": "On this day in space! Nov. 13, 1971: NASA's Mariner 9 becomes 1st spacecraft to orbit Mars, revealing close looks at the Martian surface", + "url": "https://www.space.com/space-exploration/on-this-day-in-space-november-13-2026", + "source": "Space.com", + "section": "Space & Aircraft", + "section_id": "space", + "tag": "", + "dek": "On Nov. 13, 1971, Mariner 9 became the first spacecraft to orbit Mars. At the time, scientists didn't know much about the Red Planet. NASA had several Mars flybys before Mariner 9 . Those flybys showed that Mars had a cratered surface , so...", + "published_iso": "2026-11-13T21:25:55+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 15 + }, + { + "id": "5459684a14", + "title": "ULA delays to Saturday Vulcan rocket launch of 40 Leo satellites for Amazon", + "url": "https://spaceflightnow.com/2026/11/13/ulas-vulcan-rocket-plans-return-to-flight-with-launch-of-40-leo-satellites-for-amazon/", + "source": "Spaceflight Now", + "section": "Space & Aircraft", + "section_id": "space", + "tag": "", + "dek": "Liftoff from Space Launch Complex 41 at Cape Canaveral Space Force Station is now scheduled for 12:32 p.m. EST (1732 UTC) on Saturday.", + "published_iso": "2026-11-13T12:25:06+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 16 + }, + { + "id": "002f283667", + "title": "SpaceX Starship is going for a catch that could change spaceflight forever", + "url": "https://www.teslarati.com/spacex-starship-flight-15-double-catch/", + "source": "Teslarati", + "section": "Space & Aircraft", + "section_id": "space", + "tag": "", + "dek": "SpaceX will try to catch both Starship stages today, a first for any orbital rocket.", + "published_iso": "2026-11-13T15:13:56+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 17 + }, + { + "id": "1cbd2bca9b", + "title": "Outbursts from a Massive Star", + "url": "https://www.nasa.gov/image-article/outbursts-from-a-massive-star/", + "source": "NASA", + "section": "Space & Aircraft", + "section_id": "space", + "tag": "", + "dek": "Using NASA’s Chandra X-ray Observatory, astronomers have discovered what is triggering repeated outbursts from a giant, unstable star located in the galaxy shown in this image released on Nov. 9, 2026. The volatile star, known as AT...", + "published_iso": "2026-11-13T16:39:33+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 18 + }, + { + "id": "17ce9d8841", + "title": "Gemini Astronaut Photos Inspired Early Earth Observation Program", + "url": "https://www.nasa.gov/image-article/gemini-astronaut-photos-inspired-early-earth-observation-program/", + "source": "NASA", + "section": "Space & Aircraft", + "section_id": "space", + "tag": "", + "dek": "Astronauts James A. Lovell Jr. and Edwin E. “Buzz” Aldrin Jr. orbited Earth from November 11 to 15, 1966, on Gemini XII, the final mission of NASA’s Gemini program. In addition to successfully completing rendezvous maneuvers and...", + "published_iso": "2026-11-13T16:05:54+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 19 + }, + { + "id": "f1d12a18d6", + "title": "Klook Promo Code: 20% Off November 2026", + "url": "https://www.wired.com/story/klook-promo-code/", + "source": "Wired", + "section": "Technology", + "section_id": "technology", + "tag": "App", + "dek": "Maximize your travel budget with our expert guide to finding Klook coupons, app-exclusive deals, and payment discounts on tours, hotels…", + "published_iso": "2026-11-14T06:00:00+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 20 + }, + { + "id": "57ae9f9b1f", + "title": "Amazon’s new Kindle Colorsoft is a premium e-reader with a premium price", + "url": "https://techcrunch.com/2026/11/13/amazons-new-kindle-colorsoft-is-a-premium-e-reader-with-a-premium-price/", + "source": "TechCrunch", + "section": "Technology", + "section_id": "technology", + "tag": "", + "dek": "The latest model is lighter and thinner than its predecessor and features an edge-to-edge display. But if you already own the first-generation Colorsoft, there’s little reason to upgrade.", + "published_iso": "2026-11-13T23:00:00+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 21 + }, + { + "id": "1b8f2f598f", + "title": "FAA allows flights to resume after hours-long ground stop across the US", + "url": "https://www.engadget.com/2316792/faa-flights-resume-ground-stop-old-equipment/", + "source": "Engadget", + "section": "Technology", + "section_id": "technology", + "tag": "", + "dek": "The FAA grounded flights for hours on November 13, and the transportation secretary blamed it on old and malfunctioning air traffic control system.", + "published_iso": "2026-11-14T06:35:40+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 22 + }, + { + "id": "8e5422c7b6", + "title": "Google now lets people connect Gemini to their bank accounts", + "url": "https://www.engadget.com/2316384/google-now-lets-people-connect-gemini-to-their-bank-accounts/", + "source": "Engadget", + "section": "Technology", + "section_id": "technology", + "tag": "", + "dek": "This is for spending analysis and not for making purchases.", + "published_iso": "2026-11-13T19:15:32+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 23 + }, + { + "id": "c407c87afb", + "title": "This young startup wants to power floating data centers with the ocean’s heat, and investors are already on board", + "url": "https://techcrunch.com/2026/11/13/this-young-startup-wants-to-power-floating-data-centers-with-the-oceans-heat-and-investors-are-already-on-board/", + "source": "TechCrunch", + "section": "Technology", + "section_id": "technology", + "tag": "", + "dek": "Drogue is harvesting heat from the ocean to power data centers on floating vessels. The first prototype should be bobbing around the ocean, crunching bits, in less than two years.", + "published_iso": "2026-11-13T20:07:24+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 24 + }, + { + "id": "d539edcfb8", + "title": "Bitdefender Antivirus Plus Review: Undercutting the Competition", + "url": "https://www.wired.com/review/bitdefender-antivirus-plus/", + "source": "Wired", + "section": "Technology", + "section_id": "technology", + "tag": "", + "dek": "It’s not promoted very much, but Bitdefender’s stand-alone Antivirus Plus software is both effective and affordable.", + "published_iso": "2026-11-13T10:00:00+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 25 + }, + { + "id": "af22ab90f6", + "title": "AMD and other chip stocks slide as Wall Street rethinks the AI trade without Jensen Huang", + "url": "https://www.marketwatch.com/story/amd-and-other-chip-stocks-slide-as-wall-street-rethinks-the-ai-trade-without-jensen-huang-2fb74547?mod=mw_rss_topstories", + "source": "MarketWatch", + "section": "Markets", + "section_id": "markets_news", + "tag": "Nvidia", + "dek": "At first, it looked like Nvidia’s pain might be the rest of the chip sector’s gain. But now, investors aren’t so sure.", + "published_iso": "2026-11-13T21:10:00+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 26 + }, + { + "id": "f8e3a26700", + "title": "AI risks are the biggest threat to the financial system, Fed survey finds", + "url": "https://www.cnbc.com/2026/11/13/ai-risks-financial-system-federal-reserve.html", + "source": "CNBC", + "section": "Markets", + "section_id": "markets_news", + "tag": "", + "dek": "The report comes with Wall Street on edge over high-flying AI hyperscalers, while communities are on alert for the rapid spread of data centers.", + "published_iso": "2026-11-13T21:18:30+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 27 + }, + { + "id": "0cc207e01d", + "title": "FAA lifts rare nationwide ground stop on flights; air travel may take days to recover", + "url": "https://www.marketwatch.com/story/faa-declares-rare-widespread-ground-outage-bdc46393?mod=mw_rss_topstories", + "source": "MarketWatch", + "section": "Markets", + "section_id": "markets_news", + "tag": "", + "dek": "Homeland Security Secretary Mullin said that there was ‘zero evidence’ of cyberattack as fewer flights amid fuel-cost surge may hamper rebookings.", + "published_iso": "2026-11-14T02:42:00+00:00", + "coverage": 2, + "related": [], + "is_lead": false, + "order": 28 + }, + { + "id": "d8c2d586f2", + "title": "Chicago Fed's Goolsbee says central bank 'has definitely got work to do' on inflation", + "url": "https://www.cnbc.com/2026/11/13/chicago-feds-goolsbee-says-central-bank-has-definitely-got-work-to-do-on-inflation.html", + "source": "CNBC", + "section": "Markets", + "section_id": "markets_news", + "tag": "", + "dek": "The policymaker expressed concern about the latest inflation data, telling CNBC that the central bank likely will need to take action.", + "published_iso": "2026-11-13T14:22:07+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 29 + }, + { + "id": "5b165154ff", + "title": "Raja Salman Siapkan Umrah Gratis Buat 1.000 Orang, Warga RI Bisa Dapat", + "url": "https://www.cnbcindonesia.com/lifestyle/20261114114346-33-785212/raja-salman-siapkan-umrah-gratis-buat-1000-orang-warga-ri-bisa-dapat", + "source": "CNBC Indonesia", + "section": "Markets", + "section_id": "markets_news", + "tag": "", + "dek": "Raja Arab Saudi Salman bin Abdulaziz Al Saud memberikan program pembiayaan umrah bagi 1.000 jemaah laki-laki dan perempuan dari berbagai negara di dunia", + "published_iso": "2026-11-14T07:15:02+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 30 + }, + { + "id": "9f871e6f50", + "title": "Video: Efek MSCI - Gibran Mundur ke Prospek Investasi dan IPO Saham RI", + "url": "https://www.cnbcindonesia.com/market/20261112145645-19-784694/video-efek-msci--gibran-mundur-ke-prospek-investasi-ipo-saham-ri", + "source": "CNBC Indonesia", + "section": "Markets", + "section_id": "markets_news", + "tag": "", + "dek": "Bos Sekuritas Ungkap Efek MSCI - Gibran Mundur ke Prospek Investasi - IPO Saham RI", + "published_iso": "2026-11-14T07:00:15+00:00", + "coverage": 1, + "related": [], + "is_lead": false, + "order": 31 + } + ], + "posts": [], + "bot": { + "/start": "Morrow Brief\nTech and markets, read across the outlets. One brief a day, every story sourced.\n\n/today the five stories that matter\n/ai AI & Chips\n/bigtech Big US Tech\n/robotics Robotics\n/space Space & Aircraft\n/tech Technology\n/markets Markets\n/tape index moves\n/sources outlets in today's edition\n/find word search today's stories\n/about how this works", + "/help": "Morrow Brief\nTech and markets, read across the outlets. One brief a day, every story sourced.\n\n/today the five stories that matter\n/ai AI & Chips\n/bigtech Big US Tech\n/robotics Robotics\n/space Space & Aircraft\n/tech Technology\n/markets Markets\n/tape index moves\n/sources outlets in today's edition\n/find word search today's stories\n/about how this works", + "/today": "Morrow Brief · Sat 14 Nov\n\n• SpaceX launches Starship on ambitious 22-hour Flight 15 test, but aborts booster 'chopsticks' catch (video) (Space.com +4)\n• Nvidia stock suffers longest weekly losing streak since 2012 as market value sinks by $1.7 trillion (CNBC +2)\n• ChatGPT Wallet lets OpenAI’s Dots agents make purchases for you (The Verge +2)\n• Outlook Went Down? Microsoft May Owe You Money (Wired +1)\n• Jensen Huang’s departure blasts big hole in AI startup funding, VCs and founders say (TechCrunch +1)\n\nMarkets: S&P 500 ▼0.40% · Nasdaq ▼0.91% · Dow Jones ▲0.10% · IHSG (Jakarta) ▲1.78%", + "/about": "Morrow Brief is built from Morrow, a daily paper that reads hundreds of headlines and merges coverage of the same story across outlets. Selection is automated: the most widely reported stories come first. Nothing is rewritten; headlines belong to their outlets and every story links to them. Not financial advice.\n\nFull edition", + "/ai": "AI & Chips · Sat 14 Nov\n\n• ChatGPT Wallet lets OpenAI’s Dots agents make purchases for you (The Verge +2)\n• Jensen Huang’s departure blasts big hole in AI startup funding, VCs and founders say (TechCrunch +1)\n• Export-banned Nvidia chips are behind China’s most powerful AI models — uh-oh, Jensen! (TechCrunch +1)\n• Let’s talk about Jensen Huang’s many affairs! (The Verge +1)\n• How Microcredentials Help Engineers Build Trustworthy AI (IEEE Spectrum)", + "/bigtech": "Big US Tech · Sat 14 Nov\n\n• Nvidia stock suffers longest weekly losing streak since 2012 as market value sinks by $1.7 trillion (CNBC +2)\n• Outlook Went Down? Microsoft May Owe You Money (Wired +1)\n• Analysis: Companies love a ‘rockstar’ CEO. Until they depend on one (CNBC)\n• Microsoft’s global outage may be worse than Amazon's or CrowdStrike's, expert says (Ars Technica)\n• Starship flight 15 powers to orbit—setting up a possible catch on Saturday (Ars Technica)\n• Amazon has a plan to index the world’s online shopping data (The Verge)", + "/robotics": "Robotics · Sat 14 Nov\n\n• Video Friday: NASA PROMISEs a New Moon Rover (IEEE Spectrum)\n• Narwal builds robot vacuums to serve regional needs (The Robot Report)\n• Six AGIBOT G2 Robots Complete 1,730 Tasks in 24-Hour Shanghai Convenience Store Trial (The Robot Report)", + "/space": "Space & Aircraft · Sat 14 Nov\n\n• SpaceX launches Starship on ambitious 22-hour Flight 15 test, but aborts booster 'chopsticks' catch (video) (Space.com +4)\n• On this day in space! Nov. 13, 1971: NASA's Mariner 9 becomes 1st spacecraft to orbit Mars, revealing close looks at… (Space.com)\n• ULA delays to Saturday Vulcan rocket launch of 40 Leo satellites for Amazon (Spaceflight Now)\n• SpaceX Starship is going for a catch that could change spaceflight forever (Teslarati)\n• Outbursts from a Massive Star (NASA)\n• Gemini Astronaut Photos Inspired Early Earth Observation Program (NASA)", + "/tech": "Technology · Sat 14 Nov\n\n• Klook Promo Code: 20% Off November 2026 (Wired)\n• Amazon’s new Kindle Colorsoft is a premium e-reader with a premium price (TechCrunch)\n• FAA allows flights to resume after hours-long ground stop across the US (Engadget)\n• Google now lets people connect Gemini to their bank accounts (Engadget)\n• This young startup wants to power floating data centers with the ocean’s heat, and investors are already on board (TechCrunch)\n• Bitdefender Antivirus Plus Review: Undercutting the Competition (Wired)", + "/markets": "Markets · Sat 14 Nov\n\n• FAA lifts rare nationwide ground stop on flights; air travel may take days to recover (MarketWatch +1)\n• AMD and other chip stocks slide as Wall Street rethinks the AI trade without Jensen Huang (MarketWatch)\n• AI risks are the biggest threat to the financial system, Fed survey finds (CNBC)\n• Chicago Fed's Goolsbee says central bank 'has definitely got work to do' on inflation (CNBC)\n• Raja Salman Siapkan Umrah Gratis Buat 1.000 Orang, Warga RI Bisa Dapat (CNBC Indonesia)\n• Video: Efek MSCI - Gibran Mundur ke Prospek Investasi dan IPO Saham RI (CNBC Indonesia)", + "/tape": "Tape · Sat 14 Nov\n\nS&P 500 7,561.67 ▼0.40%\nNasdaq 25,136.27 ▼0.91%\nDow Jones 52,347.55 ▲0.10%\nIHSG (Jakarta) 5,344.41 ▲1.78%\nGold (XAU) 3,886.30 ▲0.03%\n\nNot financial advice.", + "/sources": "32 stories from 15 outlets · Sat 14 Nov\n\nArs Technica, CNBC, CNBC Indonesia, Engadget, IEEE Spectrum, MarketWatch, NASA, NASASpaceflight, Space.com, Spaceflight Now, TechCrunch, Teslarati, The Robot Report, The Verge, Wired" + } + }, + "cases": [ + { + "text": "/start", + "private": true, + "reply": "Morrow Brief\nTech and markets, read across the outlets. One brief a day, every story sourced.\n\n/today the five stories that matter\n/ai AI & Chips\n/bigtech Big US Tech\n/robotics Robotics\n/space Space & Aircraft\n/tech Technology\n/markets Markets\n/tape index moves\n/sources outlets in today's edition\n/find word search today's stories\n/about how this works" + }, + { + "text": "/help", + "private": true, + "reply": "Morrow Brief\nTech and markets, read across the outlets. One brief a day, every story sourced.\n\n/today the five stories that matter\n/ai AI & Chips\n/bigtech Big US Tech\n/robotics Robotics\n/space Space & Aircraft\n/tech Technology\n/markets Markets\n/tape index moves\n/sources outlets in today's edition\n/find word search today's stories\n/about how this works" + }, + { + "text": "/today", + "private": true, + "reply": "Morrow Brief · Sat 14 Nov\n\n• SpaceX launches Starship on ambitious 22-hour Flight 15 test, but aborts booster 'chopsticks' catch (video) (Space.com +4)\n• Nvidia stock suffers longest weekly losing streak since 2012 as market value sinks by $1.7 trillion (CNBC +2)\n• ChatGPT Wallet lets OpenAI’s Dots agents make purchases for you (The Verge +2)\n• Outlook Went Down? Microsoft May Owe You Money (Wired +1)\n• Jensen Huang’s departure blasts big hole in AI startup funding, VCs and founders say (TechCrunch +1)\n\nMarkets: S&P 500 ▼0.40% · Nasdaq ▼0.91% · Dow Jones ▲0.10% · IHSG (Jakarta) ▲1.78%" + }, + { + "text": "/TODAY", + "private": true, + "reply": "Morrow Brief · Sat 14 Nov\n\n• SpaceX launches Starship on ambitious 22-hour Flight 15 test, but aborts booster 'chopsticks' catch (video) (Space.com +4)\n• Nvidia stock suffers longest weekly losing streak since 2012 as market value sinks by $1.7 trillion (CNBC +2)\n• ChatGPT Wallet lets OpenAI’s Dots agents make purchases for you (The Verge +2)\n• Outlook Went Down? Microsoft May Owe You Money (Wired +1)\n• Jensen Huang’s departure blasts big hole in AI startup funding, VCs and founders say (TechCrunch +1)\n\nMarkets: S&P 500 ▼0.40% · Nasdaq ▼0.91% · Dow Jones ▲0.10% · IHSG (Jakarta) ▲1.78%" + }, + { + "text": "/today@MorrowBriefBot", + "private": false, + "reply": "Morrow Brief · Sat 14 Nov\n\n• SpaceX launches Starship on ambitious 22-hour Flight 15 test, but aborts booster 'chopsticks' catch (video) (Space.com +4)\n• Nvidia stock suffers longest weekly losing streak since 2012 as market value sinks by $1.7 trillion (CNBC +2)\n• ChatGPT Wallet lets OpenAI’s Dots agents make purchases for you (The Verge +2)\n• Outlook Went Down? Microsoft May Owe You Money (Wired +1)\n• Jensen Huang’s departure blasts big hole in AI startup funding, VCs and founders say (TechCrunch +1)\n\nMarkets: S&P 500 ▼0.40% · Nasdaq ▼0.91% · Dow Jones ▲0.10% · IHSG (Jakarta) ▲1.78%" + }, + { + "text": "/ai", + "private": true, + "reply": "AI & Chips · Sat 14 Nov\n\n• ChatGPT Wallet lets OpenAI’s Dots agents make purchases for you (The Verge +2)\n• Jensen Huang’s departure blasts big hole in AI startup funding, VCs and founders say (TechCrunch +1)\n• Export-banned Nvidia chips are behind China’s most powerful AI models — uh-oh, Jensen! (TechCrunch +1)\n• Let’s talk about Jensen Huang’s many affairs! (The Verge +1)\n• How Microcredentials Help Engineers Build Trustworthy AI (IEEE Spectrum)" + }, + { + "text": "/bigtech", + "private": true, + "reply": "Big US Tech · Sat 14 Nov\n\n• Nvidia stock suffers longest weekly losing streak since 2012 as market value sinks by $1.7 trillion (CNBC +2)\n• Outlook Went Down? Microsoft May Owe You Money (Wired +1)\n• Analysis: Companies love a ‘rockstar’ CEO. Until they depend on one (CNBC)\n• Microsoft’s global outage may be worse than Amazon's or CrowdStrike's, expert says (Ars Technica)\n• Starship flight 15 powers to orbit—setting up a possible catch on Saturday (Ars Technica)\n• Amazon has a plan to index the world’s online shopping data (The Verge)" + }, + { + "text": "/robotics", + "private": true, + "reply": "Robotics · Sat 14 Nov\n\n• Video Friday: NASA PROMISEs a New Moon Rover (IEEE Spectrum)\n• Narwal builds robot vacuums to serve regional needs (The Robot Report)\n• Six AGIBOT G2 Robots Complete 1,730 Tasks in 24-Hour Shanghai Convenience Store Trial (The Robot Report)" + }, + { + "text": "/space", + "private": true, + "reply": "Space & Aircraft · Sat 14 Nov\n\n• SpaceX launches Starship on ambitious 22-hour Flight 15 test, but aborts booster 'chopsticks' catch (video) (Space.com +4)\n• On this day in space! Nov. 13, 1971: NASA's Mariner 9 becomes 1st spacecraft to orbit Mars, revealing close looks at… (Space.com)\n• ULA delays to Saturday Vulcan rocket launch of 40 Leo satellites for Amazon (Spaceflight Now)\n• SpaceX Starship is going for a catch that could change spaceflight forever (Teslarati)\n• Outbursts from a Massive Star (NASA)\n• Gemini Astronaut Photos Inspired Early Earth Observation Program (NASA)" + }, + { + "text": "/tech", + "private": true, + "reply": "Technology · Sat 14 Nov\n\n• Klook Promo Code: 20% Off November 2026 (Wired)\n• Amazon’s new Kindle Colorsoft is a premium e-reader with a premium price (TechCrunch)\n• FAA allows flights to resume after hours-long ground stop across the US (Engadget)\n• Google now lets people connect Gemini to their bank accounts (Engadget)\n• This young startup wants to power floating data centers with the ocean’s heat, and investors are already on board (TechCrunch)\n• Bitdefender Antivirus Plus Review: Undercutting the Competition (Wired)" + }, + { + "text": "/markets", + "private": true, + "reply": "Markets · Sat 14 Nov\n\n• FAA lifts rare nationwide ground stop on flights; air travel may take days to recover (MarketWatch +1)\n• AMD and other chip stocks slide as Wall Street rethinks the AI trade without Jensen Huang (MarketWatch)\n• AI risks are the biggest threat to the financial system, Fed survey finds (CNBC)\n• Chicago Fed's Goolsbee says central bank 'has definitely got work to do' on inflation (CNBC)\n• Raja Salman Siapkan Umrah Gratis Buat 1.000 Orang, Warga RI Bisa Dapat (CNBC Indonesia)\n• Video: Efek MSCI - Gibran Mundur ke Prospek Investasi dan IPO Saham RI (CNBC Indonesia)" + }, + { + "text": "/tape", + "private": true, + "reply": "Tape · Sat 14 Nov\n\nS&P 500 7,561.67 ▼0.40%\nNasdaq 25,136.27 ▼0.91%\nDow Jones 52,347.55 ▲0.10%\nIHSG (Jakarta) 5,344.41 ▲1.78%\nGold (XAU) 3,886.30 ▲0.03%\n\nNot financial advice." + }, + { + "text": "/sources", + "private": true, + "reply": "32 stories from 15 outlets · Sat 14 Nov\n\nArs Technica, CNBC, CNBC Indonesia, Engadget, IEEE Spectrum, MarketWatch, NASA, NASASpaceflight, Space.com, Spaceflight Now, TechCrunch, Teslarati, The Robot Report, The Verge, Wired" + }, + { + "text": "/about", + "private": true, + "reply": "Morrow Brief is built from Morrow, a daily paper that reads hundreds of headlines and merges coverage of the same story across outlets. Selection is automated: the most widely reported stories come first. Nothing is rewritten; headlines belong to their outlets and every story links to them. Not financial advice.\n\nFull edition" + }, + { + "text": "/find nvidia", + "private": true, + "reply": "6 stories on “nvidia”\n\n• Nvidia stock suffers longest weekly losing streak since 2012 as market value sinks by $1.7 trillion (CNBC +2)\n• Jensen Huang’s departure blasts big hole in AI startup funding, VCs and founders say (TechCrunch +1)\n• Export-banned Nvidia chips are behind China’s most powerful AI models — uh-oh, Jensen! (TechCrunch +1)\n• Let’s talk about Jensen Huang’s many affairs! (The Verge +1)\n• Analysis: Companies love a ‘rockstar’ CEO. Until they depend on one (CNBC)\n• AMD and other chip stocks slide as Wall Street rethinks the AI trade without Jensen Huang (MarketWatch)" + }, + { + "text": "/find@MorrowBriefBot SpaceX starship ", + "private": false, + "reply": "2 stories on “SpaceX starship”\n\n• SpaceX launches Starship on ambitious 22-hour Flight 15 test, but aborts booster 'chopsticks' catch (video) (Space.com +4)\n• SpaceX Starship is going for a catch that could change spaceflight forever (Teslarati)" + }, + { + "text": "/find", + "private": true, + "reply": "Send /find and a word, for example: /find nvidia" + }, + { + "text": "/find a", + "private": true, + "reply": "Send /find and a word, for example: /find nvidia" + }, + { + "text": "/find zzzzqqq ", + "private": true, + "reply": "Nothing on “zzzzqqq <b>” in today's edition. Try /today." + }, + { + "text": "/unknown", + "private": true, + "reply": "Morrow Brief\nTech and markets, read across the outlets. One brief a day, every story sourced.\n\n/today the five stories that matter\n/ai AI & Chips\n/bigtech Big US Tech\n/robotics Robotics\n/space Space & Aircraft\n/tech Technology\n/markets Markets\n/tape index moves\n/sources outlets in today's edition\n/find word search today's stories\n/about how this works" + }, + { + "text": "hello there", + "private": true, + "reply": "Morrow Brief\nTech and markets, read across the outlets. One brief a day, every story sourced.\n\n/today the five stories that matter\n/ai AI & Chips\n/bigtech Big US Tech\n/robotics Robotics\n/space Space & Aircraft\n/tech Technology\n/markets Markets\n/tape index moves\n/sources outlets in today's edition\n/find word search today's stories\n/about how this works" + }, + { + "text": "hello there", + "private": false, + "reply": null + }, + { + "text": "", + "private": true, + "reply": null + } + ] +} diff --git a/tests/social/fixtures/edition.html b/tests/social/fixtures/edition.html new file mode 100644 index 0000000..7bf79b5 --- /dev/null +++ b/tests/social/fixtures/edition.html @@ -0,0 +1,540 @@ + + + +Morrow · Saturday, November 14, 2026 + + + +
+
+

Morrow

+ + My portfolio + + +
+ +
+
+
+

Markets, AI, tech & space - curated every morning

+

32 stories 14 outlets 308 headlines scanned 10 multi-source

+
+ +
+
+
+
+
+
+

Space & Aircraft · SpaceX

+

SpaceX launches Starship on ambitious 22-hour Flight 15 test, but aborts booster 'chopsticks' catch (video)

+

STARBASE, Texas — SpaceX launched the 15th test flight of its massive Starship rocket today (Nov. 13) and is preparing the vehicle's upper stage for its first-ever return to the launch site in less than 24 hours. Flight 15 lifted off from…

+
Space.com5 sources
+ +
+
+
+ + +
+
+ +
+
+
+

Markets

Last close · 5-day trend

+
+
+
S&P 500▼ -0.40%
+
7,561.67
+
+
+
Nasdaq▼ -0.91%
+
25,136.27
+
+
+
Dow Jones▲ +0.10%
+
52,347.55
+
+
+
IHSG (Jakarta)▲ +1.78%
+
5,344.41
+
+
+
Gold (XAU)▲ +0.03%
+
3,886.30
+
+
+

Before IDX opens

+
    +
  • +USD/IDR +18,190.00 ▲ +3.16% +Weaker rupiah: foreign outflow risk for banks, help for exporters +
  • +
  • +Coal (Rotterdam) +104.75 ■ 0.00% +Coal miners (ADRO, PTBA, ITMG); Newcastle is not on the free feed +
  • +
  • +Brent oil +95.84 ▲ +0.02% +Energy names and the fuel-subsidy bill +
  • +
  • +Nickel proxy: Vale Indonesia +3,690 ▼ -1.07% +No free LME nickel quote; INCO tracks the nickel price +
  • +
  • +BI-Rate +5.75% +Held at the October Board meeting; deposit facility raised to 5.00%, decided 2026-10-21. Bank Indonesia +
+

Last close from Yahoo Finance; BI-Rate from the latest Bank Indonesia Board decision.

+
+

S&P 500 fell 0.40% to 7,561.67; Nasdaq -0.91%; Dow Jones +0.10%.

IHSG rose 1.78% to 5,344.41.

+

US watchlist

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
METAMeta641.69+2.90%
KOCoca-Cola99.25+1.86%
IBMIBM247.13+1.01%
BRK-BBerkshire Hathaway592.94+0.53%
AAPLApple356.27+0.39%
MSFTMicrosoft523.88+0.17%
AMZNAmazon266.66-0.29%
MARMarriott423.16-0.35%
GOOGLGoogle (Alphabet)295.77-0.98%
TSMTSMC398.73-1.67%
TSLATesla390.22-1.82%
NVDANvidia162.85-2.01%
INTCIntel59.66-2.52%
MUMicron778.31-2.52%
STXSeagate528.29-3.09%
AMDAMD534.06-8.75%

Indonesia watchlist

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
TLKMTelkom Indonesia2,160+1.41%
BBRIBank Rakyat Indonesia3,060+0.99%
BBCABank Central Asia5,500+0.46%
ANTMAneka Tambang2,810+0.36%
PGEOPertamina Geothermal9250.00%
BMRIBank Mandiri3,8900.00%
+

SpaceX Private company - no ticker; covered in the Space & Aircraft section

+
+
+

Big US Tech

Apple, Microsoft, Google, Amazon, Meta, Tesla and the giants

+
+ +
+ + + + +
+
+

Amazon has a plan to index the world’s online shopping data

+

Amazon has plans to build a vault of data on every product sold anywhere on the internet, even products Amazon doesn't sell, and make the information available through its own site, according to Reuters. The project, called the "World...

+
The Verge
+
+
+
+
+
+

AI & Chips

AI models, semiconductors and compute

+
+ +
+
+
+

ChatGPT Wallet lets OpenAI’s Dots agents make purchases for you

+

After launching its Dots AI agents platform earlier this year, OpenAI is launching a wallet for the agents. ChatGPT Wallet is rolling out starting today for ChatGPT Pro subscribers in the US aged 18 and older using a US billing address and...

+
The Verge3 sources
+
+
+ +
+
+

Let’s talk about Jensen Huang’s many affairs!

+

Usually, we'd be offering to answer your questions about a news event, but honestly we don't really have any answers about what the hell is going on over at Nvidia. I can't speak for Nilay but at the rate Jensen Huang's new mistresses are...

+
The Verge2 sources
+
+
+
+
+

How Microcredentials Help Engineers Build Trustworthy AI

+

Engineering teams are moving at breakneck speed to integrate artificial intelligence, but they are running into a formidable new barrier: compliance issues. Translating high-level AI ethics , governance , and auditing into daily...

+
IEEE Spectrum
+
+
+
+
+
+
+

Robotics

Humanoids, automation and industrial robots

+
+
+
+

Robotics · Robotics

+

Video Friday: NASA PROMISEs a New Moon Rover

+

Video Friday is your weekly selection of awesome robotics videos, collected by your friends at IEEE Spectrum robotics. We also post a…

+
IEEE Spectrum
+
+
+ +
+
+
+
+

Space & Aircraft

Launches, spaceflight and aviation

+
+ +
+ + + +
+
+

Outbursts from a Massive Star

+

Using NASA’s Chandra X-ray Observatory, astronomers have discovered what is triggering repeated outbursts from a giant, unstable star located in the galaxy shown in this image released on Nov. 9, 2026. The volatile star, known as AT...

+
NASA
+
+
+
+
+

Gemini Astronaut Photos Inspired Early Earth Observation Program

+

Astronauts James A. Lovell Jr. and Edwin E. “Buzz” Aldrin Jr. orbited Earth from November 11 to 15, 1966, on Gemini XII, the final mission of NASA’s Gemini program. In addition to successfully completing rendezvous maneuvers and...

+
NASA
+
+
+
+
+
+

Technology

General technology, gadgets and security

+
+
+
+

Technology · App

+

Klook Promo Code: 20% Off November 2026

+

Maximize your travel budget with our expert guide to finding Klook coupons, app-exclusive deals, and payment discounts on tours, hotels…

+
Wired
+
+
+ + + + +
+
+
+
+

Markets

US and Indonesia market headlines

+
+ +
+ + + + +
+
+
+
+ +
+
+

Morrow · Saturday, November 14, 2026 · generated 2026-11-14 14:30

+

Sources: Ars Technica, CNBC, CNBC Indonesia, Engadget, IEEE Spectrum, MarketWatch, NASA, Space.com, Spaceflight Now, TechCrunch, Teslarati, The Robot Report, The Verge, Wired. Headlines and summaries belong to their publishers; every story links to the original.

+
+
+ + diff --git a/tests/social/helpers.py b/tests/social/helpers.py new file mode 100644 index 0000000..cbb96a6 --- /dev/null +++ b/tests/social/helpers.py @@ -0,0 +1,152 @@ +"""Shared pieces for the social tests: the real fixture, a synthetic page builder and a fake HTTP client.""" +from __future__ import annotations + +import html +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +import social_feed as feed # noqa: E402 + +FIXTURES = HERE / "fixtures" +SECTIONS = { + "ai": "AI & Chips", + "big_tech": "Big US Tech", + "robotics": "Robotics", + "space": "Space & Aircraft", + "technology": "Technology", + "markets_news": "Markets", +} + + +def real_edition() -> dict: + return feed.parse_edition((FIXTURES / "edition.html").read_text(encoding="utf-8")) + + +def story(number: int, section: str = "ai", coverage: int = 1, **fields) -> dict: + base = { + "title": f"Story number {number} about something that happened", + "url": f"https://example.com/story-{number}", + "source": f"Outlet {number}", + "section": section, + "tag": "Topic", + "dek": f"A one line summary of story {number}.", + "coverage": coverage, + "related": [], + "kind": "card", + } + base.update(fields) + return base + + +def page(stories: list[dict], date: str = "November 14, 2026", updated: str = "09:15", markets: bool = True) -> str: + """A small page in the published markup: a front section with the lead, then one block per section.""" + def article(item: dict) -> str: + kicker = "" + if item["kind"] != "row": + kicker = f'

{html.escape(SECTIONS[item["section"]])} · {html.escape(item["tag"])}

' + coverage = "" + if item["coverage"] > 1: + coverage = f'{item["coverage"]}' + related = "" + if item["related"]: + links = "".join( + f'
  • {html.escape(link["source"])} ' + f'{html.escape(link["title"])}
  • ' + for link in item["related"] + ) + related = f'' + return ( + f'' + ) + + front = "".join(article(item) for item in stories if item["kind"] == "lead") + blocks = "" + for section_id, name in SECTIONS.items(): + members = [item for item in stories if item["section"] == section_id and item["kind"] != "lead"] + if members: + blocks += ( + f'

    {html.escape(name)}

    ' + + "".join(article(item) for item in members) + "
    " + ) + tape = "" + if markets: + tape = ( + '
    ' + '
    S&P 5006,812.40' + '-0.40%
    ' + '
    IHSG (Jakarta)8,120.10' + '+1.78%
    ' + ) + return ( + f"Morrow · {date}" + f'
    ' + f'{tape}
    {front}
    {blocks}' + ) + + +def edition_from(stories: list[dict], **options) -> dict: + return feed.parse_edition(page(stories, **options)) + + +class FakeResponse: + def __init__(self, status: int = 200, body=None, headers: dict | None = None): + self.status_code = status + self._body = body + self.headers = headers or {} + + def json(self): + if self._body is None: + raise ValueError("no JSON") + return self._body + + +class FakeHttp: + """Stands in for `requests`. Answers come from queues; the last answer of a queue repeats.""" + + def __init__(self, channels: list[dict] | None = None): + self.calls: list[tuple[str, dict]] = [] + self.channels = channels if channels is not None else [] + self.telegram: list = [FakeResponse(200, {"ok": True, "result": {"message_id": 7}})] + self.create: list = [FakeResponse(200, {"data": {"createPost": { + "__typename": "PostActionSuccess", "post": {"id": "post-1", "status": "sent"}}}})] + self.organizations: list = [FakeResponse(200, {"data": {"account": {"organizations": [{"id": "org-1"}]}}})] + self.images: list = [FakeResponse(200, None, {"Content-Type": "image/png"})] + + @staticmethod + def _next(queue: list): + item = queue.pop(0) if len(queue) > 1 else queue[0] + if isinstance(item, Exception): + raise item + return item + + def post(self, url: str, **kwargs): + self.calls.append((url, kwargs)) + if "api.telegram.org" in url: + return self._next(self.telegram) + query = kwargs["json"]["query"] + if "createPost" in query: + return self._next(self.create) + if "organizations" in query: + return self._next(self.organizations) + return FakeResponse(200, {"data": {"channels": self.channels}}) + + def get(self, url: str, **kwargs): + self.calls.append((url, kwargs)) + return self._next(self.images) + + def count(self, fragment: str) -> int: + """How many calls went to a URL, or carried a GraphQL query, containing the fragment.""" + total = 0 + for url, kwargs in self.calls: + query = (kwargs.get("json") or {}).get("query", "") if isinstance(kwargs.get("json"), dict) else "" + if fragment in url or fragment in query: + total += 1 + return total diff --git a/tests/social/make_fixtures.py b/tests/social/make_fixtures.py new file mode 100644 index 0000000..1207ac2 --- /dev/null +++ b/tests/social/make_fixtures.py @@ -0,0 +1,81 @@ +"""Rebuild the test fixtures from a published page. + + python tests/social/make_fixtures.py index.html + +Writes fixtures/edition.html (the page with scripts, styles and pictures removed) and +fixtures/bot_cases.json (messages and the replies both bot implementations must give). +Run it only when the page's markup changes; the tests then pin the new behaviour. +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path + +from bs4 import BeautifulSoup, Comment + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent.parent / "scripts")) + +import social_bot as bot # noqa: E402 +import social_feed as feed # noqa: E402 + +MESSAGES = [ + ("/start", True), + ("/help", True), + ("/today", True), + ("/TODAY", True), + ("/today@MorrowBriefBot", False), + ("/ai", True), + ("/bigtech", True), + ("/robotics", True), + ("/space", True), + ("/tech", True), + ("/markets", True), + ("/tape", True), + ("/sources", True), + ("/about", True), + ("/find nvidia", True), + ("/find@MorrowBriefBot SpaceX starship ", False), + ("/find", True), + ("/find a", True), + ("/find zzzzqqq ", True), + ("/unknown", True), + ("hello there", True), + ("hello there", False), + ("", True), +] + + +def trim(page_html: str) -> str: + soup = BeautifulSoup(page_html, "html.parser") + for tag in soup.select("script, style, svg, img, noscript, link, meta, #analysis, template, canvas, form, dialog"): + tag.decompose() + for comment in soup.find_all(string=lambda node: isinstance(node, Comment)): + comment.extract() + lines = [line.rstrip() for line in str(soup).splitlines() if line.strip()] + return "\n".join(lines) + "\n" + + +def main() -> int: + page = Path(sys.argv[1] if len(sys.argv) > 1 else "index.html").read_text(encoding="utf-8") + fixtures = HERE / "fixtures" + fixtures.mkdir(exist_ok=True) + trimmed = trim(page) + (fixtures / "edition.html").write_text(trimmed, encoding="utf-8", newline="\n") + edition = feed.parse_edition(trimmed) + feed_data = feed.public_feed(edition, []) + cases = [ + {"text": text, "private": private, "reply": bot.reply_for(feed_data, text, private)} + for text, private in MESSAGES + ] + (fixtures / "bot_cases.json").write_text( + json.dumps({"feed": feed_data, "cases": cases}, indent=1, ensure_ascii=False) + "\n", + encoding="utf-8", newline="\n", + ) + print(f"edition.html {len(trimmed) // 1024} KB, {len(edition['stories'])} stories, {len(cases)} bot cases") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/social/test_bot.py b/tests/social/test_bot.py new file mode 100644 index 0000000..c70a6f8 --- /dev/null +++ b/tests/social/test_bot.py @@ -0,0 +1,254 @@ +"""The Telegram bot: its replies, and the poller's handling of offsets, webhooks and failures.""" +import contextlib +import io +import json +import os +import unittest +from unittest import mock + +import requests + +import helpers +from helpers import FIXTURES, FakeHttp, FakeResponse, real_edition + +import social_bot as bot +import social_feed as feed + +TOKEN = "123456789:AAExampleBotTokenThatMustNeverLeak" + + +def load_cases() -> dict: + return json.loads((FIXTURES / "bot_cases.json").read_text(encoding="utf-8")) + + +def message(update_id: int, text: str, chat_type: str = "private", chat_id: int = 500) -> dict: + return {"update_id": update_id, "message": {"text": text, "chat": {"id": chat_id, "type": chat_type}}} + + +def updates(*items: dict) -> FakeResponse: + return FakeResponse(200, {"ok": True, "result": list(items)}) + + +SENT = FakeResponse(200, {"ok": True, "result": {"message_id": 1}}) +NOTHING_NEW = updates() + + +def calls_to(http: FakeHttp, method: str) -> list[dict]: + return [kwargs["json"] for url, kwargs in http.calls if url.endswith("/" + method)] + + +class ReplyTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.fixture = load_cases() + cls.feed = cls.fixture["feed"] + + def test_every_recorded_message_gets_the_recorded_reply(self): + self.assertGreaterEqual(len(self.fixture["cases"]), 20) + for case in self.fixture["cases"]: + with self.subTest(text=case["text"], private=case["private"]): + self.assertEqual(bot.reply_for(self.feed, case["text"], case["private"]), case["reply"]) + + def test_the_fixture_matches_what_the_feed_builder_writes_today(self): + self.assertEqual(self.feed["bot"], feed.public_feed(real_edition(), [])["bot"]) + + def test_every_reply_fits_a_telegram_message_and_has_no_emoji(self): + for command, reply in self.feed["bot"].items(): + with self.subTest(command=command): + self.assertLessEqual(feed.visible_length(reply), feed.LIMITS["telegram_message"]) + self.assertFalse(feed.has_emoji(reply)) + self.assertTrue(reply.strip()) + + def test_every_section_on_the_page_has_its_command(self): + sections = {item["section_id"] for item in self.feed["stories"]} + for section in sections: + self.assertIn(feed.SECTION_COMMANDS[section], self.feed["bot"]) + for command in ("/start", "/help", "/today", "/tape", "/sources", "/about"): + self.assertIn(command, self.feed["bot"]) + + def test_today_lists_the_five_digest_stories_with_links(self): + reply = bot.reply_for(self.feed, "/today") + self.assertEqual(reply.count("\n• "), 5) + self.assertEqual(reply.count("") + self.assertIn("<b>", miss) + self.assertNotIn("", miss) + + def test_a_group_is_only_answered_when_it_sends_a_command(self): + self.assertIsNone(bot.reply_for(self.feed, "hello there", private=False)) + self.assertIsNotNone(bot.reply_for(self.feed, "/today@MorrowBriefBot", private=False)) + self.assertEqual(bot.reply_for(self.feed, "hello there", private=True), self.feed["bot"]["/help"]) + + def test_before_the_first_edition_the_bot_says_so(self): + for empty in ({}, {"bot": {}}): + self.assertIn("not ready", bot.reply_for(empty, "/today")) + self.assertIn("not ready", bot.reply_for(empty, "hi")) + + def test_a_section_missing_from_the_page_gets_a_plain_answer(self): + thin = {"bot": {"/help": "menu", "/today": "today"}, "stories": []} + self.assertIn("Nothing under that heading", bot.reply_for(thin, "/space")) + self.assertEqual(bot.reply_for(thin, "/nonsense"), "menu") + + def test_command_of_drops_the_bot_name_and_case(self): + self.assertEqual(bot.command_of("/Find@MorrowBriefBot nvidia chips "), ("/find", "nvidia chips")) + self.assertEqual(bot.command_of("/today"), ("/today", "")) + + +class PollTest(unittest.TestCase): + def setUp(self): + self.feed = load_cases()["feed"] + self.http = FakeHttp() + + def poll(self, listen: int = 0) -> list[str]: + return bot.poll(self.http, TOKEN, self.feed, listen, clock=lambda: 0.0) + + def test_answers_then_acknowledges_so_nothing_repeats(self): + self.http.telegram = [ + updates(message(10, "/today"), message(11, "hello", "supergroup")), + SENT, + NOTHING_NEW, + ] + log = self.poll() + self.assertEqual(log, ["answered /today"]) + fetches = calls_to(self.http, "getUpdates") + self.assertEqual(len(fetches), 2) + self.assertNotIn("offset", fetches[0]) + self.assertEqual(fetches[1]["offset"], 12) + self.assertEqual(fetches[0]["allowed_updates"], ["message"]) + + def test_the_reply_is_html_without_a_link_preview(self): + self.http.telegram = [updates(message(10, "/today", chat_id=42)), SENT, NOTHING_NEW] + self.poll() + sent = calls_to(self.http, "sendMessage") + self.assertEqual(len(sent), 1) + self.assertEqual(sent[0]["chat_id"], 42) + self.assertEqual(sent[0]["text"], self.feed["bot"]["/today"]) + self.assertEqual(sent[0]["parse_mode"], "HTML") + self.assertEqual(sent[0]["link_preview_options"], {"is_disabled": True}) + + def test_a_group_message_that_is_not_a_command_gets_no_reply(self): + self.http.telegram = [updates(message(10, "good morning all", "group")), NOTHING_NEW] + self.assertEqual(self.poll(), []) + self.assertEqual(calls_to(self.http, "sendMessage"), []) + self.assertEqual(calls_to(self.http, "getUpdates")[-1]["offset"], 11) + + def test_a_webhook_means_the_worker_answers_and_polling_stops(self): + self.http.telegram = [FakeResponse(409, {"ok": False, "description": "Conflict: webhook is active"})] + log = self.poll() + self.assertEqual(len(self.http.calls), 1) + self.assertIn("webhook", log[0]) + + def test_a_message_that_cannot_be_answered_is_skipped_not_retried(self): + blocked = FakeResponse(403, {"ok": False, "description": "Forbidden: bot was blocked by the user"}) + self.http.telegram = [updates(message(10, "/today")), blocked, NOTHING_NEW] + log = self.poll() + self.assertIn("could not answer /today", log[0]) + self.assertEqual(calls_to(self.http, "getUpdates")[-1]["offset"], 11) + self.assertEqual(len(calls_to(self.http, "sendMessage")), 1) + + def test_the_token_never_reaches_the_log(self): + leaky = FakeResponse(401, {"ok": False, "description": f"Unauthorized for bot{TOKEN} at /bot{TOKEN}/getUpdates"}) + self.http.telegram = [leaky] + log = self.poll() + self.assertTrue(log) + self.assertNotIn(TOKEN, " ".join(log)) + self.assertNotIn("AAExample", " ".join(log)) + self.http = FakeHttp() + self.http.telegram = [updates(message(10, "/today")), leaky, NOTHING_NEW] + self.assertNotIn("AAExample", " ".join(self.poll())) + + def test_no_network_ends_the_run_quietly(self): + self.http.telegram = [requests.exceptions.ConnectionError("down")] + log = self.poll() + self.assertEqual(len(log), 1) + self.assertIn("unreachable", log[0]) + + def test_a_reply_that_times_out_does_not_stop_the_run(self): + self.http.telegram = [ + updates(message(10, "/today"), message(11, "/tape")), + requests.exceptions.ReadTimeout("slow"), + SENT, + NOTHING_NEW, + ] + log = self.poll() + self.assertEqual(log, ["could not answer: ReadTimeout", "answered /tape"]) + self.assertEqual(calls_to(self.http, "getUpdates")[-1]["offset"], 12) + + def test_a_busy_chat_cannot_keep_the_run_alive_forever(self): + # Every call returns one more message, so the run must end by its own limit. + counter = {"next": 100} + + class Endless(FakeHttp): + def post(inner, url, **kwargs): + inner.calls.append((url, kwargs)) + if url.endswith("/getUpdates"): + counter["next"] += 1 + return updates(message(counter["next"], "/today")) + return SENT + + self.http = Endless() + log = self.poll() + fetches = calls_to(self.http, "getUpdates") + self.assertEqual(len(fetches), bot.MAX_CLOSING_ROUNDS + 2) + self.assertEqual(len(log), bot.MAX_CLOSING_ROUNDS + 1) + # The last call acknowledges everything that was answered and asks for at most one more. + self.assertEqual(fetches[-1]["limit"], 1) + self.assertEqual(fetches[-1]["offset"], 100 + bot.MAX_CLOSING_ROUNDS + 2) + self.assertEqual(len(calls_to(self.http, "sendMessage")), bot.MAX_CLOSING_ROUNDS + 1) + + def test_listening_uses_long_polls_until_the_time_is_up(self): + ticks = iter([0.0, 0.0, 30.0, 61.0, 61.0]) + self.http.telegram = [NOTHING_NEW] + bot.poll(self.http, TOKEN, self.feed, 60, clock=lambda: next(ticks)) + waits = [payload["timeout"] for payload in calls_to(self.http, "getUpdates")] + self.assertEqual(waits, [bot.LONG_POLL_SECONDS, bot.LONG_POLL_SECONDS, 0]) + + +class CommandTest(unittest.TestCase): + def run_main(self, argv: list[str], env: dict) -> tuple[int, str]: + output = io.StringIO() + with mock.patch.dict(os.environ, env, clear=False), contextlib.redirect_stdout(output): + for name in ("TELEGRAM_BOT_TOKEN", "TELEGRAM_WEBHOOK_SECRET"): + if name not in env: + os.environ.pop(name, None) + code = bot.main(argv) + return code, output.getvalue() + + def test_poll_without_a_token_does_nothing_and_succeeds(self): + code, text = self.run_main(["poll", "--feed", "missing.json", "--listen", "0"], {}) + self.assertEqual(code, 0) + self.assertIn("not set", text) + + def test_set_webhook_needs_both_secrets_and_https(self): + code, text = self.run_main(["set-webhook", "--url", "https://example.workers.dev"], {"TELEGRAM_BOT_TOKEN": TOKEN}) + self.assertEqual(code, 1) + self.assertNotIn(TOKEN, text) + code, text = self.run_main( + ["set-webhook", "--url", "http://example.workers.dev"], + {"TELEGRAM_BOT_TOKEN": TOKEN, "TELEGRAM_WEBHOOK_SECRET": "s3cret"}, + ) + self.assertEqual(code, 1) + self.assertIn("https://", text) + + def test_set_webhook_sends_the_secret_and_only_asks_for_messages(self): + http = FakeHttp() + with mock.patch.object(bot, "requests", http): + code, text = self.run_main( + ["set-webhook", "--url", "https://example.workers.dev"], + {"TELEGRAM_BOT_TOKEN": TOKEN, "TELEGRAM_WEBHOOK_SECRET": "s3cret"}, + ) + self.assertEqual(code, 0) + self.assertIn("webhook set", text) + payload = calls_to(http, "setWebhook")[0] + self.assertEqual(payload["secret_token"], "s3cret") + self.assertEqual(payload["allowed_updates"], ["message"]) + self.assertNotIn("s3cret", text) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/social/test_cards.py b/tests/social/test_cards.py new file mode 100644 index 0000000..89c06d8 --- /dev/null +++ b/tests/social/test_cards.py @@ -0,0 +1,203 @@ +"""The cards: gate 5 (1080 x 1350 PNG, readable colours) and a layout that stays inside its margins.""" +import tempfile +import unittest +from pathlib import Path + +from PIL import Image + +import helpers +from helpers import edition_from, real_edition, story + +import social_cards as cards +import social_feed as feed + +LONG_TITLE = ("Regulators on three continents open parallel inquiries into the largest chip supply agreement " + "ever signed, as rivals warn of shortages lasting well into the next decade and beyond " * 2)[:300] +LONG_NAME = "The Extremely Long Named International Gazette of Record and Commerce" + + +def assert_inside_the_page(test: unittest.TestCase, metrics: dict) -> None: + test.assertEqual(metrics["size"], (1080, 1350)) + test.assertFalse(metrics["overlaps"], metrics) + test.assertGreaterEqual(metrics["min_left"], cards.MARGIN - 0.5, metrics) + test.assertLessEqual(metrics["max_right"], cards.WIDTH - cards.MARGIN + 0.5, metrics) + test.assertLessEqual(metrics["max_bottom"], cards.HEIGHT - cards.MARGIN + 0.5, metrics) + test.assertLessEqual(set(metrics["colours"]), set(cards.TEXT_COLOURS), metrics) + + +class PaletteTest(unittest.TestCase): + def test_gate_5_every_text_colour_reads_on_paper(self): + for colour in cards.TEXT_COLOURS: + with self.subTest(colour=colour): + self.assertGreaterEqual(cards.contrast_ratio(colour), 4.5) + + def test_contrast_matches_the_known_extremes(self): + cards.PALETTE["_white"], cards.PALETTE["_black"] = "#ffffff", "#000000" + try: + self.assertAlmostEqual(cards.contrast_ratio("_black", "_white"), 21.0, places=2) + self.assertAlmostEqual(cards.contrast_ratio("_white", "_white"), 1.0, places=5) + finally: + del cards.PALETTE["_white"], cards.PALETTE["_black"] + + def test_one_accent_and_no_purple(self): + for name, value in cards.PALETTE.items(): + red, green, blue = cards.rgb(name) + self.assertFalse(blue > red > green and blue - green > 40, f"{name} {value} reads as purple") + + +class StoryCardTest(unittest.TestCase): + def test_every_story_on_todays_page_fits(self): + edition = real_edition() + for item in edition["stories"]: + with self.subTest(title=item["title"][:40]): + assert_inside_the_page(self, cards.render_story_card(item, edition["date"])) + + def test_a_300_character_headline_still_fits(self): + edition = edition_from([ + story(1, "space", 12, kind="lead", title=LONG_TITLE, dek=LONG_TITLE, source=LONG_NAME, + related=[{"source": f"{LONG_NAME} {n}", "title": "t", "url": f"https://example.com/r{n}"} + for n in range(1, 8)]), + story(2, "ai", 4, title=LONG_TITLE, dek=LONG_TITLE, source=LONG_NAME, tag="A Remarkably Long Topic Label " * 3), + story(3, "ai", 1, title=LONG_TITLE, dek="", source=LONG_NAME), + ]) + self.assertEqual(len(edition["stories"]), 3) + for item in edition["stories"]: + with self.subTest(coverage=item["coverage"]): + metrics = cards.render_story_card(item, edition["date"]) + assert_inside_the_page(self, metrics) + self.assertGreaterEqual(metrics["headline_size"], cards.HEADLINE_SIZES[-1]) + + def test_todays_long_lead_headline_stays_within_four_lines(self): + edition = real_edition() + lead = next(item for item in edition["stories"] if item["is_lead"]) + metrics = cards.render_story_card(lead, edition["date"]) + assert_inside_the_page(self, metrics) + self.assertLessEqual(metrics["headline_lines"], cards.HEADLINE_LINES) + self.assertGreaterEqual(metrics["headline_size"], cards.HEADLINE_FLOOR) + + def test_a_short_headline_is_set_large(self): + edition = edition_from([story(1, "ai", 3, title="Nvidia tops forecasts", dek="A short summary.")]) + metrics = cards.render_story_card(edition["stories"][0], edition["date"]) + assert_inside_the_page(self, metrics) + self.assertEqual(metrics["headline_size"], cards.HEADLINE_SIZES[0]) + self.assertEqual(metrics["dek_lines"], 1) + + def test_no_summary_on_todays_cards_is_cut(self): + edition = real_edition() + with_summary = 0 + for item in edition["stories"]: + with self.subTest(title=item["title"][:40]): + text = cards.render_story_card(item, edition["date"])["dek_text"] + self.assertFalse(text.endswith(("\u2026", "...")), text) + if text: + with_summary += 1 + self.assertTrue(item["dek"].startswith(text), text) + if text != " ".join(item["dek"].split()): + # Shorter than the outlet's summary, so it must stop at the end of a sentence. + self.assertRegex(text, r"[.!?][\"'\u201d\u2019)]*$") + self.assertGreater(with_summary, 0) + + def test_an_outlets_cut_off_fragment_is_left_off_the_card(self): + whole = "Nvidia poured tens of billions into AI startups under Huang." + edition = edition_from([ + story(1, "ai", 3, title="Founders brace for smaller checks", dek=f"{whole} Without him, VCs are bracing for\u2026"), + story(2, "ai", 2, title="A story whose only sentence is cut", dek="Regulators said on Friday that the inquiry would\u2026"), + ]) + first, second = (cards.render_story_card(item, edition["date"]) for item in edition["stories"]) + self.assertEqual(first["dek_text"], whole) + self.assertEqual(second["dek_lines"], 0) + for metrics in (first, second): + assert_inside_the_page(self, metrics) + + def test_a_short_summary_is_set_larger_than_a_long_one(self): + long_summary = " ".join(["The company said the change takes effect next quarter."] * 4) + edition = edition_from([ + story(1, "ai", 3, title="Nvidia tops forecasts again", dek="A short summary."), + story(2, "ai", 2, title="Nvidia tops forecasts again", dek=long_summary), + ]) + short, long = (cards.render_story_card(item, edition["date"]) for item in edition["stories"]) + self.assertEqual(short["dek_size"], cards.DEK_SIZES[0]) + self.assertGreaterEqual(long["dek_lines"], 3) + self.assertLessEqual(long["dek_size"], short["dek_size"]) + self.assertIn(long["dek_size"], cards.DEK_SIZES) + + def test_gate_5_the_saved_card_is_a_png_of_the_right_size(self): + edition = real_edition() + with tempfile.TemporaryDirectory() as folder: + path = Path(folder) / "cards" / "one.png" + cards.render_story_card(edition["stories"][0], edition["date"], path) + self.assertEqual(path.read_bytes()[:8], b"\x89PNG\r\n\x1a\n") + with Image.open(path) as image: + self.assertEqual(image.format, "PNG") + self.assertEqual(image.size, (1080, 1350)) + self.assertEqual(image.getpixel((4, 4)), cards.rgb("accent")) + self.assertEqual(image.getpixel((cards.WIDTH - 5, cards.ACCENT_BAR - 1)), cards.rgb("accent")) + self.assertEqual(image.getpixel((4, cards.ACCENT_BAR + 8)), cards.rgb("paper")) + self.assertLess(path.stat().st_size, 1_000_000) + + +class CoverCardTest(unittest.TestCase): + def test_todays_cover_fits(self): + edition = real_edition() + digest = feed.pick_digest(edition["stories"]) + metrics = cards.render_cover_card(digest, edition["markets"], edition["date"], feed.outlet_total(edition["stories"])) + assert_inside_the_page(self, metrics) + self.assertEqual(metrics["rows"], 5) + self.assertFalse(metrics["clipped"], "a headline on today's cover was cut") + self.assertGreaterEqual(metrics["lead_size"], 56) + self.assertGreaterEqual(metrics["title_size"], 30) + self.assertGreaterEqual(metrics["lead_size"], cards.COVER_LEAD_RATIO * metrics["title_size"]) + + def test_a_short_lead_is_set_large_on_the_cover(self): + edition = edition_from([story(n, "ai", 6 - n, title=f"Nvidia tops forecasts {n}") for n in range(1, 6)]) + metrics = cards.render_cover_card(edition["stories"], edition["markets"], edition["date"], 9) + assert_inside_the_page(self, metrics) + self.assertEqual(metrics["lead_size"], cards.COVER_LEAD_SIZES[0]) + self.assertEqual(metrics["title_size"], cards.COVER_ROW_SIZES[0]) + self.assertFalse(metrics["clipped"]) + + def test_long_titles_three_stories_and_no_markets_still_fit(self): + edition = edition_from( + [story(n, section, 3, title=LONG_TITLE, source=LONG_NAME) + for n, section in ((1, "ai"), (2, "space"), (3, "robotics"), (4, "big_tech"), (5, "technology"))], + markets=False, + ) + for count in (3, 5): + with self.subTest(count=count): + metrics = cards.render_cover_card(edition["stories"][:count], [], edition["date"], 40) + assert_inside_the_page(self, metrics) + self.assertTrue(metrics["clipped"]) + + def test_long_market_names_stay_in_their_column(self): + edition = real_edition() + markets = [dict(quote, name="A Very Long Index Name Indeed (Composite)") for quote in edition["markets"]] + metrics = cards.render_cover_card(feed.pick_digest(edition["stories"]), markets, edition["date"], 15) + assert_inside_the_page(self, metrics) + + +class PostCardsTest(unittest.TestCase): + def test_a_digest_gets_a_cover_and_one_card_per_story_and_they_never_change(self): + edition = real_edition() + post = feed.digest_post(feed.pick_digest(edition["stories"]), edition["markets"], edition["date"]) + with tempfile.TemporaryDirectory() as folder: + out_dir = Path(folder) + written = cards.render_post_cards(post, edition, out_dir) + self.assertEqual(len(written), 6) + self.assertEqual([path.relative_to(out_dir).as_posix() for path in written], post["cards"]) + before = {path: path.read_bytes() for path in written} + (out_dir / post["cards"][1]).write_bytes(b"already sent") + self.assertEqual(cards.render_post_cards(post, edition, out_dir), []) + self.assertEqual((out_dir / post["cards"][1]).read_bytes(), b"already sent") + self.assertEqual((out_dir / post["cards"][0]).read_bytes(), before[written[0]]) + + def test_fit_line_never_exceeds_the_width(self): + face = cards.font("sans_bold", 28) + for text in ("SHORT", "A LABEL THAT GOES ON FOR A VERY LONG TIME " * 4, "ONEVERYLONGWORDWITHOUTANYSPACES" * 5): + fitted = cards.fit_line(text, face, 400, 3) + self.assertLessEqual(cards.text_width(fitted, face, 3), 400) + self.assertTrue(fitted) + self.assertEqual(cards.fit_line("SHORT", face, 400, 3), "SHORT") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/social/test_feed.py b/tests/social/test_feed.py new file mode 100644 index 0000000..e7c8c5b --- /dev/null +++ b/tests/social/test_feed.py @@ -0,0 +1,362 @@ +"""Parsing, story selection and the text of every post.""" +import unittest + +import helpers +from helpers import edition_from, real_edition, story + +import social_feed as feed + +LONG = "An extraordinarily long headline that keeps going well past any sensible length " * 6 + + +def all_texts(post: dict) -> list[str]: + return [post["telegram"], post["threads"], post["instagram"], *post["x"], *post["alts"]] + + +def hard_edition() -> dict: + """Worst-case input: very long titles, summaries and outlet names, with markup characters in them.""" + stories = [ + story(1, "space", 9, kind="lead", title=LONG, dek=LONG * 3, source="The Extremely Long Named Gazette of Record", + related=[{"source": f"Another Very Long Outlet Name {n}", "title": LONG, "url": f"https://example.com/r{n}"} + for n in range(1, 9)]), + story(2, "ai", 5, title=LONG, dek=LONG * 3, source="A Bold & Co"), + story(3, "ai", 4, title=LONG, url="https://example.com/" + "p" * 600), + story(4, "big_tech", 3, title=LONG), + story(5, "robotics", 3, title=LONG, kind="row"), + story(6, "markets_news", 3, title=LONG, dek=""), + story(7, "technology", 1, title=LONG), + ] + return edition_from(stories) + + +class ParsingTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.edition = real_edition() + + def test_reads_the_published_page(self): + edition = self.edition + self.assertEqual(edition["date"], "2026-11-14") + self.assertRegex(edition["updated"], r"^\d{1,2}:\d{2}$") + self.assertEqual(len(edition["stories"]), 32) + self.assertEqual(feed.outlet_total(edition["stories"]), 15) + self.assertEqual(len(edition["markets"]), 5) + self.assertEqual(len({item["id"] for item in edition["stories"]}), 32) + + def test_every_story_is_complete(self): + for item in self.edition["stories"]: + self.assertTrue(item["title"] and item["source"] and item["section"], item) + self.assertTrue(item["url"].startswith("http"), item) + self.assertIn(item["section_id"], feed.SECTION_COMMANDS, item) + self.assertGreaterEqual(item["coverage"], 1) + + def test_lead_carries_the_other_outlets(self): + leads = [item for item in self.edition["stories"] if item["is_lead"]] + self.assertEqual(len(leads), 1) + self.assertEqual(leads[0]["coverage"], 5) + self.assertEqual(len(feed.outlet_names(leads[0])), 5) + + def test_markets_have_direction_and_size(self): + for quote in self.edition["markets"]: + self.assertIn(quote["dir"], ("up", "down", "flat")) + self.assertGreaterEqual(quote["pct"], 0) + self.assertTrue(quote["name"]) + + def test_page_without_a_date_gives_an_empty_date(self): + edition = feed.parse_edition("Morrow") + self.assertEqual(edition["date"], "") + self.assertEqual(edition["stories"], []) + + def test_emoji_is_removed_from_headlines_and_summaries(self): + edition = edition_from([story(1, title="\U0001F680 Rocket lifts off \U0001F525\uFE0F today", dek="Big \u2728 day")]) + item = edition["stories"][0] + self.assertEqual(item["title"], "Rocket lifts off today") + self.assertEqual(item["dek"], "Big day") + + def test_a_story_shown_twice_is_kept_once_with_the_higher_count(self): + twice = [story(1, "ai", 2, kind="lead"), story(1, "ai", 4), story(2, "ai", 1)] + edition = edition_from(twice) + self.assertEqual(len(edition["stories"]), 2) + self.assertEqual(edition["stories"][0]["coverage"], 4) + + +class SelectionTest(unittest.TestCase): + def test_digest_is_five_stories_lead_first_two_per_section(self): + edition = real_edition() + digest = feed.pick_digest(edition["stories"]) + self.assertEqual(len(digest), 5) + self.assertTrue(digest[0]["is_lead"]) + self.assertTrue(digest[0]["title"].startswith("SpaceX launches Starship")) + for section in {item["section_id"] for item in digest}: + self.assertLessEqual(sum(item["section_id"] == section for item in digest), feed.MAX_PER_SECTION) + counts = [item["coverage"] for item in digest[1:]] + self.assertEqual(counts, sorted(counts, reverse=True)) + + def test_alerts_need_three_outlets_and_skip_used_stories(self): + edition = edition_from([story(1, "ai", 4), story(2, "ai", 3), story(3, "space", 2), story(4, "space", 1)]) + ids = [item["id"] for item in edition["stories"]] + candidates = feed.alert_candidates(edition["stories"], set()) + self.assertEqual([item["id"] for item in candidates], ids[:2]) + self.assertEqual([item["id"] for item in feed.alert_candidates(edition["stories"], {ids[0]})], [ids[1]]) + + def test_todays_page_has_no_alert_outside_the_digest(self): + edition = real_edition() + used = {item["id"] for item in feed.pick_digest(edition["stories"])} + self.assertEqual(feed.alert_candidates(edition["stories"], used), []) + + +class TextTest(unittest.TestCase): + def test_x_counts_a_link_as_23_and_wide_characters_as_2(self): + self.assertEqual(feed.x_length("abc"), 3) + self.assertEqual(feed.x_length("see https://example.com/" + "a" * 200), 4 + 23) + self.assertEqual(feed.x_length("\u6f22\u5b57"), 4) + + def test_telegram_length_ignores_tags_and_links(self): + self.assertEqual(feed.visible_length('Hi there & you'), 14) + + def test_truncate_cuts_at_a_word_and_never_exceeds_the_limit(self): + text = "one two three four five six" + for limit in range(0, len(text) + 2): + cut = feed.truncate(text, limit) + self.assertLessEqual(len(cut), limit) + if cut and cut != text: + self.assertTrue(cut.endswith("\u2026")) + self.assertTrue(text.startswith(cut[:-1])) + if len(cut) > len("one\u2026") and cut != text: + self.assertEqual(text[len(cut) - 1], " ", cut) + self.assertEqual(feed.truncate(text, len(text)), text) + + def test_truncate_cuts_inside_a_single_word_that_cannot_fit(self): + self.assertEqual(feed.truncate("one two", 3), "on\u2026") + self.assertEqual(feed.truncate("a" * 50, 10), "a" * 9 + "\u2026") + self.assertEqual(feed.truncate("abc def", 1), "") + self.assertEqual(feed.truncate("abc def", 0), "") + + def test_sentences_do_not_break_on_abbreviations(self): + text = "SpaceX flew today (Nov. 13) from Texas. The U.S. Navy watched. Acme Inc. said it was pleased." + self.assertEqual(feed.sentences(text), [ + "SpaceX flew today (Nov. 13) from Texas.", + "The U.S. Navy watched.", + "Acme Inc. said it was pleased.", + ]) + self.assertEqual(feed.sentences("No full stop here"), ["No full stop here"]) + + def test_a_summary_is_kept_as_whole_sentences(self): + first = "OpenAI is launching a wallet for its agents this week." + dek = f"{first} ChatGPT Wallet is rolling out\u2026" + self.assertEqual(feed.fit_dek(dek, 500), first) + self.assertEqual(feed.fit_dek(dek, len(first)), first) + self.assertEqual(feed.fit_dek(dek, len(first) - 1, cut=False), "") + cut = feed.fit_dek(dek, len(first) - 1) + self.assertTrue(cut.endswith("\u2026") and len(cut) <= len(first) - 1, cut) + self.assertEqual(feed.fit_dek("Short and whole.", 200), "Short and whole.") + self.assertEqual(feed.fit_dek("A summary with no full stop that the outlet cut off\u2026", 200, cut=False), "") + self.assertEqual(feed.fit_dek(first, feed.MIN_DEK - 1), "") + self.assertEqual(feed.fit_dek("", 500), "") + for room in range(0, 130): + self.assertLessEqual(len(feed.fit_dek(dek, room)), room) + self.assertLessEqual(feed.x_length(feed.fit_dek(dek, room, feed.x_length)), room) + + def test_outlet_phrase_states_only_what_the_page_shows(self): + edition = edition_from([ + story(1, "ai", 5, kind="lead", source="CNBC", + related=[{"source": name, "title": "t", "url": f"https://example.com/{name}"} + for name in ("Reuters", "Wired", "Ars Technica", "Axios")]), + story(2, "ai", 3, source="The Verge"), + story(3, "ai", 1, source="Wired"), + ]) + lead, counted, single = edition["stories"] + self.assertEqual(feed.outlets_phrase(lead, 2), "5 outlets: CNBC, Reuters +3") + self.assertEqual(feed.outlets_phrase(lead, 9), "5 outlets: CNBC, Reuters, Wired, Ars Technica, Axios") + self.assertEqual(feed.outlets_phrase(counted), "3 outlets, first: The Verge") + self.assertEqual(feed.outlets_phrase(single), "via Wired") + + +class PostTest(unittest.TestCase): + def posts(self, edition: dict) -> list[dict]: + digest_stories = feed.pick_digest(edition["stories"]) + posts = [feed.digest_post(digest_stories, edition["markets"], edition["date"])] + posts += [feed.story_post(item, edition["date"]) for item in edition["stories"]] + return posts + + def test_gate_1_every_text_fits_its_platform_on_todays_page(self): + for post in self.posts(real_edition()): + self.assertEqual(feed.post_problems(post), [], post["id"]) + + def test_gate_1_every_text_fits_its_platform_on_worst_case_input(self): + for post in self.posts(hard_edition()): + self.assertEqual(feed.post_problems(post), [], post["id"]) + self.assertLessEqual(feed.visible_length(post["telegram"]), 1024) + self.assertLessEqual(len(post["threads"]), 500) + self.assertLessEqual(len(post["instagram"]), 2200) + for text in post["x"]: + self.assertLessEqual(feed.x_length(text), 280) + for alt in post["alts"]: + self.assertLessEqual(len(alt), 400) + + def test_gate_2_every_post_names_its_outlet_and_links_to_it(self): + for edition in (real_edition(), hard_edition()): + by_id = {item["id"]: item for item in edition["stories"]} + for post in self.posts(edition): + for story_id in post["story_ids"]: + item = by_id[story_id] + source = item["source"] + self.assertIn(feed.esc(source), post["telegram"], post["id"]) + self.assertIn(source, post["instagram"], post["id"]) + self.assertIn(source, post["threads"], post["id"]) + self.assertTrue(any(source in text for text in post["x"]), post["id"]) + self.assertIn(feed.htmllib.escape(item["url"], quote=True), post["telegram"], post["id"]) + self.assertTrue(any(item["url"] in text for text in post["x"]), post["id"]) + self.assertIn("http", post["threads"]) + + def test_gate_3_no_emoji_in_any_text(self): + edition = edition_from([ + story(n, "ai", 4 - n % 3, title=f"\U0001F4A5 Loud headline {n} \u2757", dek="Wow \U0001F62E") + for n in range(1, 7) + ]) + for post in self.posts(edition) + self.posts(real_edition()): + for text in all_texts(post): + self.assertFalse(feed.has_emoji(text), text) + for reply in feed.bot_replies(real_edition()).values(): + self.assertFalse(feed.has_emoji(reply), reply) + + def test_markup_in_a_headline_is_escaped_for_telegram(self): + edition = edition_from([story(1, "ai", 3, title='Chips