diff --git a/packages/junior-evals/README.md b/packages/junior-evals/README.md
index d454afa8e3..1071f50e0a 100644
--- a/packages/junior-evals/README.md
+++ b/packages/junior-evals/README.md
@@ -323,7 +323,14 @@ cache keeps the file for the later runs of a pull request.
- The fixture always mocks the search provider of `webSearch`. No test turns
the mock on or off, and no search reaches the real provider. A search finds
nothing by default. `mockWebSearchResults()` from `src/fixture/web.ts` sets
- the results for one test.
+ the results for one test. Give it a list for the same results on every
+ query, or a function that gets the query, so that each search can find the
+ page that it asks for.
+- `mockWebSites()` from the same file gives `webFetch` one page for each host
+ in one test. Every path on the host, with or without `www.`, gets that page.
+ Use a real page of the site, so the agent reads what a real user's agent
+ would read. Other hosts go to Roach. The `bash` tool in the Vercel Sandbox
+ does not use these mocks.
## Recordings
diff --git a/packages/junior-evals/evals/agent/delegation.eval.ts b/packages/junior-evals/evals/agent/delegation.eval.ts
index bd65fb4843..a89706e92e 100644
--- a/packages/junior-evals/evals/agent/delegation.eval.ts
+++ b/packages/junior-evals/evals/agent/delegation.eval.ts
@@ -1,4 +1,5 @@
import { describe, expect } from "vitest";
+import { mockDatabaseDocs } from "@junior-evals/fixture/database-docs";
import { slackMention } from "@junior-evals/fixture/inputs";
import { completedToolCalls } from "@junior-evals/fixture/results";
import { test } from "@junior-evals/fixture/test";
@@ -12,6 +13,7 @@ describe("Implicit Delegation", () => {
// A small budget keeps each child short. The case measures the parent's
// choice to delegate, not the quality of the children's research.
const { run } = await agent({ limits: { maxToolCallsPerTurn: 6 } });
+ mockDatabaseDocs();
const conversation = await run(
slackMention(
"I need to pick a database for a new service. Research how PostgreSQL, MySQL, and SQLite each handle adding a NOT NULL column with a default to a very large table: locking, rewrite cost, and replication impact. Go to each project's own docs, then give me the tradeoffs.",
diff --git a/packages/junior-evals/evals/coding/subscriptions.eval.ts b/packages/junior-evals/evals/coding/subscriptions.eval.ts
index 1ab51dd402..0c5745df44 100644
--- a/packages/junior-evals/evals/coding/subscriptions.eval.ts
+++ b/packages/junior-evals/evals/coding/subscriptions.eval.ts
@@ -70,7 +70,7 @@ function watchedEvents(conversation: { toolCalls: ToolCall[] }) {
}
describe("Watches", () => {
- test("looks up and subscribes to an exact deployment before GitHub creates it", async ({
+ test("subscribes to an exact deployment before GitHub creates it", async ({
run,
}) => {
const commitSha = "c610b5d6a88c9da5d65627a1cdb3829b05c14f75";
@@ -93,15 +93,6 @@ describe("Watches", () => {
}),
);
- expect(
- completedToolCalls("github_getDeployment", conversation).map(
- (call) => call.input,
- ),
- ).toContainEqual({
- commitSha,
- environment: "Production",
- repo: "getsentry/junior-prod",
- });
expect(
completedToolCalls("watchEvents", conversation).map((call) => call.input),
).toContainEqual(
diff --git a/packages/junior-evals/src/fixture/database-docs.ts b/packages/junior-evals/src/fixture/database-docs.ts
new file mode 100644
index 0000000000..3fe4698d6f
--- /dev/null
+++ b/packages/junior-evals/src/fixture/database-docs.ts
@@ -0,0 +1,84 @@
+/**
+ * Real docs of PostgreSQL, MySQL, and SQLite for tests where agents research
+ * databases on the web.
+ *
+ * There is one real docs page for each database. Each file is the live page
+ * with the scripts, styles, and parts that `webFetch` does not read taken
+ * out. It is cut after the first 12,000 characters, which is the most that
+ * `webFetch` returns. `webFetch` returns the same text for these files as for
+ * the live pages.
+ */
+import { readFileSync } from "node:fs";
+import {
+ mockWebSearchResults,
+ mockWebSites,
+ type WebSearchResult,
+} from "./web";
+
+const DATABASE_DOCS = [
+ {
+ host: "postgresql.org",
+ file: "postgresql-alter-table.html",
+ url: "https://www.postgresql.org/docs/current/sql-altertable.html",
+ title: "PostgreSQL: Documentation: 18: ALTER TABLE",
+ excerpt: "ALTER TABLE — change the definition of a table",
+ },
+ {
+ host: "dev.mysql.com",
+ file: "mysql-online-ddl-operations.html",
+ url: "https://dev.mysql.com/doc/refman/8.4/en/innodb-online-ddl-operations.html",
+ title:
+ "MySQL :: MySQL 8.4 Reference Manual :: 17.12.1 Online DDL Operations",
+ excerpt:
+ "Online support details, syntax examples, and usage notes for DDL operations are provided under the following topics in this section.",
+ },
+ {
+ host: "sqlite.org",
+ file: "sqlite-alter-table.html",
+ url: "https://www.sqlite.org/lang_altertable.html",
+ title: "ALTER TABLE",
+ excerpt: "SQLite supports a limited subset of ALTER TABLE.",
+ },
+];
+
+/**
+ * Return the docs pages, or only those on the hosts of `site:` filters. As on
+ * a search engine, a site also matches its subdomains.
+ */
+function searchDatabaseDocs(query: string): WebSearchResult[] {
+ const sites = query
+ .toLowerCase()
+ .split(/\s+/)
+ .filter((word) => word.startsWith("site:"))
+ .map(
+ (word) =>
+ word
+ .slice("site:".length)
+ .replace(/^(?:https?:\/\/)?(?:www\.)?/, "")
+ .split("/")[0],
+ );
+ return DATABASE_DOCS.filter(
+ (doc) =>
+ sites.length === 0 ||
+ sites.some((site) => doc.host === site || doc.host.endsWith(`.${site}`)),
+ ).map(({ excerpt, title, url }) => ({ excerpt, title, url }));
+}
+
+/**
+ * Serve the database docs in this test. A search finds the three pages, and
+ * a read of any path on a docs host gets the page of that host.
+ */
+export function mockDatabaseDocs(): void {
+ mockWebSearchResults(searchDatabaseDocs);
+ mockWebSites(
+ Object.fromEntries(
+ DATABASE_DOCS.map((doc) => [
+ doc.host,
+ readFileSync(
+ new URL(`./database-docs/${doc.file}`, import.meta.url),
+ "utf8",
+ ),
+ ]),
+ ),
+ );
+}
diff --git a/packages/junior-evals/src/fixture/database-docs/mysql-online-ddl-operations.html b/packages/junior-evals/src/fixture/database-docs/mysql-online-ddl-operations.html
new file mode 100644
index 0000000000..f88a5cd840
--- /dev/null
+++ b/packages/junior-evals/src/fixture/database-docs/mysql-online-ddl-operations.html
@@ -0,0 +1,522 @@
+
MySQL :: MySQL 8.4 Reference Manual :: 17.12.1 Online DDL Operations
+
+ The following table provides an overview of online DDL support
+ for index operations. An asterisk indicates additional
+ information, an exception, or a dependency. For details, see
+ Syntax and Usage Notes.
+
+
+
Table 17.15 Online DDL Support for Index Operations
+
+
+
Operation
+
Instant
+
In Place
+
Rebuilds Table
+
Permits Concurrent DML
+
Only Modifies Metadata
+
+
Creating or adding a secondary index
+
No
+
Yes
+
No
+
Yes
+
No
+
+
Dropping an index
+
No
+
Yes
+
No
+
Yes
+
Yes
+
+
Renaming an index
+
No
+
Yes
+
No
+
Yes
+
Yes
+
+
Adding a FULLTEXT index
+
No
+
Yes*
+
No*
+
No
+
No
+
+
Adding a SPATIAL index
+
No
+
Yes
+
No
+
No
+
No
+
+
Changing the index type
+
Yes
+
Yes
+
No
+
Yes
+
Yes
+
+
+
+
+
Syntax and Usage Notes
+
+
+ Creating or adding a secondary index
+
CREATE INDEX name ON table (col_list);
ALTER TABLE tbl_name ADD INDEX name (col_list);
+ The table remains available for read and write operations
+ while the index is being created. The
+ CREATE INDEX statement only
+ finishes after all transactions that are accessing the table
+ are completed, so that the initial state of the index
+ reflects the most recent contents of the table.
+
+ Online DDL support for adding secondary indexes means that
+ you can generally speed the overall process of creating and
+ loading a table and associated indexes by creating the table
+ without secondary indexes, then adding secondary indexes
+ after the data is loaded.
+
+ A newly created secondary index contains only the committed
+ data in the table at the time the
+ CREATE INDEX or
+ ALTER TABLE statement
+ finishes executing. It does not contain any uncommitted
+ values, old versions of values, or values marked for
+ deletion but not yet removed from the old index.
+
+ The table remains available for read and write operations
+ while the index is being dropped. The
+ DROP INDEX statement only
+ finishes after all transactions that are accessing the table
+ are completed, so that the initial state of the index
+ reflects the most recent contents of the table.
+
+ Renaming an index
+
ALTER TABLE tbl_name RENAME INDEX old_index_name TO new_index_name, ALGORITHM=INPLACE, LOCK=NONE;
+ Adding a FULLTEXT index
+
CREATE FULLTEXT INDEX name ON table(column);
+ Adding the first FULLTEXT index rebuilds
+ the table if there is no user-defined
+ FTS_DOC_ID column. Additional
+ FULLTEXT indexes may be added without
+ rebuilding the table.
+
+ Changing the index type (USING {BTREE |
+ HASH})
+
ALTER TABLE tbl_name DROP INDEX i1, ADD INDEX i1(key_part,...) USING BTREE, ALGORITHM=INSTANT;
+
+
+
+
+
+
+
Primary Key Operations
+
+
+
+
+ The following table provides an overview of online DDL support
+ for primary key operations. An asterisk indicates additional
+ information, an exception, or a dependency. See
+ Syntax and Usage Notes.
+
+
+
Table 17.16 Online DDL Support for Primary Key Operations
+
+
+
Operation
+
Instant
+
In Place
+
Rebuilds Table
+
Permits Concurrent DML
+
Only Modifies Metadata
+
+
Adding a primary key
+
No
+
Yes*
+
Yes*
+
Yes
+
No
+
+
Dropping a primary key
+
No
+
No
+
Yes
+
No
+
No
+
+
Dropping a primary key and adding another
+
No
+
Yes
+
Yes
+
Yes
+
No
+
+
+
+
+
Syntax and Usage Notes
+
+
+ Adding a primary key
+
ALTER TABLE tbl_name ADD PRIMARY KEY (column), ALGORITHM=INPLACE, LOCK=NONE;
+ Rebuilds the table in place. Data is reorganized
+ substantially, making it an expensive operation.
+ ALGORITHM=INPLACE is not permitted under
+ certain conditions if columns have to be converted to
+ NOT NULL.
+
+ Restructuring the
+ clustered index
+ always requires copying of table data. Thus, it is best to
+ define the primary
+ key when you create a table, rather than issuing
+ ALTER TABLE ... ADD PRIMARY KEY later.
+
+ When you create a UNIQUE or
+ PRIMARY KEY index, MySQL must do some
+ extra work. For UNIQUE indexes, MySQL
+ checks that the table contains no duplicate values for the
+ key. For a PRIMARY KEY index, MySQL also
+ checks that none of the PRIMARY KEY
+ columns contains a NULL.
+
+ When you add a primary key using the
+ ALGORITHM=COPY clause, MySQL converts
+ NULL values in the associated columns to
+ default values: 0 for numbers, an empty string for
+ character-based columns and BLOBs, and 0000-00-00 00:00:00
+ for DATETIME. This is a non-standard
+ behavior that Oracle recommends you not rely on. Adding a
+ primary key using ALGORITHM=INPLACE is
+ only permitted when the
+ SQL_MODE setting includes
+ the strict_trans_tables or
+ strict_all_tables flags; when the
+ SQL_MODE setting is strict,
+ ALGORITHM=INPLACE is permitted, but the
+ statement can still fail if the requested primary key
+ columns contain NULL values. The
+ ALGORITHM=INPLACE behavior is more
+ standard-compliant.
+
+ If you create a table without a primary key,
+ InnoDB chooses one for you, which can be
+ the first UNIQUE key defined on
+ NOT NULL columns, or a system-generated
+ key. To avoid uncertainty and the potential space
+ requirement for an extra hidden column, specify the
+ PRIMARY KEY clause as part of the
+ CREATE TABLE statement.
+
+ MySQL creates a new clustered index by copying the existing
+ data from the original table to a temporary table that has
+ the desired index structure. Once the data is completely
+ copied to the temporary table, the original table is renamed
+ with a different temporary table name. The temporary table
+ comprising the new clustered index is renamed with the name
+ of the original table, and the original table is dropped
+ from the database.
+
+ The online performance enhancements that apply to operations
+ on secondary indexes do not apply to the primary key index.
+ The rows of an InnoDB table are stored in a
+ clustered index
+ organized based on the
+ primary key, forming
+ what some database systems call an “index-organized
+ table”. Because the table structure is closely tied
+ to the primary key, redefining the primary key still
+ requires copying the data.
+
+ When an operation on the primary key uses
+ ALGORITHM=INPLACE, even though the data
+ is still copied, it is more efficient than using
+ ALGORITHM=COPY because:
+
+
+
+ No undo logging or associated redo logging is required
+ for ALGORITHM=INPLACE. These
+ operations add overhead to DDL statements that use
+ ALGORITHM=COPY.
+
+ The secondary index entries are pre-sorted, and so can
+ be loaded in order.
+
+ The change buffer is not used, because there are no
+ random-access inserts into the secondary indexes.
+
+
+
+ Dropping a primary key
+
ALTER TABLE tbl_name DROP PRIMARY KEY, ALGORITHM=COPY;
+ Only ALGORITHM=COPY supports dropping a
+ primary key without adding a new one in the same
+ ALTER TABLE statement.
+
+ Dropping a primary key and adding another
+
ALTER TABLE tbl_name DROP PRIMARY KEY, ADD PRIMARY KEY (column), ALGORITHM=INPLACE, LOCK=NONE;
+ Data is reorganized substantially, making it an expensive
+ operation.
+
+
+
+
+
+
+
+
Column Operations
+
+
+
+
+ The following table provides an overview of online DDL support
+ for column operations. An asterisk indicates additional
+ information, an exception, or a dependency. For details, see
+ Syntax and Usage Notes.
+
+
+
Table 17.17 Online DDL Support for Column Operations
+
+
+
Operation
+
Instant
+
In Place
+
Rebuilds Table
+
Permits Concurrent DML
+
Only Modifies Metadata
+
+
Adding a column
+
Yes*
+
Yes
+
No*
+
Yes*
+
Yes
+
+
Dropping a column
+
Yes*
+
Yes
+
Yes
+
Yes
+
Yes
+
+
Renaming a column
+
Yes*
+
Yes
+
No
+
Yes*
+
Yes
+
+
Reordering columns
+
No
+
Yes
+
Yes
+
Yes
+
No
+
+
Setting a column default value
+
Yes
+
Yes
+
No
+
Yes
+
Yes
+
+
Changing the column data type
+
No
+
No
+
Yes
+
No
+
No
+
+
Extending VARCHAR column size
+
No
+
Yes
+
No
+
Yes
+
Yes
+
+
Dropping the column default value
+
Yes
+
Yes
+
No
+
Yes
+
Yes
+
+
Changing the auto-increment value
+
No
+
Yes
+
No
+
Yes
+
No*
+
+
Making a column NULL
+
No
+
Yes
+
Yes*
+
Yes
+
No
+
+
Making a column NOT NULL
+
No
+
Yes*
+
Yes*
+
Yes
+
No
+
+
Modifying the definition of an ENUM or
+ SET column
+ALTER TABLE [ IF EXISTS ] [ ONLY ] name [ * ]
+ action [, ... ]
+ALTER TABLE [ IF EXISTS ] [ ONLY ] name [ * ]
+ RENAME [ COLUMN ] column_name TO new_column_name
+ALTER TABLE [ IF EXISTS ] [ ONLY ] name [ * ]
+ RENAME CONSTRAINT constraint_name TO new_constraint_name
+ALTER TABLE [ IF EXISTS ] name
+ RENAME TO new_name
+ALTER TABLE [ IF EXISTS ] name
+ SET SCHEMA new_schema
+ALTER TABLE ALL IN TABLESPACE name [ OWNED BY role_name [, ... ] ]
+ SET TABLESPACE new_tablespace [ NOWAIT ]
+ALTER TABLE [ IF EXISTS ] name
+ ATTACH PARTITION partition_name { FOR VALUES partition_bound_spec | DEFAULT }
+ALTER TABLE [ IF EXISTS ] name
+ DETACH PARTITION partition_name [ CONCURRENTLY | FINALIZE ]
+
+where action is one of:
+
+ ADD [ COLUMN ] [ IF NOT EXISTS ] column_namedata_type [ COLLATE collation ] [ column_constraint [ ... ] ]
+ DROP [ COLUMN ] [ IF EXISTS ] column_name [ RESTRICT | CASCADE ]
+ ALTER [ COLUMN ] column_name [ SET DATA ] TYPE data_type [ COLLATE collation ] [ USING expression ]
+ ALTER [ COLUMN ] column_name SET DEFAULT expression
+ ALTER [ COLUMN ] column_name DROP DEFAULT
+ ALTER [ COLUMN ] column_name { SET | DROP } NOT NULL
+ ALTER [ COLUMN ] column_name SET EXPRESSION AS ( expression )
+ ALTER [ COLUMN ] column_name DROP EXPRESSION [ IF EXISTS ]
+ ALTER [ COLUMN ] column_name ADD GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY [ ( sequence_options ) ]
+ ALTER [ COLUMN ] column_name { SET GENERATED { ALWAYS | BY DEFAULT } | SET sequence_option | RESTART [ [ WITH ] restart ] } [...]
+ ALTER [ COLUMN ] column_name DROP IDENTITY [ IF EXISTS ]
+ ALTER [ COLUMN ] column_name SET STATISTICS { integer | DEFAULT }
+ ALTER [ COLUMN ] column_name SET ( attribute_option = value [, ... ] )
+ ALTER [ COLUMN ] column_name RESET ( attribute_option [, ... ] )
+ ALTER [ COLUMN ] column_name SET STORAGE { PLAIN | EXTERNAL | EXTENDED | MAIN | DEFAULT }
+ ALTER [ COLUMN ] column_name SET COMPRESSION compression_method
+ ADD table_constraint [ NOT VALID ]
+ ADD table_constraint_using_index
+ ALTER CONSTRAINT constraint_name [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] [ ENFORCED | NOT ENFORCED ]
+ ALTER CONSTRAINT constraint_name [ INHERIT | NO INHERIT ]
+ VALIDATE CONSTRAINT constraint_name
+ DROP CONSTRAINT [ IF EXISTS ] constraint_name [ RESTRICT | CASCADE ]
+ DISABLE TRIGGER [ trigger_name | ALL | USER ]
+ ENABLE TRIGGER [ trigger_name | ALL | USER ]
+ ENABLE REPLICA TRIGGER trigger_name
+ ENABLE ALWAYS TRIGGER trigger_name
+ DISABLE RULE rewrite_rule_name
+ ENABLE RULE rewrite_rule_name
+ ENABLE REPLICA RULE rewrite_rule_name
+ ENABLE ALWAYS RULE rewrite_rule_name
+ DISABLE ROW LEVEL SECURITY
+ ENABLE ROW LEVEL SECURITY
+ FORCE ROW LEVEL SECURITY
+ NO FORCE ROW LEVEL SECURITY
+ CLUSTER ON index_name
+ SET WITHOUT CLUSTER
+ SET WITHOUT OIDS
+ SET ACCESS METHOD { new_access_method | DEFAULT }
+ SET TABLESPACE new_tablespace
+ SET { LOGGED | UNLOGGED }
+ SET ( storage_parameter [= value] [, ... ] )
+ RESET ( storage_parameter [, ... ] )
+ INHERIT parent_table
+ NO INHERIT parent_table
+ OF type_name
+ NOT OF
+ OWNER TO { new_owner | CURRENT_ROLE | CURRENT_USER | SESSION_USER }
+ REPLICA IDENTITY { DEFAULT | USING INDEX index_name | FULL | NOTHING }
+
+and partition_bound_spec is:
+
+IN ( partition_bound_expr [, ...] ) |
+FROM ( { partition_bound_expr | MINVALUE | MAXVALUE } [, ...] )
+ TO ( { partition_bound_expr | MINVALUE | MAXVALUE } [, ...] ) |
+WITH ( MODULUS numeric_literal, REMAINDER numeric_literal )
+
+and column_constraint is:
+
+[ CONSTRAINT constraint_name ]
+{ NOT NULL [ NO INHERIT ] |
+ NULL |
+ CHECK ( expression ) [ NO INHERIT ] |
+ DEFAULT default_expr |
+ GENERATED ALWAYS AS ( generation_expr ) [ STORED | VIRTUAL ] |
+ GENERATED { ALWAYS | BY DEFAULT } AS IDENTITY [ ( sequence_options ) ] |
+ UNIQUE [ NULLS [ NOT ] DISTINCT ] index_parameters |
+ PRIMARY KEY index_parameters |
+ REFERENCES reftable [ ( refcolumn ) ] [ MATCH FULL | MATCH PARTIAL | MATCH SIMPLE ]
+ [ ON DELETE referential_action ] [ ON UPDATE referential_action ] }
+[ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] [ ENFORCED | NOT ENFORCED ]
+
+and table_constraint is:
+
+[ CONSTRAINT constraint_name ]
+{ CHECK ( expression ) [ NO INHERIT ] |
+ NOT NULL column_name [ NO INHERIT ] |
+ UNIQUE [ NULLS [ NOT ] DISTINCT ] ( column_name [, ... ] [, column_name WITHOUT OVERLAPS ] ) index_parameters |
+ PRIMARY KEY ( column_name [, ... ] [, column_name WITHOUT OVERLAPS ] ) index_parameters |
+ EXCLUDE [ USING index_method ] ( exclude_element WITH operator [, ... ] ) index_parameters [ WHERE ( predicate ) ] |
+ FOREIGN KEY ( column_name [, ... ] [, PERIOD column_name ] ) REFERENCES reftable [ ( refcolumn [, ... ] [, PERIOD refcolumn ] ) ]
+ [ MATCH FULL | MATCH PARTIAL | MATCH SIMPLE ] [ ON DELETE referential_action ] [ ON UPDATE referential_action ] }
+[ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] [ ENFORCED | NOT ENFORCED ]
+
+and table_constraint_using_index is:
+
+ [ CONSTRAINT constraint_name ]
+ { UNIQUE | PRIMARY KEY } USING INDEX index_name
+ [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ]
+
+index_parameters in UNIQUE, PRIMARY KEY, and EXCLUDE constraints are:
+
+[ INCLUDE ( column_name [, ... ] ) ]
+[ WITH ( storage_parameter [= value] [, ... ] ) ]
+[ USING INDEX TABLESPACE tablespace_name ]
+
+exclude_element in an EXCLUDE constraint is:
+
+{ column_name | ( expression ) } [ COLLATE collation ] [ opclass [ ( opclass_parameter = value [, ... ] ) ] ] [ ASC | DESC ] [ NULLS { FIRST | LAST } ]
+
+referential_action in a FOREIGN KEY/REFERENCES constraint is:
+
+{ NO ACTION | RESTRICT | CASCADE | SET NULL [ ( column_name [, ... ] ) ] | SET DEFAULT [ ( column_name [, ... ] ) ] }
+
+
+
+
Description
+
ALTER TABLE changes the definition of an existing table. There are several subforms described below. Note that the lock level required may differ for each subform. An ACCESS EXCLUSIVE lock is acquired unless explicitly noted. When multiple subcommands are given, the lock acquired will be the strictest one required by any subcommand.
This form adds a new column to the table, using the same syntax as CREATE TABLE. If IF NOT EXISTS is specified and a column already exists with this name, no error is thrown.
This form drops a column from a table. Indexes and table constraints involving the column will be automatically dropped as well. Multivariate statistics referencing the dropped column will also be removed if the removal of the column would cause the statistics to contain data for only a single column. You will need to say CASCADE if anything outside the table depends on the column, for example, foreign key references or views. If IF EXISTS is specified and the column does not exist, no error is thrown. In this case a notice is issued instead.
This form changes the type of a column of a table. Indexes and simple table constraints involving the column will be automatically converted to use the new column type by reparsing the originally supplied expression. The optional COLLATE clause specifies a collation for the new column; if omitted, the collation is the default for the new column type. The optional USING clause specifies how to compute the new column value from the old; if omitted, the default conversion is the same as an assignment cast from old data type to new. A USING clause must be provided if there is no implicit or assignment cast from old to new type.
+
When this form is used, the column's statistics are removed, so running ANALYZE on the table afterwards is recommended. For a virtual generated column, ANALYZE is not necessary because such columns never have statistics.
These forms set or remove the default value for a column (where removal is equivalent to setting the default value to NULL). The new default value will only apply in subsequent INSERT or UPDATE commands; it does not cause rows already in the table to change.
These forms change whether a column is marked to allow null values or to reject null values.
+
SET NOT NULL may only be applied to a column provided none of the records in the table contain a NULL value for the column. Ordinarily this is checked during the ALTER TABLE by scanning the entire table, unless NOT VALID is specified; however, if a valid CHECK constraint exists (and is not dropped in the same command) which proves no NULL can exist, then the table scan is skipped. If a column has an invalid not-null constraint, SET NOT NULL validates it.
SQLite supports a limited subset of ALTER TABLE.
+The ALTER TABLE command in SQLite allows
+these alterations of an existing table:
+it can be renamed;
+a column can be renamed;
+a column can be added to it;
+or
+a column can be dropped from it.
+
+
2. ALTER TABLE RENAME
+
The RENAME TO syntax changes the name of table-name
+to new-table-name.
+This command
+cannot be used to move a table between attached databases, only to rename
+a table within the same database.
+If the table being renamed has triggers or indices, then these remain
+attached to the table after it has been renamed.
+
+Compatibility Note:
+The behavior of ALTER TABLE when renaming a table was enhanced
+in versions 3.25.0 (2018-09-15) and 3.26.0 (2018-12-01)
+in order to carry the rename operation forward into triggers and
+views that reference the renamed table. This is considered an
+improvement. Applications that depend on the older (and
+arguably buggy) behavior can use the
+PRAGMA legacy_alter_table=ON statement or the
+SQLITE_DBCONFIG_LEGACY_ALTER_TABLE configuration parameter
+on sqlite3_db_config() interface to make ALTER TABLE RENAME
+behave as it did prior to version 3.25.0.
+
+
+Beginning with release 3.25.0 (2018-09-15), references to the table
+within trigger bodies and view definitions are also renamed.
+
+
Prior to version 3.26.0 (2018-12-01), FOREIGN KEY references
+to a table that is renamed were only edited with
+PRAGMA foreign_keys=ON, or in other words if
+foreign key constraints were being enforced. With
+PRAGMA foreign_keys=OFF, FOREIGN KEY constraints would not be changed
+when the table that the foreign key referred to (the "parent table")
+was renamed. Beginning with version 3.26.0, FOREIGN KEY constraints
+are always converted when a table is renamed, unless the
+PRAGMA legacy_alter_table=ON setting is engaged. The following
+table summarizes the difference:
The RENAME COLUMN TO syntax changes the
+column-name of table table-name
+into new-column-name. The column name is changed both
+within the table definition itself and also within all indexes, triggers,
+and views that reference the column. If the column name change would
+result in a semantic ambiguity in a trigger or view, then the RENAME
+COLUMN fails with an error and no changes are applied.
+
+
4. ALTER TABLE ADD COLUMN
+
The ADD COLUMN syntax
+is used to add a new column to an existing table.
+The new column is always appended to the end of the list of existing columns.
+The column-def rule defines the characteristics of the new column.
+The new column may take any of the forms permissible in a CREATE TABLE
+statement, with the following restrictions:
+
+
The column may not have a PRIMARY KEY or UNIQUE constraint.
+
The column may not have a default value of CURRENT_TIME, CURRENT_DATE,
+ CURRENT_TIMESTAMP, or an expression in parentheses.
+
If a NOT NULL constraint is specified, then the column must have a
+ default value other than NULL.
+
When adding a column with a CHECK constraint, or a NOT NULL constraint
+on a generated column, the added constraints are tested against all
+preexisting rows in the table and the ADD COLUMN fails
+if any constraint fails. The testing of added constraints
+against preexisting rows is a new enhancement as of SQLite version
+3.37.0 (2021-11-27).
+
The ALTER TABLE command works by modifying the SQL text of the schema
+stored in the sqlite_schema table.
+No changes are made to table content for renames or column addition without
+constraints.
+Because of this,
+the execution time of such ALTER TABLE commands is independent of
+the amount of data in the table and such commands will
+run as quickly on a table with 10 million rows as on a table with 1 row.
+When adding new columns that have CHECK constraints, or adding generated
+columns with NOT NULL constraints, or when deleting columns, then all
+existing data in the table must be either read (to test new constraints
+against existing rows) or written (to remove deleted columns). In those
+cases, the ALTER TABLE command takes time that is proportional to the
+amount of content in the table being altered.
+
diff --git a/packages/junior-evals/src/fixture/web.ts b/packages/junior-evals/src/fixture/web.ts
index 349628daaf..25f8c8cf4b 100644
--- a/packages/junior-evals/src/fixture/web.ts
+++ b/packages/junior-evals/src/fixture/web.ts
@@ -7,7 +7,8 @@
* `mockWebSearchResults()`.
*
* The pages that the `webFetch` tool reads go to Roach, which
- * replays them. See `src/recording-rules.ts`.
+ * replays them. See `src/recording-rules.ts`. A test can answer the pages of
+ * some sites itself with `mockWebSites()`.
*/
import { randomUUID } from "node:crypto";
import { bypass, http, HttpResponse } from "msw";
@@ -15,12 +16,37 @@ import { onTestFinished } from "vitest";
import { USER_AGENT } from "@/chat/tools/web/constants";
import { mswServer } from "@junior-tests/msw/server";
+let mockedSites: Record = {};
+
+/** The host of a URL without `www.`, so both forms are the same site. */
+function siteHost(url: URL): string {
+ return url.hostname.replace(/^www\./, "");
+}
+
+/**
+ * Answer every `webFetch` read on each host with its HTML in this test, for
+ * any path, with or without `www.`. Give each host one real page. Other hosts
+ * still go to Roach.
+ */
+export function mockWebSites(pages: Record): void {
+ mockedSites = pages;
+ onTestFinished(() => {
+ mockedSites = {};
+ });
+}
+
/** Send the pages that `webFetch` reads to Roach. */
export function installWebReplay(): void {
mswServer.use(
http.get("*", ({ request }) => {
// Only `webFetch` sends this user agent.
if (request.headers.get("user-agent") !== USER_AGENT) return undefined;
+ const page = mockedSites[siteHost(new URL(request.url))];
+ if (page !== undefined) {
+ return new Response(page, {
+ headers: { "content-type": "text/html; charset=utf-8" },
+ });
+ }
// `webFetch` uses `node:https`, which does not use the proxy. `fetch`
// uses the proxy dispatcher of the test. `webFetch` follows redirects
// itself, so each redirect is its own request.
@@ -39,16 +65,19 @@ export interface WebSearchResult {
const SEARCH_REQUEST_URL = "https://ai-gateway.vercel.sh/v4/ai/language-model";
const SEARCH_TOOL_ID = "gateway.parallel_search";
-let searchResults: WebSearchResult[] = [];
+type SearchResults = WebSearchResult[] | ((query: string) => WebSearchResult[]);
+
+let searchResults: SearchResults = [];
/**
- * Set the results that the mocked search provider returns in this test, for
- * every query.
+ * Set the results that the mocked search provider returns in this test. A
+ * list answers every query. A function gets the query and returns its
+ * results, so each search can find the pages that it asks for.
*
* The fixture always mocks the search provider. This function does not turn
* the mock on. Without it, `webSearch` finds nothing.
*/
-export function mockWebSearchResults(results: WebSearchResult[]): void {
+export function mockWebSearchResults(results: SearchResults): void {
searchResults = results;
onTestFinished(() => {
searchResults = [];
@@ -66,6 +95,7 @@ export async function answerWebSearch(
return undefined;
}
const payload = (await request.clone().json()) as {
+ prompt?: Array<{ content?: unknown }>;
tools?: Array<{
args?: { maxResults?: number };
id?: string;
@@ -74,6 +104,17 @@ export async function answerWebSearch(
};
const tool = payload.tools?.find((entry) => entry.id === SEARCH_TOOL_ID);
if (!tool) return undefined;
+ // `webSearch` sends the query as the text of the prompt.
+ const query = (payload.prompt ?? [])
+ .flatMap((message) =>
+ Array.isArray(message.content) ? message.content : [],
+ )
+ .flatMap((part: { text?: unknown }) =>
+ typeof part.text === "string" ? [part.text] : [],
+ )
+ .join("\n");
+ const results =
+ typeof searchResults === "function" ? searchResults(query) : searchResults;
const toolCallId = `call_${randomUUID()}`;
const tokens = { total: 0, noCache: 0, cacheRead: 0, cacheWrite: 0 };
return HttpResponse.json({
@@ -90,7 +131,7 @@ export async function answerWebSearch(
toolCallId,
toolName: tool.name,
result: {
- results: searchResults
+ results: results
.slice(0, tool.args?.maxResults)
.map(({ excerpt, title, url }) => ({
excerpts: [excerpt],
diff --git a/packages/junior-evals/src/recording-rules.ts b/packages/junior-evals/src/recording-rules.ts
index e1cc579418..bcc05342b9 100644
--- a/packages/junior-evals/src/recording-rules.ts
+++ b/packages/junior-evals/src/recording-rules.ts
@@ -77,11 +77,6 @@ const ALLOWED_ORIGINS = [
"https://oidc.vercel.com",
// Pages that `webFetch` evals read.
"https://docs.slack.dev",
- // Database docs that the child agents of the delegation eval read.
- "https://dev.mysql.com",
- "https://sqlite.org",
- "https://www.postgresql.org",
- "https://www.sqlite.org",
];
/**