diff --git a/.gitignore b/.gitignore index aa93f49..4e0b6cf 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ node_modules/ dist/ target/ gen/ +.cache/ .impeccable/ .DS_Store *.log diff --git a/CLAUDE.md b/CLAUDE.md index 3ef21a9..65e1e34 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -63,8 +63,10 @@ Desde la raíz, con Node 24 y Rust estable: | `npm run instance-test` | Comprueba que una segunda copia de la app se cierra sola | | `npm run licenses` | Revisa las licencias de todas las dependencias (regla 8) | | `npm run fixtures` | Valida el material de `fixtures/` | +| `npm run eval -- --provider ollama --model gemma4:latest` | Evalúa el análisis contra `fixtures/` y registra el resultado. Ver `fixtures/README.md` | +| `npm run eval:login` | Conecta OpenRouter para la evaluación. La clave queda fuera del repositorio | -La evaluación del análisis aún no existe: llega en M3. Hasta entonces, `npm run fixtures` solo comprueba que el material es coherente. +La evaluación usa la red y modelos reales, así que no está en el CI. Por OpenRouter solo acepta modelos gratuitos. Usa `--cache-only` para volver a puntuar sin gastar ninguna petición. CI en `.github/workflows/ci.yml`: compila, prueba y arranca la app en Windows, macOS y Linux, y ejecuta las pruebas anteriores en cada uno. Un cambio en guardado, Yjs o sync debe pasar `npm run crash-test`. diff --git a/ROADMAP.md b/ROADMAP.md index bf0a51e..14e6919 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -37,6 +37,8 @@ Extracción, resolución de identidades, comparación, cola sin conexión y arn **Salida:** se alcanzan los umbrales de `fixtures/README.md` con al menos dos modelos. +**Cerrado.** `gemma4:latest`, `nemotron-3.5-super` y `gemma4:e2b` alcanzan los umbrales en `la-veda`. En la novela de control, `la-feria`, `gemma4:latest` también y `gemma4:e2b` no. Es el core y el arnés: la app aún no analiza. Resultados, decisiones y límites en `docs/adr/0011-canal-de-analisis.md`. + ## M4 · Fichas y enlaces Fichas de todos los tipos, tarjeta Nombres, fusionar, separar, notas, enlaces en vivo sin IA, navegación bidireccional y `user_overrides`. diff --git a/SPEC.md b/SPEC.md index a6e0f51..89fe1ef 100644 --- a/SPEC.md +++ b/SPEC.md @@ -141,8 +141,8 @@ SQLite local para estructura. Yjs para la prosa. Esquema orientativo: | `scene_snapshots` | id, scene_id, created_at, ydoc, label, word_count | | `entities` | id, project_id, type, canonical_name, status, summary | | `entity_aliases` | entity_id, alias, source | -| `mentions` | id, scene_id, entity_id, anchor | -| `facts` | id, scene_id, entity_id, attribute, value, quote, anchor, story_time | +| `mentions` | id, scene_id, entity_id, quote, anchor | +| `facts` | id, scene_id, entity_id, category, attribute, value, quote, sentence, anchor, story_time, in_dialogue | | `relations` | id, from_entity, to_entity, label, scene_id, anchor | | `user_overrides` | id, kind (merge, split, note, correction, dismiss), payload, created_at | | `issues` | id, kind, category, status, hint_steps, fact_a, fact_b, anchor, resolution | @@ -166,7 +166,7 @@ Reglas: **Pasos:** 1. **Extraer** de la escena entidades, alias, hechos, relaciones y tiempo narrativo, con la cita literal de cada dato. Una cita que no aparezca literalmente en la escena se descarta. 2. **Resolver identidades** contra las entidades existentes, teniendo en cuenta epítetos («la doctora») y el sujeto omitido del español («llegó tarde»). -3. **Comparar** los hechos nuevos con el registro, sin releer la novela. Clasificar en contradicción, evolución o sin relación. +3. **Comparar** los hechos nuevos con el registro, sin releer la novela, en grupos pequeños. Clasificar en contradicción, evolución o sin relación. Cómo se forman los grupos: `docs/adr/0011-canal-de-analisis.md`. 4. **Generar** resúmenes de ficha, pistas graduales y tareas. 5. **Revisar ortotipografía** por escena, incluidas las rayas de diálogo. @@ -177,11 +177,13 @@ Contrato de salida del paso 1 (validar con esquema; si falla, un reintento de re ```json { "entities": [{ "name": "Marta Rey", "type": "person", "aliases": ["la doctora"] }], - "facts": [{ "entity": "Marta Rey", "attribute": "eye_color", "value": "verdes", "quote": "los ojos verdes de su madre", "story_time": null }], + "facts": [{ "entity": "Marta Rey", "category": "physical", "attribute": "eye_color", "value": "verdes", "quote": "los ojos verdes de su madre", "story_time": null }], "relations": [{ "from": "Marta Rey", "to": "Tomás", "label": "hermana de", "quote": "Tomás, su hermano" }] } ``` +`category` es una lista cerrada, en `packages/core/src/analysis/contract.ts`. + La calidad se mide con `fixtures/`. Ningún cambio de prompt o modelo se da por bueno sin ejecutar esa evaluación. ## 6. Conexión de IA diff --git a/biome.json b/biome.json index d0c1980..2dd2bb3 100644 --- a/biome.json +++ b/biome.json @@ -10,7 +10,8 @@ "!**/dist", "!**/target", "!**/gen", - "!**/icons" + "!**/icons", + "!**/.cache" ] }, "formatter": { diff --git a/docs/adr/0011-canal-de-analisis.md b/docs/adr/0011-canal-de-analisis.md new file mode 100644 index 0000000..3c6c2ac --- /dev/null +++ b/docs/adr/0011-canal-de-analisis.md @@ -0,0 +1,140 @@ +# 0011 · Canal de análisis y evaluación + +**Estado:** aceptada. + +**Contexto.** M3 construye el análisis en el core: extraer, resolver identidades, comparar, revisar la ortotipografía, una cola sin conexión y un arnés que lo mida. Es el mayor riesgo del producto: un aviso falso hace perder la confianza del autor y una contradicción perdida quita valor a la herramienta. Se hizo sin gastar, con Ollama y con modelos gratuitos de OpenRouter. + +## Qué se probó antes de decidir + +Cada fila es una ejecución real sobre `la-veda` con un modelo pequeño, `gemma4:e2b`. Las cifras son de la métrica de entonces. + +| Diseño | Detectadas | Falsas alarmas | Qué falló | +|---|---|---|---| +| Primer prompt, sin razonamiento | 2 de 8 | — | Se pierde en la lista | +| Primer prompt, con razonamiento | 5 de 8 | 0 | Fusiona a los dos Luis y resuelve mal las referencias | +| Lista larga de reglas por escena | 4 de 8 | 2 | Más reglas, más falsas alarmas | +| Grupos pequeños por categoría, con su regla | 5 de 8 | 0 | Depende de que el modelo elija bien la categoría | + +Leer cada pregunta y cada respuesta (`--trace`) cambió el diagnóstico. Casi nada de lo que faltaba era incapacidad del modelo: + +- Un hecho correcto se descartaba en silencio si el modelo no listaba su entidad. +- Una respuesta con `"F1, F2"` se tiraba porque el código esperaba una sola etiqueta. +- El modelo parte una frase en varios hechos, y cada grupo veía solo un trozo. +- Entierros y fechas se archivan como `action`, y nunca se comparaban con lo que debían. + +## Decisiones + +### Lo que se puede comprobar leyendo el texto no se le pregunta a un modelo + +Da la misma respuesta siempre, con cualquier modelo y sin coste. Los modelos probados fallan justo aquí, por muy claras que sean las instrucciones. + +| Comprobación | Cómo | +|---|---| +| Citas (regla 3) | Coincidencia literal. Lo que no aparece se descarta y se cuenta | +| Variantes de un nombre | Palabras con mayúscula del texto contra los nombres conocidos, aunque el modelo las haya unificado | +| Personas distintas con un nombre | «Luis, el cartero» y «Luis Ferrer, el alcalde»: un nombre dentro del otro, cada uno con su descripción. Se separan, y sus hechos también | +| Raya de diálogo | Una regla sobre cada línea | +| Hecho sin entidad listada | Si la escena nombra esa entidad, se crea | +| Hecho archivado bajo un nombre inventado | Si su cita nombra a una sola entidad conocida, es de ella | +| Frases sin datos | Se calculan, y solo esas se preguntan en una segunda lectura | + +Separar dos personas que en realidad son una se arregla con un clic en M4. Fusionar dos personas en silencio no se ve. + +### Comparar en grupos que no dependen de la categoría + +Los hechos se comparan en grupos pequeños, sin releer la novela, y un grupo solo se pregunta si tiene algo nuevo: + +1. **Por categoría y persona:** físico, edad, estado civil, familia, trabajo, vida o muerte. +2. **Por categoría entre personas:** fechas, duración de viajes y quién es mayor, que se pueden contar desde cualquiera de los dos lados. +3. **Por palabra compartida en las citas,** sea cual sea la categoría y la persona que eligió el modelo. No cuentan los nombres de personas, las palabras comunes ni las de tiempo. Como mucho, 5 grupos por escena. +4. **Difuntos:** la muerte se reconoce por categoría o por un léxico breve, y se le suma lo que esa persona hace después, aunque esté archivado bajo otra. + +El modelo ve la frase completa de cada hecho y no solo la cita, por lo que cada hecho guarda su frase. Un hallazgo necesita al menos un hecho nuevo, y dos hechos de la misma frase no se contradicen entre sí. + +### Una corrección debe parecer una errata + +Se aplican con un clic (regla 1), así que se comprueba lo que cambia. **En español se compara cómo suena:** la h no suena, b y v son iguales, ll e y también. Por letras, `halla` está igual de cerca de `haya`, que es lo correcto, que de `había`, que cambia el sentido. Por sonido, no. Así se rechazan `Deshizo → Abrió` y `halla → había`, y pasa `Haber si → A ver si`. Se descartan también las que añaden o quitan palabras. En otros idiomas se compara por letras. + +Hacerlo con un prompt no funcionó: pedir que el arreglo no cambie de tiempo verbal no mejoró nada y añadió una sugerencia nueva de reescritura. + +### Razonamiento activo por defecto + +Cuesta tiempo y tokens, pero fue lo que más mejoró al modelo pequeño. `--no-think` permite comparar. + +### Los fallos se clasifican por qué hacer con ellos + +| Fallo | Ejemplos | Qué hace la cola | +|---|---|---| +| `later` | Sin conexión, límite de peticiones, error del servidor | Se detiene y reintenta sola, esperando cada vez más, hasta una hora | +| `refused` | Clave inválida, sin crédito | Se detiene y no reintenta: solo el autor puede arreglarlo. Nunca cambia de proveedor (regla 10) | +| `bad_answer` | Respuesta inservible tras el reintento | Salta esa escena y sigue. No la repite hasta que cambie el texto | + +Si un servidor pide esperar más de un minuto, como con un límite diario, no se espera: la escena sigue en cola. + +### La cola vive en la base + +Una escena está en cola mientras `content_hash` difiera de `analyzed_hash`. Por eso ni el cierre forzado, ni estar sin conexión, ni un límite pueden perder trabajo: solo retrasarlo. + +- `analyzed_hash` describe el texto que se leyó, no el actual. Si el autor escribe mientras el modelo piensa, la escena sigue en cola. +- El resultado de una escena se guarda en una sola transacción. Si falla, siguen los datos anteriores. +- Al abrir un proyecto se recalculan todos los hashes, porque un cierre forzado puede dejar texto guardado con un hash antiguo. El guardado de la prosa no se ha tocado. +- Teclear solo reinicia un temporizador. Se analiza tras una pausa o al cerrar la escena (SPEC §5). +- Una escena vacía no llama a ningún modelo: se borra lo que decía y ya está. +- La papelera oculta los datos de una escena y los avisos que la citan. Al restaurarla vuelven sin reanalizar. + +### Las entidades no se borran al reanalizar + +Una entidad sin escenas que la respalden deja de ofrecerse al modelo, pero se queda. Si su nombre vuelve, recupera su identificador, y es lo que permitirá que `user_overrides` sobreviva en M4 (regla 4). El análisis solo puede añadir alias: nunca renombra una entidad ya guardada. + +### Alcance + +M3 es el core y el arnés. **La app todavía no analiza nada:** el transporte en Rust y los ajustes de conexión son de M6, las fichas y `user_overrides` de M4, y las pistas de M5. Los datos guardan la cita y no el `anchor` de Yjs de SPEC §4: lo añadirá la migración de M4, cuando haya que situarlos en el editor. + +## El arnés + +`npm run eval`. Ver `fixtures/README.md`. + +- **Sin gastar, por construcción.** Por OpenRouter solo acepta modelos que OpenRouter declare gratuitos, se detiene en la primera respuesta con coste y no empieza si el cupo no alcanza. +- **Caché en disco.** El cupo gratuito es de 50 peticiones al día, así que una pregunta sin cambios no se repite. Con `--cache-only` es imposible gastar cupo al volver a puntuar. Se añadió tras gastar peticiones por error al repuntuar. +- **Fuera del CI,** porque depende de la red y de modelos reales. + +## Evaluar sin engañarse + +Tras afinar con `la-veda`, varios ejemplos de los prompts casi copiaban sus frases. Al sustituirlos por otros, `gemma4:e2b` bajó de 7 a 3 de 8 con la misma novela, antes de corregir los fallos de programación que esa bajada ayudó a encontrar. Por eso: + +- **Los prompts no usan frases de las novelas de prueba.** +- **`la-feria` es una novela de control:** mismos tipos de caso, escritos de otra forma, y no se afina mirándola. +- **La métrica se endureció** al ver que premiaba de más. «La doctora» contaba como resuelta aunque se la asignaran a otra persona, y una corrección contaba como detectada con un arreglo equivocado. También se corrigió en sentido contrario: una cita que acababa en punto contaba como fallo estando bien. + +Después de ver `la-feria` se corrigieron dos cosas, ambas de código y no de prompts: los hallazgos sin ningún hecho nuevo, que se vieron en ella, y el filtro de erratas, que se vio en las dos y se comprobó con su `Haber si`. Un retoque del prompt de ortografía se revirtió porque no aportaba nada en `la-veda`. No se tocó el léxico de la muerte, aunque habría recuperado un caso, y por eso esa novela sigue diciendo algo. + +## Resultado + +Con los prompts definitivos. Las ejecuciones están en `fixtures/results/`. + +| Modelo | Vía | `la-veda` | `la-feria` (control) | +|---|---|---|---| +| `gemma4:latest` | Ollama | **Supera:** 6 de 8 | **Supera:** 6 de 8 | +| `nvidia/nemotron-3.5-super-vl:free` | OpenRouter | **Supera:** 7 de 8 | Pendiente, por el cupo diario | +| `gemma4:e2b` | Ollama | **Supera:** 6 de 8 | **No supera:** 3 de 8 y 1 de 3 referencias | + +Salvo donde se indica, todas dan 2 de 3 referencias. En todas: 0 falsas alarmas, 0 citas inventadas y las dos personas con un mismo nombre separadas. + +**Se cumple el criterio de salida de M3:** tres modelos alcanzan los umbrales en `la-veda`. + +Tres matices: + +- **Hay variación entre ejecuciones,** aun con temperatura 0. `gemma4:latest` dio 7 de 8 y 6 de 8 con la misma novela, sin ningún cambio. Ese 6 está justo en el umbral. +- **Un modelo muy pequeño no basta:** `gemma4:e2b` pasa con la novela de trabajo y falla con la de control. La vía local necesita al menos un modelo del tamaño de `gemma4:latest`, de 6,6 GB. +- **Las ausencias se repiten entre modelos:** `o_halla` en `la-veda`, y `c_edad` y `c_difunto` en `la-feria`. No son errores al azar, sino los límites que se describen a continuación. + +## Lo que sigue sin resolver + +Se documenta para no darlo por bueno por haber pasado los umbrales: + +- **Epítetos.** Ningún modelo resuelve de forma fiable «la doctora» o «la maestra». Es el paso 2 de SPEC §5, y el umbral se cumple solo porque admite un fallo de tres. +- **`halla leído`.** Todos los modelos ven el error y proponen `había leído`, que es incorrecto. El filtro impide que llegue al autor, así que la errata se pierde. Es preferible a un arreglo malo a un clic, pero sigue sin detectarse. +- **Avisos espurios y razones absurdas.** Los umbrales no los puntúan. `gemma4:latest` llegó a justificar un conflicto real de edades con que «Marta no puede ser un hermano». M5 debería tratar la razón como una ayuda y no como una afirmación. +- **Casos sin palabra en común.** «Su hermana pequeña» y «le llevaba diez años» dependen de que el modelo elija la categoría. El léxico de la muerte tampoco cubre un entierro contado sin nombrarla. +- **Novelas largas sin medir.** Estas tienen 8 escenas. No se sabe cómo se comportan los grupos por palabra con miles de hechos: el recorte a 12 por grupo podría dejar fuera justo el hecho que importa. Hace falta el fixture largo que pide `fixtures/README.md`, también para medir el inglés. +- **Coste.** Son unas 5,5 peticiones por escena. Con el cupo gratuito de OpenRouter dan para unas 9 escenas al día. Ollama no tiene ese límite. diff --git a/fixtures/README.md b/fixtures/README.md index 8690d4d..fe15b8f 100644 --- a/fixtures/README.md +++ b/fixtures/README.md @@ -1,8 +1,39 @@ # Evaluación del análisis -El mayor riesgo del producto es la calidad de las fichas y de la detección de contradicciones. Esta novela breve permite medirla. +El mayor riesgo del producto es la calidad de las fichas y de la detección de contradicciones. Estas novelas breves permiten medirla. -`node fixtures/validate.mjs` comprueba que el material es coherente: toda cita existe literalmente en su escena. +`npm run fixtures` comprueba que el material es coherente: toda cita existe literalmente en su escena. + +## Cómo se evalúa + +``` +npm run eval -- --provider ollama --model gemma4:latest +npm run eval -- --provider ollama --model gemma4:latest --novel la-feria +npm run eval:login +npm run eval -- --provider openrouter --model nvidia/nemotron-3.5-super-vl:free +``` + +| Opción | Qué hace | +|---|---| +| `--novel` | Qué novela evaluar. Por defecto, `la-veda` | +| `--verbose` | Muestra entidades, hechos y avisos | +| `--trace ` | Guarda cada pregunta y cada respuesta | +| `--cache-only` | Vuelve a puntuar con lo guardado. Si falta una respuesta, falla en vez de preguntar | +| `--no-cache` | Pregunta todo de nuevo | +| `--no-record` | No registra la ejecución | +| `--no-think` | Pide al modelo que no razone antes de responder | +| `--resume` | Continúa aunque el cupo del día no alcance para una ejecución completa | + +**Sin gastar.** Por OpenRouter solo acepta modelos que OpenRouter declare gratuitos, se detiene en la primera respuesta con coste y no empieza si el cupo diario no alcanza. Nunca pide búsqueda web, que se cobra incluso con modelos gratuitos. + +**Registro.** Cada ejecución se guarda en `fixtures/results/`, y al terminar se compara con la anterior del mismo modelo y novela. + +## Las dos novelas + +- **`la-veda`** es la de trabajo: se puede mirar cada fallo y afinar el canal con ella. +- **`la-feria`** es la de control: tiene los mismos tipos de caso, escritos de otra forma. **No se afinan los prompts mirando sus resultados**, porque dejaría de decir nada sobre texto que el canal no ha visto. Si revela un fallo de programación, se puede corregir diciendo que se corrigió. + +Los prompts no deben usar como ejemplo frases de ninguna de las dos. ## Qué cubre `la-veda` @@ -21,9 +52,9 @@ El mayor riesgo del producto es la calidad de las fichas y de la detección de c | Dos personas con el mismo nombre | `n_dos_luis` | | Sujeto omitido, epíteto, pronombre | `must_resolve_reference` | -## Umbrales iniciales +`la-feria` cubre lo mismo con otras formas. Por ejemplo, al difunto se lo llevan «al cementerio» sin que se nombre la muerte, y los dos Ramones no se presentan con la misma construcción. -Son una propuesta que debe calibrarse en M3 con resultados reales. +## Umbrales | Métrica | Umbral | |---|---| @@ -35,4 +66,6 @@ Son una propuesta que debe calibrarse en M3 con resultados reales. Registrar cada ejecución con modelo, vía y fecha. Si un cambio empeora una métrica, no se integra sin justificarlo. -Conviene ampliar el material con un segundo fixture largo, en inglés y con pasado histórico. +Los umbrales no puntúan los avisos que no están ni entre los esperados ni entre los prohibidos, aunque molesten al autor. La evaluación los cuenta como «otros avisos»: hay que mirarlos también. + +Conviene ampliar el material con un fixture largo, en inglés y con pasado histórico. diff --git a/fixtures/la-feria/expected.json b/fixtures/la-feria/expected.json new file mode 100644 index 0000000..9821df9 --- /dev/null +++ b/fixtures/la-feria/expected.json @@ -0,0 +1,88 @@ +{ + "entities": [ + { "id": "adela", "type": "person", "names": ["Adela Montero", "Adela", "la maestra"] }, + { "id": "clara", "type": "person", "names": ["Clara"] }, + { "id": "anselmo", "type": "person", "names": ["don Anselmo"] }, + { "id": "rufino", "type": "person", "names": ["Rufino"], "variants_to_flag": ["Ruffino"] }, + { "id": "pedro", "type": "person", "names": ["Pedro"] }, + { "id": "ramon_herrero", "type": "person", "names": ["Ramón", "el herrero"], "note": "El herrero. No fusionar con el notario." }, + { "id": "ramon_notario", "type": "person", "names": ["Ramón Salas", "el notario"], "note": "El notario. No fusionar con el herrero." }, + { "id": "valdelagua", "type": "place", "names": ["Valdelagua"] }, + { "id": "almaden", "type": "place", "names": ["Almadén"] }, + { "id": "feria", "type": "lore", "names": ["feria de ganado"] } + ], + "relations": [ + { "from": "adela", "to": "clara", "kind": "hermanas", "quote": "Clara, su hermana pequeña", "scene": "s01" }, + { "from": "pedro", "to": "rufino", "kind": "hijo de", "quote": "Pedro, el hijo de Rufino", "scene": "s02" } + ], + "must_resolve_reference": [ + { "scene": "s01", "quote": "Deshizo el baúl antes de cenar", "entity": "adela", "why": "Sujeto omitido" }, + { "scene": "s07", "quote": "la maestra corregía los cuadernos", "entity": "adela", "why": "Epíteto dicho en otra escena" }, + { "scene": "s05", "quote": "Lo encontraron de madrugada", "entity": "rufino", "why": "Pronombre resuelto por la frase anterior" } + ], + "must_flag": [ + { + "id": "c_pierna", "category": "coherence", + "a": { "scene": "s02", "quote": "cojeaba de la pierna derecha" }, + "b": { "scene": "s06", "quote": "arrastrando la pierna izquierda" } + }, + { + "id": "c_feria", "category": "coherence", + "a": { "scene": "s02", "quote": "La feria de ganado es el quince de octubre" }, + "b": { "scene": "s05", "quote": "El tres de octubre se celebró la feria de ganado" } + }, + { + "id": "c_nombre", "category": "coherence", + "a": { "scene": "s02", "quote": "Rufino, el conserje" }, + "b": { "scene": "s04", "quote": "Ruffino" } + }, + { + "id": "c_viaje", "category": "coherence", + "a": { "scene": "s02", "quote": "a tres días de mula de Valdelagua" }, + "b": { "scene": "s07", "quote": "salió de Almadén después de comer y llegó a Valdelagua para la cena" } + }, + { + "id": "c_edad", "category": "coherence", + "a": { "scene": "s01", "quote": "Clara, su hermana pequeña" }, + "b": { "scene": "s06", "quote": "Clara, que le llevaba diez años a Adela" } + }, + { + "id": "c_difunto", "category": "coherence", + "a": { "scene": "s07", "quote": "se llevaron a don Anselmo al cementerio" }, + "b": { "scene": "s08", "quote": "don Anselmo presidió el pleno" } + }, + { + "id": "o_haber", "category": "correction", + "a": { "scene": "s04", "quote": "Haber si aprendéis" }, + "expected_fix": "A ver si aprendéis" + }, + { + "id": "o_hechado", "category": "correction", + "a": { "scene": "s04", "quote": "había hechado de menos" }, + "expected_fix": "había echado de menos" + } + ], + "must_not_flag": [ + { + "id": "n_trabajo", "why": "Evolución: dejó de ser maestra", + "a": { "scene": "s01", "quote": "Era la nueva maestra del pueblo" }, + "b": { "scene": "s08", "quote": "Adela dejó la escuela ese verano" } + }, + { + "id": "n_recuerdo", "why": "Recuerdo: siete años entonces, casi treinta ahora", + "a": { "scene": "s04", "quote": "A los siete años" }, + "b": { "scene": "s01", "quote": "Aún no cumplía los treinta" } + }, + { + "id": "n_mentira", "why": "Un personaje miente en un diálogo: no es un error de la autora", + "a": { "scene": "s03", "quote": "Yo no he tocado una baraja en mi vida" }, + "b": { "scene": "s05", "quote": "Rufino perdió el jornal a las cartas" } + }, + { + "id": "n_dos_ramones", "why": "Dos personas distintas con el mismo nombre", + "namesakes": ["ramon_herrero", "ramon_notario"], + "a": { "scene": "s03", "quote": "el herrero, que se llamaba Ramón" }, + "b": { "scene": "s03", "quote": "Ramón Salas, el notario" } + } + ] +} diff --git a/fixtures/la-feria/manuscript.json b/fixtures/la-feria/manuscript.json new file mode 100644 index 0000000..f5e1144 --- /dev/null +++ b/fixtures/la-feria/manuscript.json @@ -0,0 +1,55 @@ +{ + "title": "La feria", + "language": "es", + "era": "años cincuenta", + "scenes": [ + { + "id": "s01", + "chapter": "Otoño", + "title": "El tren de las nueve", + "text": "Adela Montero llegó a Valdelagua en el tren de las nueve. Era la nueva maestra del pueblo y traía un baúl de libros y una carta de Clara, su hermana pequeña. El alcalde, don Anselmo, la recibió en el andén con el sombrero en la mano.\n\nLe dieron la habitación de encima de la panadería. Deshizo el baúl antes de cenar. Aún no cumplía los treinta, y aquella noche se durmió con la ventana abierta." + }, + { + "id": "s02", + "chapter": "Otoño", + "title": "La escuela", + "text": "La escuela tenía dos aulas y un mapa viejo de la península. Adela cojeaba de la pierna derecha desde niña, y los alumnos tardaron una semana en dejar de mirarla.\n\n—La feria de ganado es el quince de octubre —le avisó Rufino, el conserje—. Ese día no viene nadie a clase.\n\nPedro, el hijo de Rufino, trabajaba en las minas de Almadén, a tres días de mula de Valdelagua por la sierra." + }, + { + "id": "s03", + "chapter": "Otoño", + "title": "Los dos Ramones", + "text": "Esa tarde el herrero, que se llamaba Ramón, le arregló la pata a la pizarra de la escuela y no quiso cobrar. Al salir, Adela se cruzó con Ramón Salas, el notario, que le preguntó cuántos alumnos tenía. No eran parientes, aunque en el pueblo los confundían por carta.\n\nEn la taberna, Rufino juraba a quien quisiera oírlo:\n\n—Yo no he tocado una baraja en mi vida." + }, + { + "id": "s04", + "chapter": "Otoño", + "title": "Soria", + "text": "Adela recordaba bien Soria. A los siete años iba a por agua a la fuente de la plaza, y su madre le hacía repetir las tablas en el patio. Ahora era ella quien las explicaba.\n\n—Haber si aprendéis las tablas antes de Navidad —les dijo a los de segundo.\n\nRuffino, desde la puerta, se reía por lo bajo. Aquella semana había hechado de menos a su hijo más que nunca." + }, + { + "id": "s05", + "chapter": "Otoño", + "title": "El ganado", + "text": "El tres de octubre se celebró la feria de ganado, y la escuela se quedó vacía. Rufino perdió el jornal a las cartas. Lo encontraron de madrugada junto al lavadero, con los bolsillos vacíos." + }, + { + "id": "s06", + "chapter": "Otoño", + "title": "La visita", + "text": "Clara, que le llevaba diez años a Adela, vino a verla por Todos los Santos. La encontró más delgada. Adela bajó a recibirla arrastrando la pierna izquierda, la que le fallaba desde niña.\n\nClara trajo membrillo y noticias de Soria, y se marchó al día siguiente." + }, + { + "id": "s07", + "chapter": "Invierno", + "title": "Febrero", + "text": "Pedro salió de Almadén después de comer y llegó a Valdelagua para la cena, con la mula reventada. Venía a ver a su padre.\n\nAquel febrero se llevaron a don Anselmo al cementerio en un carro tirado por mulas, y cerraron la escuela tres días. Por las noches, la maestra corregía los cuadernos junto a la estufa." + }, + { + "id": "s08", + "chapter": "Primavera", + "title": "Mayo", + "text": "En mayo, don Anselmo presidió el pleno del ayuntamiento y felicitó a Adela por los exámenes.\n\nAdela dejó la escuela ese verano y abrió una imprenta en la plaza. Seguía corrigiendo, decía, pero ahora cobraba por ello." + } + ] +} diff --git a/fixtures/la-veda/expected.json b/fixtures/la-veda/expected.json index d57c16e..93e5e40 100644 --- a/fixtures/la-veda/expected.json +++ b/fixtures/la-veda/expected.json @@ -5,8 +5,8 @@ { "id": "ines", "type": "person", "names": ["Inés Cabrera", "Inés"] }, { "id": "remedios", "type": "person", "names": ["tía Remedios"] }, { "id": "julian", "type": "person", "names": ["Julián Sosa"] }, - { "id": "luis_cartero", "type": "person", "names": ["Luis"], "note": "El cartero. No fusionar con el alcalde." }, - { "id": "luis_alcalde", "type": "person", "names": ["Luis Ferrer"], "note": "El alcalde. No fusionar con el cartero." }, + { "id": "luis_cartero", "type": "person", "names": ["Luis", "el cartero"], "note": "El cartero. No fusionar con el alcalde." }, + { "id": "luis_alcalde", "type": "person", "names": ["Luis Ferrer", "el alcalde"], "note": "El alcalde. No fusionar con el cartero." }, { "id": "puerto", "type": "place", "names": ["Puerto Viejo"] }, { "id": "sal", "type": "place", "names": ["calle de la Sal"], "parent": "puerto" }, { "id": "faro", "type": "place", "names": ["faro de la Punta", "faro"] }, @@ -62,7 +62,7 @@ { "id": "o_raya", "category": "correction", "a": { "scene": "s07", "quote": "- Nunca pensé verte de blanco" }, - "expected_fix": "Usar raya de diálogo" + "expected_fix": "—Nunca pensé verte de blanco" } ], "must_not_flag": [ @@ -83,6 +83,7 @@ }, { "id": "n_dos_luis", "why": "Dos personas distintas con el mismo nombre", + "namesakes": ["luis_cartero", "luis_alcalde"], "a": { "scene": "s03", "quote": "Luis, el cartero" }, "b": { "scene": "s03", "quote": "Luis Ferrer, el alcalde" } } diff --git a/fixtures/results/2026-11-09T09-13-30-793Z-la-veda-ollama-gemma4_latest.json b/fixtures/results/2026-11-09T09-13-30-793Z-la-veda-ollama-gemma4_latest.json new file mode 100644 index 0000000..1288f5e --- /dev/null +++ b/fixtures/results/2026-11-09T09-13-30-793Z-la-veda-ollama-gemma4_latest.json @@ -0,0 +1,38 @@ +{ + "date": "2026-11-09T09:13:30.793Z", + "novel": "la-veda", + "provider": "ollama", + "model": "gemma4:latest", + "reasoning": "default", + "score": { + "mustFlag": { + "found": [ + "c_ojos", + "c_veda", + "c_nombre", + "c_viaje", + "c_difunta", + "o_raya" + ], + "missed": [ + "c_edad_hermanos", + "o_halla" + ] + }, + "falseAlarms": [], + "inventedQuotes": 0, + "discardedQuotes": 0, + "twoLuisMerged": false, + "references": { + "found": 2, + "total": 3, + "missed": [ + "la doctora atendió a Inés Cabrera" + ] + }, + "extraIssues": 1, + "passed": true + }, + "requests": 46, + "seconds": 266 +} diff --git a/fixtures/results/2026-11-09T09-17-16-690Z-la-feria-ollama-gemma4_latest.json b/fixtures/results/2026-11-09T09-17-16-690Z-la-feria-ollama-gemma4_latest.json new file mode 100644 index 0000000..f5f5e06 --- /dev/null +++ b/fixtures/results/2026-11-09T09-17-16-690Z-la-feria-ollama-gemma4_latest.json @@ -0,0 +1,38 @@ +{ + "date": "2026-11-09T09:17:16.690Z", + "novel": "la-feria", + "provider": "ollama", + "model": "gemma4:latest", + "reasoning": "default", + "score": { + "mustFlag": { + "found": [ + "c_pierna", + "c_feria", + "c_nombre", + "c_viaje", + "o_haber", + "o_hechado" + ], + "missed": [ + "c_edad", + "c_difunto" + ] + }, + "falseAlarms": [], + "inventedQuotes": 0, + "discardedQuotes": 0, + "twoLuisMerged": false, + "references": { + "found": 2, + "total": 3, + "missed": [ + "la maestra corregía los cuadernos" + ] + }, + "extraIssues": 0, + "passed": true + }, + "requests": 38, + "seconds": 225 +} diff --git a/fixtures/results/2026-11-09T09-20-26-567Z-la-veda-ollama-gemma4_e2b.json b/fixtures/results/2026-11-09T09-20-26-567Z-la-veda-ollama-gemma4_e2b.json new file mode 100644 index 0000000..081cddd --- /dev/null +++ b/fixtures/results/2026-11-09T09-20-26-567Z-la-veda-ollama-gemma4_e2b.json @@ -0,0 +1,38 @@ +{ + "date": "2026-11-09T09:20:26.567Z", + "novel": "la-veda", + "provider": "ollama", + "model": "gemma4:e2b", + "reasoning": "default", + "score": { + "mustFlag": { + "found": [ + "c_veda", + "c_nombre", + "c_viaje", + "c_edad_hermanos", + "c_difunta", + "o_raya" + ], + "missed": [ + "c_ojos", + "o_halla" + ] + }, + "falseAlarms": [], + "inventedQuotes": 0, + "discardedQuotes": 0, + "twoLuisMerged": false, + "references": { + "found": 2, + "total": 3, + "missed": [ + "la doctora atendió a Inés Cabrera" + ] + }, + "extraIssues": 1, + "passed": true + }, + "requests": 49, + "seconds": 189 +} diff --git a/fixtures/results/2026-11-09T09-30-31-265Z-la-veda-openrouter-nvidia_nemotron-3.5-super-vl_free.json b/fixtures/results/2026-11-09T09-30-31-265Z-la-veda-openrouter-nvidia_nemotron-3.5-super-vl_free.json new file mode 100644 index 0000000..d17f4ea --- /dev/null +++ b/fixtures/results/2026-11-09T09-30-31-265Z-la-veda-openrouter-nvidia_nemotron-3.5-super-vl_free.json @@ -0,0 +1,38 @@ +{ + "date": "2026-11-09T09:30:31.265Z", + "novel": "la-veda", + "provider": "openrouter", + "model": "nvidia/nemotron-3.5-super-vl:free", + "reasoning": "default", + "score": { + "mustFlag": { + "found": [ + "c_ojos", + "c_veda", + "c_nombre", + "c_viaje", + "c_edad_hermanos", + "c_difunta", + "o_raya" + ], + "missed": [ + "o_halla" + ] + }, + "falseAlarms": [], + "inventedQuotes": 0, + "discardedQuotes": 0, + "twoLuisMerged": false, + "references": { + "found": 2, + "total": 3, + "missed": [ + "la doctora atendió a Inés Cabrera" + ] + }, + "extraIssues": 0, + "passed": true + }, + "requests": 45, + "seconds": 1285 +} diff --git a/fixtures/results/2026-11-09T09-32-32-888Z-la-veda-openrouter-nvidia_nemotron-3.5-super-vl_free.json b/fixtures/results/2026-11-09T09-32-32-888Z-la-veda-openrouter-nvidia_nemotron-3.5-super-vl_free.json new file mode 100644 index 0000000..caf549c --- /dev/null +++ b/fixtures/results/2026-11-09T09-32-32-888Z-la-veda-openrouter-nvidia_nemotron-3.5-super-vl_free.json @@ -0,0 +1,38 @@ +{ + "date": "2026-11-09T09:32:32.888Z", + "novel": "la-veda", + "provider": "openrouter", + "model": "nvidia/nemotron-3.5-super-vl:free", + "reasoning": "default", + "score": { + "mustFlag": { + "found": [ + "c_ojos", + "c_veda", + "c_nombre", + "c_viaje", + "c_edad_hermanos", + "c_difunta", + "o_raya" + ], + "missed": [ + "o_halla" + ] + }, + "falseAlarms": [], + "inventedQuotes": 0, + "discardedQuotes": 0, + "twoLuisMerged": false, + "references": { + "found": 2, + "total": 3, + "missed": [ + "la doctora atendió a Inés Cabrera" + ] + }, + "extraIssues": 0, + "passed": true + }, + "requests": 3, + "seconds": 219 +} diff --git a/fixtures/results/2026-11-09T09-34-02-906Z-la-feria-ollama-gemma4_e2b.json b/fixtures/results/2026-11-09T09-34-02-906Z-la-feria-ollama-gemma4_e2b.json new file mode 100644 index 0000000..855c7a0 --- /dev/null +++ b/fixtures/results/2026-11-09T09-34-02-906Z-la-feria-ollama-gemma4_e2b.json @@ -0,0 +1,39 @@ +{ + "date": "2026-11-09T09:34:02.906Z", + "novel": "la-feria", + "provider": "ollama", + "model": "gemma4:e2b", + "reasoning": "default", + "score": { + "mustFlag": { + "found": [ + "c_feria", + "c_nombre", + "o_hechado" + ], + "missed": [ + "c_pierna", + "c_viaje", + "c_edad", + "c_difunto", + "o_haber" + ] + }, + "falseAlarms": [], + "inventedQuotes": 0, + "discardedQuotes": 0, + "twoLuisMerged": false, + "references": { + "found": 1, + "total": 3, + "missed": [ + "la maestra corregía los cuadernos", + "Lo encontraron de madrugada" + ] + }, + "extraIssues": 0, + "passed": false + }, + "requests": 0, + "seconds": 0 +} diff --git a/fixtures/validate.mjs b/fixtures/validate.mjs index b4e0b63..4b486c0 100644 --- a/fixtures/validate.mjs +++ b/fixtures/validate.mjs @@ -1,10 +1,15 @@ -// Comprueba que cada cita de expected.json existe literalmente en su escena. +// Comprueba que cada cita de expected.json existe literalmente en su escena, en cada novela. // Uso: node fixtures/validate.mjs -import { readFileSync } from 'node:fs'; +import { existsSync, readdirSync, readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { fileURLToPath } from 'node:url'; -const dir = join(dirname(fileURLToPath(import.meta.url)), 'la-veda'); +const root = dirname(fileURLToPath(import.meta.url)); +const novels = readdirSync(root).filter((name) => existsSync(join(root, name, 'expected.json'))); +let failed = novels.length === 0; + +for (const novel of novels) { +const dir = join(root, novel); const manuscript = JSON.parse(readFileSync(join(dir, 'manuscript.json'), 'utf8')); const expected = JSON.parse(readFileSync(join(dir, 'expected.json'), 'utf8')); const scenes = new Map(manuscript.scenes.map((s) => [s.id, s.text])); @@ -41,8 +46,16 @@ expected.must_resolve_reference.forEach((r, i) => { const ids = [...expected.must_flag, ...expected.must_not_flag].map((c) => c.id); if (new Set(ids).size !== ids.length) errors.push('Hay identificadores repetidos'); +expected.must_not_flag.forEach((c) => { + (c.namesakes ?? []).forEach((id) => checkEntity(id, c.id)); +}); + if (errors.length) { - console.error(errors.join('\n')); - process.exit(1); + console.error(`${novel}:\n${errors.join('\n')}`); + failed = true; + continue; } -console.log(`OK: ${manuscript.scenes.length} escenas, ${expected.entities.length} entidades, ${expected.must_flag.length} a señalar, ${expected.must_not_flag.length} a no señalar.`); +console.log(`OK ${novel}: ${manuscript.scenes.length} escenas, ${expected.entities.length} entidades, ${expected.must_flag.length} a señalar, ${expected.must_not_flag.length} a no señalar.`); +} + +process.exit(failed ? 1 : 0); diff --git a/package-lock.json b/package-lock.json index 1ecebe8..613d75b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,6 +16,7 @@ "@biomejs/biome": "2.6.6", "@types/node": "^24", "jsdom": "26.1.0", + "tsx": "4.23.15", "typescript": "7.0.2", "vitest": "5.1.0" }, @@ -429,6 +430,448 @@ "node": ">=18" } }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", + "integrity": "sha512-XExcO+dvLKvVtNTibSTBej1NCAbaGhWn9Ww1ZPx80qsahhPFe/8jgWP0IchNe0F3HwkU7n8ejhH8bjonqht8mQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.2.tgz", + "integrity": "sha512-kXXoiPVVGQcnIYGOeaovwOURpniDBpSq4A03qkQ+BMQqtGG6HYap3xne9C1O1yo4TR3qxlCX5IqqmX6fFo2Lqg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.2.tgz", + "integrity": "sha512-5YfKeeI8qWfBZIX+u2xZC3Zlb3Os/gLS2sbEKM+I4ZOcsWmHS2WLysCcQZDAFRslDUU5Oiq44gf6PYN1vGwG5A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.2.tgz", + "integrity": "sha512-O387ite7SzUyCcy3JQX4P4bLtEA7bLLkx+esve5JHnyYfNTxcVpXZo9jhdB0lTKN44gztELTdU7nS8Nr16Fs1Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.2.tgz", + "integrity": "sha512-n4KqkOQrraxHJcgjM1RvwbigfQKIKJVpM7xp+KsxiyUSrRdIXnt73VhrPAx0fV44hgfmIVKjxMN9J1t5jySVkw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.2.tgz", + "integrity": "sha512-uq6suIWYP37qzGddBKPw5QEQPi6HiLGsO7UmkpfyaYNQ3D+rN6w6WfwH+nuqcGXWvawGwxOEroO4YGnFh95azw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.2.tgz", + "integrity": "sha512-n+I0BTSRIoy+d6RPKnEVwql5UwBJolytvY4mAOIEJorKlqgPII8ix6slVVrfZ5Tnj7glIZvloylbB/EJPMWEXw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.2.tgz", + "integrity": "sha512-78XJTJkvPs0kz2w61301PJjXl4g7q3JqiYMZ/M/yVI73EHBrCRTgkhu9oqG7vPqq+a/yadEW8aD+agKlk5xrmg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.2.tgz", + "integrity": "sha512-XlDnu2q5yoqems+xay6wSAcg9DDD7K9RLKZEBOMZm3ckNpJBvOX20tSfby8KfrrhINDyv9V2YVZKY/SpoGJI8w==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.2.tgz", + "integrity": "sha512-pW4AC0P3it8c7do9MVM4p51FzHzdM/TZrerurgRcHJ2WTa1VQ1CIq18xncfpBJw4ojkiZZrKW2yIBWBP92j6Ug==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.2.tgz", + "integrity": "sha512-CYbnj78HsIeA+DhgUKgFCfvNsTHFhMMrinUrMZpDXJXKN8T3XViTZ/+wtHeVxEWY8ewSzTFN+nRmSwO2tZaLUQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.2.tgz", + "integrity": "sha512-buwkd8nsph4R+ajRvw0qM5Hja/TXQow3ptzWO2EbG/cqcIkHloRrdlBtQlshyYGTNFvfkfJ5tpPLVkY4DtsPfQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.2.tgz", + "integrity": "sha512-ZVykbDyk7519VwiNb9Lcj9m8XM6v5V9uKPvrEMkkEedVewf+0itkhahp4HDpgERXhwLRpWFypsGbG/J8s0QjJA==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.2.tgz", + "integrity": "sha512-CAXl+Dtd9UUuJd8pKKdwh6MLm3MUMiqMPmhZ3tTSXPqfyQ3vDl6R5hZdZ/kYojK4ofXtdfSv1tFq8XzWx3heNQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.2.tgz", + "integrity": "sha512-GeXCej4IQtU1B+QlDV8W/RRvbzI3O/Stss+/bCXv4lZls5WGRtu2a+3JkA3i4qIUlMXpcHebWpF8AkJhATowuA==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.2.tgz", + "integrity": "sha512-3H1weTYZPxt/WOhByszQZybS9w5lKzUn1FDMsgEChbHWQwHYQQRfBxgCcZvPhjHfKyJjIievvMmEUawJrdY9Dg==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.2.tgz", + "integrity": "sha512-4xTZr1FUmSoQW4XIWmit3tzQrUTZM+N3P0XV8xROKYF50XfI7xeO90+1bZvNwxIufQ9hDQVRJH5YhgPVF8A/HQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.2.tgz", + "integrity": "sha512-sSATRjPeDBg3pdgHoQfoYBob11Kk1FGa9lui5RIHZCoCkJa9QKlvl3/vKz2usCmYYjs7ymJR/2Nnsqe+Hjt5nw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.2.tgz", + "integrity": "sha512-lqnzCV+mM0gIADaKihiCg6ifgfU2L3h5E33rNQBN1Y4MaVGnzryzmvvf7UHxprpQdE8hpqLolJ9Rl+SkIRDpyw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.2.tgz", + "integrity": "sha512-AL2qJILH7lNjrDmCQDvdxMfAUIv8KMNZOvrwAQ8i8//ntL9FflhOyMJ8OZSMBb8/AWXe3/5v5S20y3zCoZWKoQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.2.tgz", + "integrity": "sha512-QtiuPytchRyC4rwUKhexJdQKvDuZ6hWloi3igqPQNUJCS1/v9EiO3UTOXR6A3FoMo4fnAKbWJdqaIwhOzh8qEw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.2.tgz", + "integrity": "sha512-WkhYDmpTjLvGlScA1rwjRUmhl4k8oXR3cIbtqWmELgU/dFeHHlEllxDvdWcNJV9rbzCexB5vz8gtNewWLgCT7Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.2.tgz", + "integrity": "sha512-GPMSkTOtMnv2U2F8gxe4Io6qmVs+YKyp832Etqqxr0hFngmXQ3rzwytelm3GIn7T4VviRUlf3sOgBOiTdvaf7g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.2.tgz", + "integrity": "sha512-PIhhEkE9uPBleRBrQEJpUn7MBnibZzbGzYWPmY3x+YoVg/95zbjB4CxPPOQ8l5tYYM4mMaCthF8/1DIfBQQyWQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.2.tgz", + "integrity": "sha512-YmJbfTlvU7Sdn9BB+4PRES4oB6pxgS37MAONj+hBr/cpXS1aBPKXxNnDbu+QCWPj0o9dgyxeq79g6c5P8KeuYA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.2.tgz", + "integrity": "sha512-5ebpxr3nWMzrL/rnUI755Jkuee0bHL/Gq0WTF9lvcpv73wAp5eu8MfBUgWK9bhWvZjj7yX8etf/8tI8Ney695g==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, "node_modules/@floating-ui/core": { "version": "1.8.0", "resolved": "https://registry.npmjs.org/@floating-ui/core/-/core-1.8.0.tgz", @@ -1989,6 +2432,48 @@ "dev": true, "license": "MIT" }, + "node_modules/esbuild": { + "version": "0.28.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.2.tgz", + "integrity": "sha512-HKVLS8dvII+xoKW9kmqxbRKrnWEXfJJr/FZhhJmiqIB0e053QNYFqOBouTMO/k5sID4MvCiUCvv8b9M4h32wIA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.2", + "@esbuild/android-arm": "0.28.2", + "@esbuild/android-arm64": "0.28.2", + "@esbuild/android-x64": "0.28.2", + "@esbuild/darwin-arm64": "0.28.2", + "@esbuild/darwin-x64": "0.28.2", + "@esbuild/freebsd-arm64": "0.28.2", + "@esbuild/freebsd-x64": "0.28.2", + "@esbuild/linux-arm": "0.28.2", + "@esbuild/linux-arm64": "0.28.2", + "@esbuild/linux-ia32": "0.28.2", + "@esbuild/linux-loong64": "0.28.2", + "@esbuild/linux-mips64el": "0.28.2", + "@esbuild/linux-ppc64": "0.28.2", + "@esbuild/linux-riscv64": "0.28.2", + "@esbuild/linux-s390x": "0.28.2", + "@esbuild/linux-x64": "0.28.2", + "@esbuild/netbsd-arm64": "0.28.2", + "@esbuild/netbsd-x64": "0.28.2", + "@esbuild/openbsd-arm64": "0.28.2", + "@esbuild/openbsd-x64": "0.28.2", + "@esbuild/openharmony-arm64": "0.28.2", + "@esbuild/sunos-x64": "0.28.2", + "@esbuild/win32-arm64": "0.28.2", + "@esbuild/win32-ia32": "0.28.2", + "@esbuild/win32-x64": "0.28.2" + } + }, "node_modules/estree-walker": { "version": "3.0.3", "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz", @@ -3111,6 +3596,25 @@ "node": ">=18" } }, + "node_modules/tsx": { + "version": "4.23.15", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.15.tgz", + "integrity": "sha512-Yiex1Ovn8z2xPpOWckIiysV1SSyRMY9BkLF++q0yKiDxCqRhosKfMg3janKkiLBwZ5c/YryloKwGZcrEmtwxKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, "node_modules/typescript": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", diff --git a/package.json b/package.json index 1b8f2b4..ed73e0f 100644 --- a/package.json +++ b/package.json @@ -27,12 +27,15 @@ "crash-test": "npm run build:debug && node scripts/crash-test.mjs", "perf-test": "npm run build:debug && node scripts/perf-test.mjs", "instance-test": "npm run build:debug && node scripts/instance-test.mjs", - "close-test": "npm run build:debug && node scripts/close-test.mjs" + "close-test": "npm run build:debug && node scripts/close-test.mjs", + "eval": "tsx packages/core/eval/run.ts", + "eval:login": "tsx packages/core/eval/login.ts" }, "devDependencies": { "@biomejs/biome": "2.6.6", "@types/node": "^24", "jsdom": "26.1.0", + "tsx": "4.23.15", "typescript": "7.0.2", "vitest": "5.1.0" } diff --git a/packages/core/eval/cache.ts b/packages/core/eval/cache.ts new file mode 100644 index 0000000..c93e688 --- /dev/null +++ b/packages/core/eval/cache.ts @@ -0,0 +1,72 @@ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import type { AiProvider, GenerateEvent, GenerateRequest } from "../src/provider/provider"; + +export interface CacheStats { + hits: number; + misses: number; +} + +function keyOf(providerId: string, request: GenerateRequest): string { + const { signal: _signal, ...content } = request; + return createHash("sha256").update(JSON.stringify({ providerId, content })).digest("hex"); +} + +/** + * Remembers completed answers on disk. The free tier allows few requests a day, so an unchanged prompt + * is never asked twice, and changing one prompt only re-runs what depends on it. + */ +export function withCache( + provider: AiProvider, + directory: string, + /** Never asks: for scoring again what's already known, so that it can't use up any allowance. */ + cacheOnly = false, +): AiProvider & { readonly stats: CacheStats } { + const stats: CacheStats = { hits: 0, misses: 0 }; + mkdirSync(directory, { recursive: true }); + return { + id: provider.id, + displayName: provider.displayName, + billing: provider.billing, + listModels: () => provider.listModels(), + stats, + async *generate(request: GenerateRequest): AsyncGenerator { + const file = join(directory, `${keyOf(provider.id, request)}.json`); + let cached: string | null = null; + try { + cached = readFileSync(file, "utf8"); + } catch { + // Not asked yet. + } + if (cached !== null) { + stats.hits += 1; + yield { type: "text", delta: (JSON.parse(cached) as { text: string }).text }; + yield { type: "end", outcome: { status: "completed" } }; + return; + } + stats.misses += 1; + if (cacheOnly) { + yield { + type: "end", + outcome: { + status: "failed", + error: { code: "not_cached", message: "This answer isn't in the cache" }, + }, + }; + return; + } + let text = ""; + for await (const event of provider.generate(request)) { + if (event.type === "text") { + text += event.delta; + } + // Only finished answers: a failure must be tried again, not remembered. + if (event.type === "end" && event.outcome.status === "completed") { + writeFileSync(file, JSON.stringify({ text })); + } + yield event; + } + }, + }; +} diff --git a/packages/core/eval/guard.ts b/packages/core/eval/guard.ts new file mode 100644 index 0000000..e5c2e16 --- /dev/null +++ b/packages/core/eval/guard.ts @@ -0,0 +1,52 @@ +import type { AiProvider, ModelInfo, Usage } from "../src/provider/provider"; + +/** M3 runs without spending. These are what makes that a guarantee and not an intention. */ + +export class SpendingRefused extends Error { + override readonly name = "SpendingRefused"; +} + +/** + * Only a model that OpenRouter itself states is free: named `:free`, with a price of zero. A paid + * model given by mistake stops here, before any request goes out. + */ +export function assertFreeModel(models: readonly ModelInfo[], id: string): void { + const model = models.find((candidate) => candidate.id === id); + if (model === undefined) { + throw new SpendingRefused(`OpenRouter has no model "${id}"`); + } + if (model.free !== true) { + throw new SpendingRefused(`"${id}" isn't free, and this evaluation runs without spending`); + } +} + +/** Stops at the first answer reporting any cost, however small. */ +export function assertNoCost(usage: Usage): void { + if (usage.cost !== null && usage.cost > 0) { + throw new SpendingRefused(`A request reported a cost of ${usage.cost} $: stopping`); + } +} + +export interface DailyAllowance { + readonly used: number; + readonly limit: number; + readonly remaining: number; +} + +/** + * The free tier allows few requests a day. Reading twice, resolving, comparing several groups and + * proofreading come to about this many per scene, measured on `fixtures/`. + */ +export function estimateRequests(uncachedScenes: number): number { + return uncachedScenes * 6; +} + +export function assertAllowance(allowance: DailyAllowance | null, needed: number): void { + if (allowance !== null && allowance.remaining < needed) { + throw new SpendingRefused( + `This run may need ${needed} requests, and ${allowance.remaining} of ${allowance.limit} remain today`, + ); + } +} + +export type { AiProvider }; diff --git a/packages/core/eval/login.ts b/packages/core/eval/login.ts new file mode 100644 index 0000000..55ecedd --- /dev/null +++ b/packages/core/eval/login.ts @@ -0,0 +1,97 @@ +/** + * Gets an OpenRouter key for the evaluation by signing in through the browser, so that nobody has to + * paste one anywhere. It's kept in the user's own folder, never in the repository, and never printed. + * + * npm run eval:login + */ +import { exec } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { homedir } from "node:os"; +import { dirname, join } from "node:path"; + +const KEY_FILE = join( + process.env.LOCALAPPDATA ?? join(homedir(), ".local", "share"), + "ink-writer-dev", + "openrouter-key", +); + +const TIMEOUT_MS = 5 * 60 * 1000; + +const base64url = (bytes: Buffer) => bytes.toString("base64url"); + +const verifier = base64url(randomBytes(64)); +const challenge = base64url(createHash("sha256").update(verifier).digest()); +const state = base64url(randomBytes(32)); + +function openBrowser(url: string): void { + const command = + process.platform === "win32" + ? `start "" "${url}"` + : process.platform === "darwin" + ? `open "${url}"` + : `xdg-open "${url}"`; + exec(command); +} + +/** Lets the program end by itself: a hard exit with a connection open trips an assertion on Windows. */ +function finish(): void { + clearTimeout(timer); + server.close(); + server.closeAllConnections(); +} + +const server = createServer(async (request, response) => { + const url = new URL(request.url ?? "/", "http://localhost"); + if (url.pathname !== "/callback") { + response.writeHead(404).end(); + return; + } + const code = url.searchParams.get("code"); + if (code === null || url.searchParams.get("state") !== state) { + response.writeHead(400).end("Respuesta no válida."); + return; + } + try { + const exchange = await fetch("https://openrouter.ai/api/v1/auth/keys", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ code, code_verifier: verifier, code_challenge_method: "S256" }), + }); + const body = (await exchange.json()) as { key?: string }; + if (!exchange.ok || typeof body.key !== "string") { + throw new Error(`OpenRouter answered ${exchange.status}`); + } + mkdirSync(dirname(KEY_FILE), { recursive: true }); + writeFileSync(KEY_FILE, body.key, { mode: 0o600 }); + response + .writeHead(200, { "content-type": "text/html; charset=utf-8" }) + .end("

Listo. Puedes cerrar esta pestaña.

"); + console.log("Conectado. La clave queda guardada en tu carpeta de usuario."); + } catch (error) { + response.writeHead(500).end("No se pudo completar."); + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } + finish(); +}); + +server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address !== null ? address.port : 0; + const auth = new URL("https://openrouter.ai/auth"); + auth.searchParams.set("callback_url", `http://localhost:${port}/callback`); + auth.searchParams.set("code_challenge", challenge); + auth.searchParams.set("code_challenge_method", "S256"); + auth.searchParams.set("state", state); + auth.searchParams.set("key_label", "Ink Writer evaluation"); + console.log("Se abre el navegador para autorizar a OpenRouter. Esperando..."); + openBrowser(auth.toString()); +}); + +const timer = setTimeout(() => { + console.error("No llegó ninguna respuesta en 5 minutos."); + process.exitCode = 1; + finish(); +}, TIMEOUT_MS); diff --git a/packages/core/eval/node-transport.ts b/packages/core/eval/node-transport.ts new file mode 100644 index 0000000..b9017ea --- /dev/null +++ b/packages/core/eval/node-transport.ts @@ -0,0 +1,32 @@ +import type { AiTransport, ProviderRoute } from "../src/provider/transport"; + +const BASES: Readonly> = { + openrouter: "https://openrouter.ai", + ollama: "http://127.0.0.1:11434", +}; + +/** The network from Node, for the evaluation. In the app, the Rust shell plays this part. */ +export function createNodeTransport(openRouterKey: string | null): AiTransport { + return { + async send({ route, method, path, body, signal }) { + const response = await fetch(`${BASES[route]}${path}`, { + method, + headers: { + ...(body === undefined ? {} : { "content-type": "application/json" }), + ...(route === "openrouter" && openRouterKey !== null + ? { authorization: `Bearer ${openRouterKey}` } + : {}), + }, + ...(body === undefined ? {} : { body: JSON.stringify(body) }), + ...(signal === undefined ? {} : { signal }), + }); + const retryAfter = Number(response.headers.get("retry-after")); + return { + status: response.status, + requestId: response.headers.get("x-generation-id") ?? response.headers.get("x-request-id"), + retryAfterSeconds: Number.isFinite(retryAfter) && retryAfter > 0 ? retryAfter : null, + body: (response.body ?? []) as AsyncIterable, + }; + }, + }; +} diff --git a/packages/core/eval/run.ts b/packages/core/eval/run.ts new file mode 100644 index 0000000..e2fd5e9 --- /dev/null +++ b/packages/core/eval/run.ts @@ -0,0 +1,315 @@ +/** + * Runs the analysis over a test novel and scores it against `fixtures/README.md`. + * + * npm run eval -- --provider ollama --model gemma4:e2b + * npm run eval -- --provider openrouter --model nvidia/nemotron-3.5-super-vl:free + * + * OpenRouter only accepts free models here. Each run is recorded in `fixtures/results/`. + */ +import { + appendFileSync, + existsSync, + mkdirSync, + readdirSync, + readFileSync, + writeFileSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; +import { type AnalysisDeps, analyzeScene, type SceneToAnalyze } from "../src/analysis/pipeline"; +import { emptyState } from "../src/analysis/state"; +import { createOllamaProvider } from "../src/provider/ollama"; +import { createOpenRouterProvider } from "../src/provider/openrouter"; +import type { AiProvider, GenerateEvent, GenerateRequest } from "../src/provider/provider"; +import { StructuredOutputError } from "../src/provider/structured"; +import { readAll } from "../src/provider/transport"; +import { withCache } from "./cache"; +import { + assertAllowance, + assertFreeModel, + assertNoCost, + type DailyAllowance, + estimateRequests, + SpendingRefused, +} from "./guard"; +import { createNodeTransport } from "./node-transport"; +import { type Expected, type Score, scoreAnalysis, THRESHOLDS } from "./score"; + +const ROOT = join(import.meta.dirname, "..", "..", ".."); +const CACHE = join(import.meta.dirname, ".cache"); +const RESULTS = join(ROOT, "fixtures", "results"); + +/** Where `npm run eval:login` keeps the key: in the user's own folder, never in the repository. */ +export const KEY_FILE = join( + process.env.LOCALAPPDATA ?? join(homedir(), ".local", "share"), + "ink-writer-dev", + "openrouter-key", +); + +function option(name: string): string | undefined { + const index = process.argv.indexOf(`--${name}`); + return index >= 0 ? process.argv[index + 1] : undefined; +} + +const flag = (name: string) => process.argv.includes(`--${name}`); + +async function readAllowance( + transport: ReturnType, +): Promise { + const response = await transport.send({ + route: "openrouter", + method: "GET", + path: "/api/v1/key", + }); + const body = JSON.parse(await readAll(response.body)) as { + data?: { free_model_daily_requests?: DailyAllowance }; + }; + return body.data?.free_model_daily_requests ?? null; +} + +/** Writes each question and its answer to a file, to see why a run went as it did. */ +function withTrace(provider: AiProvider, file: string): AiProvider { + writeFileSync(file, ""); + return { + ...provider, + listModels: () => provider.listModels(), + async *generate(request: GenerateRequest): AsyncGenerator { + let text = ""; + for await (const event of provider.generate(request)) { + if (event.type === "text") { + text += event.delta; + } + // Before the end event, as a reader can stop at that one and never come back here. + if (event.type === "end") { + appendFileSync( + file, + `=== ${request.structured?.name ?? "text"}\n${request.input}\n--- answer\n${text}\n\n`, + ); + } + yield event; + } + }, + }; +} + +interface RunRecord { + readonly date: string; + readonly novel: string; + readonly provider: string; + readonly model: string; + readonly reasoning: string; + readonly score: Score; + readonly requests: number; + readonly seconds: number; +} + +function previousRecord(novel: string, provider: string, model: string): RunRecord | null { + if (!existsSync(RESULTS)) { + return null; + } + const files = readdirSync(RESULTS) + .filter((name) => name.endsWith(".json")) + .sort(); + for (const file of files.reverse()) { + const record = JSON.parse(readFileSync(join(RESULTS, file), "utf8")) as RunRecord; + if (record.novel === novel && record.provider === provider && record.model === model) { + return record; + } + } + return null; +} + +function printScore(score: Score, previous: RunRecord | null): void { + const mark = (ok: boolean) => (ok ? "ok " : "NO "); + const { mustFlag, falseAlarms, references } = score; + const row = (ok: boolean, label: string, value: string, before?: string) => + console.log(` ${mark(ok)} ${label}: ${value}${before ? ` (antes: ${before})` : ""}`); + const old = previous?.score; + console.log("\nResultado"); + row( + mustFlag.found.length >= THRESHOLDS.mustFlagAtLeast, + `Contradicciones detectadas (mínimo ${THRESHOLDS.mustFlagAtLeast})`, + `${mustFlag.found.length} de ${mustFlag.found.length + mustFlag.missed.length}`, + old && String(old.mustFlag.found.length), + ); + if (mustFlag.missed.length > 0) console.log(` faltan: ${mustFlag.missed.join(", ")}`); + row( + falseAlarms.length <= THRESHOLDS.falseAlarmsAtMost, + `Falsas alarmas (máximo ${THRESHOLDS.falseAlarmsAtMost})`, + String(falseAlarms.length), + old && String(old.falseAlarms.length), + ); + if (falseAlarms.length > 0) console.log(` ${falseAlarms.join(", ")}`); + row( + score.inventedQuotes === 0, + "Citas inventadas conservadas (máximo 0)", + String(score.inventedQuotes), + ); + row( + !score.twoLuisMerged, + "Dos personas con un mismo nombre, separadas", + score.twoLuisMerged ? "fusionadas" : "sí", + ); + row( + references.found >= THRESHOLDS.referencesAtLeast, + `Referencias resueltas (mínimo ${THRESHOLDS.referencesAtLeast})`, + `${references.found} de ${references.total}`, + old && String(old.references.found), + ); + if (references.missed.length > 0) console.log(` faltan: ${references.missed.join(" | ")}`); + console.log(` · Citas no literales descartadas: ${score.discardedQuotes}`); + console.log(` · Otros avisos, no esperados ni prohibidos: ${score.extraIssues}`); + console.log(score.passed ? "\nSUPERA los umbrales." : "\nNo supera los umbrales."); +} + +async function main(): Promise { + const providerName = option("provider"); + const model = option("model"); + if ((providerName !== "ollama" && providerName !== "openrouter") || model === undefined) { + console.error( + "Uso: npm run eval -- --provider ollama|openrouter --model [--novel ] [--no-think] [--verbose] [--cache-only] [--trace ]", + ); + process.exit(2); + } + const verbose = flag("verbose"); + + const novel = option("novel") ?? "la-veda"; + const dir = join(ROOT, "fixtures", novel); + const manuscript = JSON.parse(readFileSync(join(dir, "manuscript.json"), "utf8")) as { + language: string; + scenes: SceneToAnalyze[]; + }; + const expected = JSON.parse(readFileSync(join(dir, "expected.json"), "utf8")) as Expected; + + const key = + providerName === "openrouter" && existsSync(KEY_FILE) + ? readFileSync(KEY_FILE, "utf8").trim() + : null; + if (providerName === "openrouter" && key === null) { + console.error("No hay clave de OpenRouter. Ejecuta: npm run eval:login"); + process.exit(2); + } + const transport = createNodeTransport(key); + const base: AiProvider = + providerName === "ollama" + ? createOllamaProvider(transport) + : createOpenRouterProvider(transport); + + if (providerName === "openrouter") { + assertFreeModel(await base.listModels(), model); + const allowance = await readAllowance(transport); + if (allowance !== null) { + console.log(`Peticiones gratuitas hoy: ${allowance.used} de ${allowance.limit}`); + } + if (!flag("resume") && !flag("cache-only")) { + assertAllowance(allowance, estimateRequests(manuscript.scenes.length)); + } + } + + const cached = flag("no-cache") ? base : withCache(base, CACHE, flag("cache-only")); + const traceFile = option("trace"); + const provider = traceFile === undefined ? cached : withTrace(cached, traceFile); + const reasoning = flag("no-think") ? "off" : "default"; + let requests = 0; + const deps: AnalysisDeps = { + provider, + model, + language: manuscript.language, + reasoning, + onStep: () => { + requests += 1; + }, + onUsage: assertNoCost, + // Numbered, so that the same run makes the same prompts, and the cache can find them. + newId: (() => { + let next = 0; + return () => { + next += 1; + return `e${next}`; + }; + })(), + }; + + const started = performance.now(); + let state = emptyState(manuscript.scenes.map((scene) => scene.id)); + let discarded = 0; + let unplaced = 0; + for (const scene of manuscript.scenes) { + const stepStart = performance.now(); + const result = await analyzeScene(state, scene, deps); + state = result.state; + discarded += result.discardedQuotes.length; + console.log( + `${scene.id}: ${state.facts.filter((fact) => fact.sceneId === scene.id).length} hechos, ${((performance.now() - stepStart) / 1000).toFixed(1)} s`, + ); + unplaced += result.unplacedFacts.length; + for (const quote of result.discardedQuotes) { + if (verbose) console.log(` descartada: «${quote}»`); + } + for (const quote of result.unplacedFacts) { + if (verbose) console.log(` sin dueño: «${quote}»`); + } + } + + if (verbose) { + console.log("\nEntidades"); + for (const entity of state.entities) { + console.log(` ${entity.canonicalName} [${entity.type}] ${entity.aliases.join(", ")}`); + } + console.log("\nHechos"); + for (const fact of state.facts) { + const name = state.entities.find((entity) => entity.id === fact.entityId)?.canonicalName; + console.log( + ` ${fact.sceneId} ${name} [${fact.category}] ${fact.attribute}: ${fact.value} | «${fact.quote}»`, + ); + } + console.log("\nAvisos"); + for (const issue of state.issues) { + const b = "b" in issue ? ` <> ${issue.b.sceneId} «${issue.b.quote}»` : ""; + console.log(` ${issue.kind}: ${issue.a.sceneId} «${issue.a.quote}»${b} — ${issue.reason}`); + } + } + + const texts = new Map(manuscript.scenes.map((scene) => [scene.id, scene.text])); + const score = scoreAnalysis(state, expected, texts, discarded); + const previous = previousRecord(novel, providerName, model); + printScore(score, previous); + console.log(` Hechos descartados por no poder asignarse a nadie: ${unplaced}`); + + const seconds = Math.round((performance.now() - started) / 1000); + const stats = + "stats" in cached ? (cached as { stats: { hits: number; misses: number } }).stats : null; + console.log( + `\n${requests} pasos, ${stats ? `${stats.misses} peticiones reales y ${stats.hits} de la caché` : "sin caché"}, ${seconds} s`, + ); + + if (!flag("no-record")) { + const date = new Date().toISOString(); + const record: RunRecord = { + date, + novel, + provider: providerName, + model, + reasoning, + score, + requests: stats?.misses ?? requests, + seconds, + }; + mkdirSync(RESULTS, { recursive: true }); + const name = `${date.replace(/[:.]/g, "-")}-${novel}-${providerName}-${model.replace(/[^a-zA-Z0-9.-]/g, "_")}.json`; + writeFileSync(join(RESULTS, name), `${JSON.stringify(record, null, 2)}\n`); + } + process.exitCode = score.passed ? 0 : 1; +} + +main().catch((error: unknown) => { + if (error instanceof SpendingRefused) { + console.error(`Detenido para no gastar: ${error.message}`); + } else if (error instanceof StructuredOutputError) { + // A daily limit or a question not in the cache. What was answered stays cached for next time. + console.error(`Detenido: ${error.message}`); + } else { + console.error(error); + } + process.exitCode = 3; +}); diff --git a/packages/core/eval/score.ts b/packages/core/eval/score.ts new file mode 100644 index 0000000..dadaedf --- /dev/null +++ b/packages/core/eval/score.ts @@ -0,0 +1,228 @@ +import { isEpithet } from "../src/analysis/names"; +import type { AnalysisState, QuoteRef } from "../src/analysis/state"; +import { sentenceAround } from "../src/analysis/topics"; + +/** + * Scores an analysis against `fixtures//expected.json`, with the thresholds in + * `fixtures/README.md`. Pure, and independent of any model, so that it can be tested by itself. + */ + +interface ExpectedRef { + readonly scene: string; + readonly quote: string; +} + +export interface Expected { + readonly entities: readonly { + readonly id: string; + readonly names: readonly string[]; + }[]; + readonly must_resolve_reference: readonly { + readonly scene: string; + readonly quote: string; + readonly entity: string; + }[]; + readonly must_flag: readonly { + readonly id: string; + readonly category: string; + readonly a: ExpectedRef; + readonly b?: ExpectedRef; + /** What a correction must say: it's applied with one click, so a wrong one doesn't count. */ + readonly expected_fix?: string; + }[]; + readonly must_not_flag: readonly { + readonly id: string; + readonly a: ExpectedRef; + readonly b?: ExpectedRef; + /** Two people with one name, by their entity ids, that must stay two. */ + readonly namesakes?: readonly [string, string]; + }[]; +} + +export const THRESHOLDS = { + mustFlagAtLeast: 6, + falseAlarmsAtMost: 1, + inventedQuotesAtMost: 0, + referencesAtLeast: 2, +} as const; + +export interface Score { + readonly mustFlag: { readonly found: readonly string[]; readonly missed: readonly string[] }; + readonly falseAlarms: readonly string[]; + readonly inventedQuotes: number; + readonly discardedQuotes: number; + readonly twoLuisMerged: boolean; + readonly references: { + readonly found: number; + readonly total: number; + readonly missed: readonly string[]; + }; + readonly extraIssues: number; + readonly passed: boolean; +} + +/** + * Whether a model's quote points at the expected place: same scene, and one inside the other, or both + * in one sentence. A model can pick another part of the same sentence, and the author still gets taken + * to the right place. It's as lenient for false alarms, so it can't only help a score. + */ +export function sameQuote( + actual: QuoteRef, + expected: ExpectedRef, + texts?: ReadonlyMap, +): boolean { + if (actual.sceneId !== expected.scene || actual.quote === "") { + return false; + } + if (actual.quote.includes(expected.quote) || expected.quote.includes(actual.quote)) { + return true; + } + const text = texts?.get(actual.sceneId); + if (text === undefined) { + return false; + } + return sentenceAround(text, expected.quote) === sentenceAround(text, actual.quote); +} + +function issueMatches( + issue: AnalysisState["issues"][number], + expected: { readonly a: ExpectedRef; readonly b?: ExpectedRef }, + texts: ReadonlyMap, +): boolean { + if (expected.b === undefined) { + return sameQuote(issue.a, expected.a, texts); + } + if (!("b" in issue)) { + return false; + } + return ( + (sameQuote(issue.a, expected.a, texts) && sameQuote(issue.b, expected.b, texts)) || + (sameQuote(issue.a, expected.b, texts) && sameQuote(issue.b, expected.a, texts)) + ); +} + +const lower = (text: string) => text.toLocaleLowerCase(); + +/** The entity an analysis gave to a name: exact match first, since "Luis" is also in "Luis Ferrer". */ +function entityForName(state: AnalysisState, name: string): string | undefined { + return state.entities.find((entity) => + [entity.canonicalName, ...entity.aliases].some((other) => lower(other) === lower(name)), + )?.id; +} + +/** + * Whether two people with one name ended up as one. Names alone can't tell: a model can attach + * "Luis, el cartero" to Luis Ferrer without ever listing "Luis" as a name. So it also looks at who + * each person's own quote was attributed to. + */ +export function mergedPeople(state: AnalysisState, expected: Expected): boolean { + const pair = expected.must_not_flag.find((item) => item.namesakes !== undefined); + if (pair?.b === undefined || pair.namesakes === undefined) { + return false; + } + const { a, b } = { a: pair.a, b: pair.b }; + const attributedTo = (ref: ExpectedRef) => + new Set( + state.facts + .filter((fact) => sameQuote({ sceneId: fact.sceneId, quote: fact.quote }, ref)) + .map((fact) => fact.entityId), + ); + const first = attributedTo(a); + if ([...attributedTo(b)].some((id) => first.has(id))) { + return true; + } + const idsOf = (id: string) => + new Set( + (expected.entities.find((entity) => entity.id === id)?.names ?? []) + .map((name) => entityForName(state, name)) + .filter((found) => found !== undefined), + ); + const other = idsOf(pair.namesakes[1]); + return [...idsOf(pair.namesakes[0])].some((id) => other.has(id)); +} + +export function scoreAnalysis( + state: AnalysisState, + expected: Expected, + texts: ReadonlyMap, + discardedQuotes: number, +): Score { + const coherenceOrCorrection = expected.must_flag; + const found = coherenceOrCorrection + .filter((item) => + state.issues.some((issue) => + item.category === "correction" + ? issue.kind === "correction" && + issueMatches(issue, item, texts) && + issue.fix.includes(item.expected_fix ?? "") + : issue.kind !== "correction" && issueMatches(issue, item, texts), + ), + ) + .map((item) => item.id); + const missed = coherenceOrCorrection.map((item) => item.id).filter((id) => !found.includes(id)); + + const falseAlarms = expected.must_not_flag + .filter((item) => + state.issues.some((issue) => issue.kind !== "correction" && issueMatches(issue, item, texts)), + ) + .map((item) => item.id); + + // Everything kept must be literally in its scene. The pipeline drops the rest, and this checks it. + const cited: QuoteRef[] = [ + ...state.facts.map((fact) => ({ sceneId: fact.sceneId, quote: fact.quote })), + ...state.relations.map((relation) => ({ sceneId: relation.sceneId, quote: relation.quote })), + ...state.issues.flatMap((issue) => ("b" in issue ? [issue.a, issue.b] : [issue.a])), + ]; + const inventedQuotes = cited.filter( + (ref) => !(texts.get(ref.sceneId) ?? "").includes(ref.quote), + ).length; + + const twoLuisMerged = mergedPeople(state, expected); + + const entityNames = new Map(expected.entities.map((entity) => [entity.id, entity.names])); + const resolvedTo = (reference: Expected["must_resolve_reference"][number]) => { + const names = entityNames.get(reference.entity) ?? []; + const ref = { scene: reference.scene, quote: reference.quote }; + // A pronoun is resolved if the fact is about that person, or if the fact's own value names them: + // "Inés lo vio" as "Inés: vio a Tomás" has understood who "lo" is. Only by a proper name, though: + // repeating "la doctora" from the text doesn't show who she was taken to be. + // For the same reason, it's the proper names that say which entity is that person: if "la doctora" + // was given to someone else, that someone else is not who was expected. + const properNames = names.filter((name) => !isEpithet(name)); + const ids = new Set(properNames.map((name) => entityForName(state, name))); + return state.facts.some( + (fact) => + sameQuote({ sceneId: fact.sceneId, quote: fact.quote }, ref, texts) && + (ids.has(fact.entityId) || properNames.some((name) => fact.value.includes(name))), + ); + }; + const references = expected.must_resolve_reference; + const resolved = references.filter(resolvedTo); + + const matchedIssue = (issue: AnalysisState["issues"][number]) => + [...expected.must_flag, ...expected.must_not_flag].some((item) => + issueMatches(issue, item, texts), + ); + + const passed = + found.length >= THRESHOLDS.mustFlagAtLeast && + falseAlarms.length <= THRESHOLDS.falseAlarmsAtMost && + inventedQuotes <= THRESHOLDS.inventedQuotesAtMost && + !twoLuisMerged && + resolved.length >= THRESHOLDS.referencesAtLeast; + + return { + mustFlag: { found, missed }, + falseAlarms, + inventedQuotes, + discardedQuotes, + twoLuisMerged, + references: { + found: resolved.length, + total: references.length, + missed: references.filter((ref) => !resolved.includes(ref)).map((ref) => ref.quote), + }, + extraIssues: state.issues.filter((issue) => !matchedIssue(issue)).length, + passed, + }; +} diff --git a/packages/core/src/analysis/contract.ts b/packages/core/src/analysis/contract.ts new file mode 100644 index 0000000..d6f9f4f --- /dev/null +++ b/packages/core/src/analysis/contract.ts @@ -0,0 +1,340 @@ +import type { Validated } from "../provider/structured"; + +/** What each step asks the model for, as schemas for the provider and validators for the answer. */ + +export type EntityType = "person" | "place" | "lore"; + +export interface ExtractedEntity { + readonly name: string; + readonly type: EntityType; + readonly aliases: readonly string[]; +} + +/** + * What a fact is about, from a closed list. The comparison groups by this, and each group is judged by + * its own rule, so that a small model only ever has to weigh a few facts of one kind. + */ +export const FACT_CATEGORIES = [ + "physical", + "age", + "relative_age", + "marital_status", + "family", + "job", + "alive_or_dead", + "date_of_event", + "travel_time", + "location", + "action", + "possession", + "thought", + "other", +] as const; + +export type FactCategory = (typeof FACT_CATEGORIES)[number]; + +export interface ExtractedFact { + readonly entity: string; + readonly category: FactCategory; + readonly attribute: string; + readonly value: string; + readonly quote: string; + readonly story_time: string | null; +} + +export interface ExtractedRelation { + readonly from: string; + readonly to: string; + readonly label: string; + readonly quote: string; +} + +/** SPEC §5, step 1. */ +export interface Extraction { + readonly entities: readonly ExtractedEntity[]; + readonly facts: readonly ExtractedFact[]; + readonly relations: readonly ExtractedRelation[]; +} + +const STRING = { type: "string" } as const; + +export const EXTRACTION_SCHEMA = { + type: "object", + additionalProperties: false, + required: ["entities", "facts", "relations"], + properties: { + entities: { + type: "array", + items: { + type: "object", + additionalProperties: false, + required: ["name", "type", "aliases"], + properties: { + name: STRING, + type: { type: "string", enum: ["person", "place", "lore"] }, + aliases: { type: "array", items: STRING }, + }, + }, + }, + facts: { + type: "array", + items: { + type: "object", + additionalProperties: false, + required: ["entity", "category", "attribute", "value", "quote", "story_time"], + properties: { + entity: STRING, + category: { type: "string", enum: FACT_CATEGORIES }, + attribute: STRING, + value: STRING, + quote: STRING, + story_time: { type: ["string", "null"] }, + }, + }, + }, + relations: { + type: "array", + items: { + type: "object", + additionalProperties: false, + required: ["from", "to", "label", "quote"], + properties: { from: STRING, to: STRING, label: STRING, quote: STRING }, + }, + }, + }, +} as const; + +export interface IdentityAnswer { + readonly name: string; + /** The known entity this name refers to, or null if it's someone or something new. */ + readonly entity_id: string | null; +} + +/** SPEC §5, step 2. */ +export const IDENTITY_SCHEMA = { + type: "object", + additionalProperties: false, + required: ["answers"], + properties: { + answers: { + type: "array", + items: { + type: "object", + additionalProperties: false, + required: ["name", "entity_id"], + properties: { name: STRING, entity_id: { type: ["string", "null"] } }, + }, + }, + }, +} as const; + +export type Verdict = "contradiction" | "evolution"; + +export interface Finding { + readonly new_fact: string; + readonly other_fact: string; + readonly verdict: Verdict; + readonly reason: string; +} + +/** SPEC §5, step 3. Facts with no relation simply aren't listed. */ +export const COMPARISON_SCHEMA = { + type: "object", + additionalProperties: false, + required: ["findings"], + properties: { + findings: { + type: "array", + items: { + type: "object", + additionalProperties: false, + required: ["new_fact", "other_fact", "verdict", "reason"], + properties: { + new_fact: STRING, + other_fact: STRING, + verdict: { type: "string", enum: ["contradiction", "evolution"] }, + reason: STRING, + }, + }, + }, + }, +} as const; + +export interface Correction { + readonly quote: string; + readonly fix: string; + readonly reason: string; +} + +/** SPEC §5, step 5. */ +export const PROOFREAD_SCHEMA = { + type: "object", + additionalProperties: false, + required: ["corrections"], + properties: { + corrections: { + type: "array", + items: { + type: "object", + additionalProperties: false, + required: ["quote", "fix", "reason"], + properties: { quote: STRING, fix: STRING, reason: STRING }, + }, + }, + }, +} as const; + +type Json = Record; + +const isObject = (value: unknown): value is Json => typeof value === "object" && value !== null; + +function fail(problem: string): Validated { + return { ok: false, problem }; +} + +function strings(entry: Json, fields: readonly string[]): string | null { + for (const field of fields) { + if (typeof entry[field] !== "string") { + return `"${field}" must be text`; + } + } + return null; +} + +function listOf(value: unknown, field: string): unknown[] | null { + if (!isObject(value) || !Array.isArray(value[field])) { + return null; + } + return value[field]; +} + +export function validateExtraction(value: unknown): Validated { + const rawEntities = listOf(value, "entities"); + const rawFacts = listOf(value, "facts"); + const rawRelations = listOf(value, "relations"); + if (rawEntities === null || rawFacts === null || rawRelations === null) { + return fail('It must have the lists "entities", "facts" and "relations".'); + } + const entities: ExtractedEntity[] = []; + for (const entry of rawEntities) { + if (!isObject(entry)) { + return fail("Each entity must be an object."); + } + const problem = strings(entry, ["name"]); + if (problem !== null) { + return fail(`In an entity, ${problem}.`); + } + const aliases = Array.isArray(entry.aliases) + ? entry.aliases.filter((alias): alias is string => typeof alias === "string") + : []; + const type: EntityType = + entry.type === "person" || entry.type === "place" ? entry.type : "lore"; + entities.push({ name: entry.name as string, type, aliases }); + } + const facts: ExtractedFact[] = []; + for (const entry of rawFacts) { + if (!isObject(entry)) { + return fail("Each fact must be an object."); + } + const problem = strings(entry, ["entity", "attribute", "value", "quote"]); + if (problem !== null) { + return fail(`In a fact, ${problem}.`); + } + facts.push({ + entity: entry.entity as string, + // An unknown category isn't worth rejecting a whole answer: it's just never compared. + category: FACT_CATEGORIES.includes(entry.category as FactCategory) + ? (entry.category as FactCategory) + : "other", + attribute: entry.attribute as string, + value: entry.value as string, + quote: entry.quote as string, + story_time: typeof entry.story_time === "string" ? entry.story_time : null, + }); + } + const relations: ExtractedRelation[] = []; + for (const entry of rawRelations) { + if (!isObject(entry)) { + return fail("Each relation must be an object."); + } + const problem = strings(entry, ["from", "to", "label", "quote"]); + if (problem !== null) { + return fail(`In a relation, ${problem}.`); + } + relations.push({ + from: entry.from as string, + to: entry.to as string, + label: entry.label as string, + quote: entry.quote as string, + }); + } + return { ok: true, value: { entities, facts, relations } }; +} + +export function validateIdentities(value: unknown): Validated { + const raw = listOf(value, "answers"); + if (raw === null) { + return fail('It must have the list "answers".'); + } + const answers: IdentityAnswer[] = []; + for (const entry of raw) { + if (!isObject(entry) || typeof entry.name !== "string") { + return fail('Each answer must have a "name".'); + } + answers.push({ + name: entry.name, + entity_id: typeof entry.entity_id === "string" ? entry.entity_id : null, + }); + } + return { ok: true, value: answers }; +} + +export function validateFindings(value: unknown): Validated { + const raw = listOf(value, "findings"); + if (raw === null) { + return fail('It must have the list "findings".'); + } + const findings: Finding[] = []; + for (const entry of raw) { + if (!isObject(entry)) { + return fail("Each finding must be an object."); + } + const problem = strings(entry, ["new_fact", "other_fact", "reason"]); + if (problem !== null) { + return fail(`In a finding, ${problem}.`); + } + if (entry.verdict !== "contradiction" && entry.verdict !== "evolution") { + return fail('"verdict" must be "contradiction" or "evolution".'); + } + findings.push({ + new_fact: entry.new_fact as string, + other_fact: entry.other_fact as string, + verdict: entry.verdict, + reason: entry.reason as string, + }); + } + return { ok: true, value: findings }; +} + +export function validateCorrections(value: unknown): Validated { + const raw = listOf(value, "corrections"); + if (raw === null) { + return fail('It must have the list "corrections".'); + } + const corrections: Correction[] = []; + for (const entry of raw) { + if (!isObject(entry)) { + return fail("Each correction must be an object."); + } + const problem = strings(entry, ["quote", "fix", "reason"]); + if (problem !== null) { + return fail(`In a correction, ${problem}.`); + } + corrections.push({ + quote: entry.quote as string, + fix: entry.fix as string, + reason: entry.reason as string, + }); + } + return { ok: true, value: corrections }; +} diff --git a/packages/core/src/analysis/names.ts b/packages/core/src/analysis/names.ts new file mode 100644 index 0000000..33111db --- /dev/null +++ b/packages/core/src/analysis/names.ts @@ -0,0 +1,86 @@ +/** Comparing names without any model: fast, free, and the same answer every time. */ + +/** Words that don't identify anyone by themselves, so "la tía Remedios" can match "Remedios". */ +const NOT_IDENTIFYING = new Set([ + "a", + "an", + "de", + "del", + "el", + "la", + "las", + "los", + "su", + "sus", + "the", + "tía", + "tío", + "un", + "una", + "don", + "doña", +]); + +/** Case-insensitive, keeping accents: "Tomás" and "Tomas" may be a real difference to flag. */ +export const nameKey = (name: string) => name.trim().toLocaleLowerCase(); + +export const withoutAccents = (text: string) => + text.normalize("NFD").replace(/\p{Diacritic}/gu, ""); + +/** The words of a name that actually identify someone. */ +export function nameTokens(name: string): string[] { + return nameKey(name) + .split(/[\s,.]+/) + .filter((token) => token !== "" && !NOT_IDENTIFYING.has(token)); +} + +/** True if every identifying word of one name is in the other: "Marta" and "Marta Rey". */ +export function isPartOfName(a: string, b: string): boolean { + const [shorter, longer] = [nameTokens(a), nameTokens(b)].sort((x, y) => x.length - y.length); + if (shorter === undefined || longer === undefined || shorter.length === 0) { + return false; + } + return shorter.every((token) => longer.includes(token)); +} + +/** + * True for a description rather than a name: "la doctora", "el cartero". Only a name's own context + * can say who these are, which is what the identity step of the analysis is for. + */ +export function isEpithet(name: string): boolean { + const first = name.trim().split(/\s+/)[0] ?? ""; + return first !== "" && first === first.toLocaleLowerCase(); +} + +export function editDistance(a: string, b: string): number { + let previous = Array.from({ length: b.length + 1 }, (_, index) => index); + for (let i = 1; i <= a.length; i += 1) { + const current = [i]; + for (let j = 1; j <= b.length; j += 1) { + const substitution = (previous[j - 1] ?? 0) + (a[i - 1] === b[j - 1] ? 0 : 1); + current.push(Math.min((previous[j] ?? 0) + 1, (current[j - 1] ?? 0) + 1, substitution)); + } + previous = current; + } + return previous[b.length] ?? 0; +} + +/** + * True for two names that look like one misspelled as the other: "Tomás" and "Thomas". Short names + * need to match more closely, or "Ana" and "Eva" would be taken for the same person. + */ +export function looksLikeVariant(a: string, b: string): boolean { + const left = withoutAccents(nameKey(a)); + const right = withoutAccents(nameKey(b)); + if (nameKey(a) === nameKey(b)) { + return false; + } + if (left === right) { + return true; + } + const shortest = Math.min(left.length, right.length); + if (shortest < 4) { + return false; + } + return editDistance(left, right) <= (shortest < 8 ? 1 : 2); +} diff --git a/packages/core/src/analysis/pipeline.ts b/packages/core/src/analysis/pipeline.ts new file mode 100644 index 0000000..0d0ece5 --- /dev/null +++ b/packages/core/src/analysis/pipeline.ts @@ -0,0 +1,973 @@ +import type { AiProvider, GenerateRequest, Usage } from "../provider/provider"; +import { generateStructured, type Validated } from "../provider/structured"; +import { + COMPARISON_SCHEMA, + type Correction, + EXTRACTION_SCHEMA, + type ExtractedEntity, + type Extraction, + type Finding, + IDENTITY_SCHEMA, + type IdentityAnswer, + PROOFREAD_SCHEMA, + validateCorrections, + validateExtraction, + validateFindings, + validateIdentities, +} from "./contract"; +import { + editDistance, + isEpithet, + isPartOfName, + looksLikeVariant, + nameKey, + nameTokens, + withoutAccents, +} from "./names"; +import { + type ComparedCategory, + comparisonInstructions, + extractionInstructions, + identityInstructions, + missedSentencesBlock, + proofreadInstructions, +} from "./prompts"; +import { + type AnalysisState, + type Entity, + type Fact, + type Issue, + liveEntities, + type Mention, + type Relation, + withoutScene, +} from "./state"; +import { + namesakesIn, + nameWords, + sentenceAround, + sentencesWithoutFacts, + speaksOfDeath, + topicWords, +} from "./topics"; + +export interface AnalysisDeps { + readonly provider: AiProvider; + readonly model: string; + readonly language: string; + readonly newId?: () => string; + readonly reasoning?: "default" | "off"; + readonly signal?: AbortSignal; + readonly sleep?: (ms: number, signal?: AbortSignal) => Promise; + readonly onUsage?: (usage: Usage) => void; + /** Which step is about to ask the model, for progress and for counting requests. */ + readonly onStep?: (step: string) => void; +} + +export interface SceneToAnalyze { + readonly id: string; + readonly title: string; + readonly text: string; +} + +export interface SceneAnalysis { + readonly state: AnalysisState; + /** Quotes the model gave that aren't in the scene. They were thrown away (rule 3). */ + readonly discardedQuotes: readonly string[]; + /** Facts about a name that isn't in the scene, so they couldn't be given to anyone. */ + readonly unplacedFacts: readonly string[]; +} + +/** Keeps a prompt small on a long novel. The latest facts are the ones kept. */ +const MAX_FACTS_IN_GROUP = 12; + +/** Keeps the requests per scene down. The topics shared by the fewest facts say the most. */ +const MAX_TOPIC_GROUPS = 5; + +/** + * How far the words a correction replaces can be from the new ones, and still be the same word. This + * keeps a model from rewriting the author's prose through a "correction" (rule 1), since the author can + * apply them with one click. + */ +const MAX_TYPO_DISTANCE = 0.34; + +function ask( + deps: AnalysisDeps, + step: string, + request: Omit, + validate: (value: unknown) => Validated, +): Promise { + deps.onStep?.(step); + return generateStructured( + deps.provider, + { + ...request, + model: deps.model, + temperature: 0, + ...(deps.reasoning === undefined ? {} : { reasoning: deps.reasoning }), + ...(deps.signal === undefined ? {} : { signal: deps.signal }), + }, + validate, + { + ...(deps.sleep === undefined ? {} : { sleep: deps.sleep }), + ...(deps.onUsage === undefined ? {} : { onUsage: deps.onUsage }), + }, + ); +} + +function knownEntitiesBlock(entities: readonly Entity[]): string { + if (entities.length === 0) { + return "KNOWN ENTITIES: none yet."; + } + const lines = entities.map((entity) => { + const others = entity.aliases.filter((alias) => alias !== entity.canonicalName); + const also = others.length > 0 ? ` (also: ${others.join(", ")})` : ""; + return `- [${entity.id}] ${entity.canonicalName}${also}, ${entity.type}`; + }); + return `KNOWN ENTITIES:\n${lines.join("\n")}`; +} + +function sceneBlock(scene: SceneToAnalyze): string { + return `SCENE "${scene.title}":\n${scene.text}`; +} + +/** Step 1 and the literal check of rule 3: a quote that isn't in the scene is thrown away. */ +async function extract( + state: AnalysisState, + scene: SceneToAnalyze, + deps: AnalysisDeps, +): Promise<{ extraction: Extraction; discarded: string[] }> { + const first = await ask( + deps, + "extract", + { + instructions: extractionInstructions(deps.language), + input: `${knownEntitiesBlock(liveEntities(state))}\n\n${sceneBlock(scene)}`, + structured: { name: "scene_extraction", schema: EXTRACTION_SCHEMA }, + }, + validateExtraction, + ); + // Every model tried skips sentences, and a fact never extracted can never be compared. + const missed = sentencesWithoutFacts( + scene.text, + first.facts.map((fact) => fact.quote), + ); + const raw = + missed.length === 0 + ? first + : mergeExtractions( + first, + await ask( + deps, + "extract-missed", + { + instructions: extractionInstructions(deps.language), + input: `${knownEntitiesBlock(liveEntities(state))}\n\n${sceneBlock(scene)}\n\n${missedSentencesBlock(missed)}`, + structured: { name: "scene_extraction", schema: EXTRACTION_SCHEMA }, + }, + validateExtraction, + ), + ); + const discarded: string[] = []; + const isLiteral = (quote: string) => { + if (quote !== "" && scene.text.includes(quote)) { + return true; + } + discarded.push(quote); + return false; + }; + const facts = raw.facts.filter((fact) => isLiteral(fact.quote)); + const relations = raw.relations.filter((relation) => isLiteral(relation.quote)); + return { + extraction: separateNamesakes( + { + entities: withUnlistedEntities( + raw.entities.map((entity) => usableAliases(entity, scene.text)), + [...facts.map((fact) => fact.entity), ...relations.flatMap((r) => [r.from, r.to])], + scene.text, + ), + facts, + relations, + }, + scene.text, + ), + discarded, + }; +} + +/** + * Models often state a fact about something they left out of their list of entities, as with a good + * fact about "veda". The fact counts if the scene really names it. Taken for "lore" here, it still + * joins any known place or person of that name in the next step. + */ +function withUnlistedEntities( + listed: readonly ExtractedEntity[], + used: readonly string[], + sceneText: string, +): ExtractedEntity[] { + const entities = [...listed]; + const text = sceneText.toLocaleLowerCase(); + for (const name of new Set(used)) { + const known = entities.some((entity) => + namesOf(entity).some( + (other) => nameKey(other) === nameKey(name) || isPartOfName(name, other), + ), + ); + if (!known && nameTokens(name).length > 0 && text.includes(name.toLocaleLowerCase())) { + entities.push({ name, type: "lore", aliases: [] }); + } + } + return entities; +} + +const namesOf = (entity: ExtractedEntity) => [entity.name, ...entity.aliases]; + +/** Adds a second reading to the first, joining the entities both of them named. */ +function mergeExtractions(first: Extraction, second: Extraction): Extraction { + const entities = [...first.entities]; + for (const entity of second.entities) { + const keys = new Set(namesOf(entity).map(nameKey)); + const index = entities.findIndex((other) => + namesOf(other).some((name) => keys.has(nameKey(name))), + ); + const same = entities[index]; + if (same === undefined) { + entities.push(entity); + } else { + entities[index] = { + ...same, + aliases: [...new Set([...same.aliases, ...namesOf(entity)])].filter( + (name) => name !== same.name, + ), + }; + } + } + return { + entities, + facts: [...first.facts, ...second.facts], + relations: [...first.relations, ...second.relations], + }; +} + +/** + * Undoes a merge the text rules out. Models tend to fold "Luis, el cartero" into "Luis Ferrer, el + * alcalde" whatever they're told, so this reads who's who from the text, and also moves their facts. + */ +function separateNamesakes(extraction: Extraction, sceneText: string): Extraction { + let { entities, facts, relations } = extraction; + for (const [shorter, longer] of namesakesIn(sceneText)) { + const names = [shorter.name, shorter.epithet, longer.name, longer.epithet].map(nameKey); + const isInvolved = (name: string) => names.includes(nameKey(name)); + const others = entities.filter((entity) => !namesOf(entity).some(isInvolved)); + const involved = entities.filter((entity) => namesOf(entity).some(isInvolved)); + const type = involved[0]?.type ?? "person"; + const inherited = involved.flatMap(namesOf).filter((name) => !isInvolved(name)); + entities = [ + ...others, + { name: shorter.name, type, aliases: [shorter.epithet] }, + { name: longer.name, type, aliases: [longer.epithet, ...new Set(inherited)] }, + ]; + const whoIn = (quote: string, current: string) => { + const sentence = sentenceAround(sceneText, quote); + if (sentence.includes(longer.name) || sentence.includes(longer.epithet)) { + return longer.name; + } + if (sentence.includes(shorter.name) || sentence.includes(shorter.epithet)) { + return shorter.name; + } + return current; + }; + facts = facts.map((fact) => + isInvolved(fact.entity) ? { ...fact, entity: whoIn(fact.quote, fact.entity) } : fact, + ); + relations = relations.map((relation) => ({ + ...relation, + from: isInvolved(relation.from) ? whoIn(relation.quote, relation.from) : relation.from, + to: isInvolved(relation.to) ? whoIn(relation.quote, relation.to) : relation.to, + })); + } + return { entities, facts, relations }; +} + +/** + * Keeps only what can really name someone. Models tend to add articles and family words, and "su + * hermano" would later match any character's brother. A name must also appear in the scene. + */ +function usableAliases(extracted: ExtractedEntity, sceneText: string): ExtractedEntity { + const usable = extracted.aliases.filter( + (alias) => + sceneText.includes(alias) && + nameTokens(alias).length > 0 && + !/^(su|sus|mi|mis|tu|tus|his|her|their|my|your)\s/i.test(alias.trim()) && + !looksLikeVariant(alias, extracted.name), + ); + return { ...extracted, aliases: [...new Set(usable)] }; +} + +/** How one extracted entity relates to the known ones, using only the names. */ +function matchByName( + extracted: ExtractedEntity, + known: readonly Entity[], + taken: ReadonlySet, +): { kind: "same"; id: string } | { kind: "ambiguous" } | { kind: "new" } { + const keys = new Set(namesOf(extracted).map(nameKey)); + const exact = known.filter((entity) => + [entity.canonicalName, ...entity.aliases].some((name) => keys.has(nameKey(name))), + ); + const [onlyExact] = exact; + if (exact.length === 1 && onlyExact !== undefined) { + return { kind: "same", id: onlyExact.id }; + } + if (exact.length > 1) { + return { kind: "ambiguous" }; + } + const partial = known.filter( + (entity) => + entity.type === extracted.type && + !taken.has(entity.id) && + namesOf(extracted).some((name) => + [entity.canonicalName, ...entity.aliases].some((other) => isPartOfName(name, other)), + ), + ); + const [onlyPartial] = partial; + if (partial.length === 1 && onlyPartial !== undefined) { + return { kind: "same", id: onlyPartial.id }; + } + return partial.length > 1 ? { kind: "ambiguous" } : { kind: "new" }; +} + +/** Step 2: code settles what names alone can, and the model only sees what's left. */ +async function resolve( + state: AnalysisState, + scene: SceneToAnalyze, + extraction: Extraction, + deps: AnalysisDeps, + newId: () => string, +): Promise<{ entities: Entity[]; nameToId: Map }> { + const entities = [...state.entities]; + const nameToId = new Map(); + const taken = new Set(); + const decided = new Map(); + const toAsk: ExtractedEntity[] = []; + + for (const extracted of extraction.entities) { + const match = matchByName(extracted, state.entities, taken); + if (match.kind === "same") { + decided.set(extracted, match.id); + taken.add(match.id); + } else if (match.kind === "ambiguous" || isEpithet(extracted.name)) { + toAsk.push(extracted); + } else { + decided.set(extracted, null); + } + } + + if (toAsk.length > 0 && state.entities.length > 0) { + const answers: readonly IdentityAnswer[] = await ask( + deps, + "resolve", + { + instructions: identityInstructions(), + input: `${knownEntitiesBlock(liveEntities(state))}\n\n${sceneBlock(scene)}\n\nNAMES TO DECIDE:\n${toAsk + .map((entity) => `- ${entity.name}`) + .join("\n")}`, + structured: { name: "identity_answers", schema: IDENTITY_SCHEMA }, + }, + validateIdentities, + ); + for (const extracted of toAsk) { + const answer = answers.find((item) => nameKey(item.name) === nameKey(extracted.name)); + const id = answer?.entity_id ?? null; + // An answer naming an entity that doesn't exist is treated as no answer. + const exists = id !== null && state.entities.some((entity) => entity.id === id); + decided.set(extracted, exists ? id : null); + } + } else { + for (const extracted of toAsk) { + decided.set(extracted, null); + } + } + + for (const extracted of extraction.entities) { + const existingId = decided.get(extracted) ?? null; + if (existingId === null) { + const id = newId(); + entities.push({ + id, + type: extracted.type, + canonicalName: extracted.name, + aliases: [...new Set(namesOf(extracted))], + }); + for (const name of namesOf(extracted)) { + nameToId.set(nameKey(name), id); + } + } else { + const index = entities.findIndex((entity) => entity.id === existingId); + const entity = entities[index]; + if (entity !== undefined) { + entities[index] = { + ...entity, + aliases: [...new Set([...entity.aliases, ...namesOf(extracted)])], + }; + } + for (const name of namesOf(extracted)) { + nameToId.set(nameKey(name), existingId); + } + } + } + return { entities, nameToId }; +} + +/** The id for a name used in a fact or a relation, which may not be listed among the entities. */ +function idForName( + name: string, + nameToId: ReadonlyMap, + entities: readonly Entity[], +) { + const direct = nameToId.get(nameKey(name)); + if (direct !== undefined) { + return direct; + } + const known = entities.filter((entity) => + [entity.canonicalName, ...entity.aliases].some( + (other) => nameKey(other) === nameKey(name) || isPartOfName(name, other), + ), + ); + return known.length === 1 ? known[0]?.id : undefined; +} + +const namesInQuote = (quote: string, name: string) => + new RegExp(`(? + [entity.canonicalName, ...entity.aliases].some( + (name) => nameTokens(name).length > 0 && namesInQuote(quote, name), + ), + ); + return named.length === 1 ? named[0]?.id : undefined; +} + +/** A fact stated in a line of dialogue: the quote sits in a line opened by a dash. */ +function isInDialogue(sceneText: string, quote: string): boolean { + const at = sceneText.indexOf(quote); + if (at < 0) { + return false; + } + const lineStart = sceneText.lastIndexOf("\n", at) + 1; + return /^\s*[—–-]/.test(sceneText.slice(lineStart, at + 1)); +} + +function mentionsIn(scene: SceneToAnalyze, entities: readonly Entity[]): Mention[] { + const mentions: Mention[] = []; + for (const entity of entities) { + for (const name of new Set([entity.canonicalName, ...entity.aliases])) { + if (name !== "" && scene.text.includes(name)) { + mentions.push({ sceneId: scene.id, entityId: entity.id, quote: name }); + } + } + } + return mentions; +} + +/** + * Capitalized words in the scene that look like a misspelled known name. It reads the text rather + * than the model's answer, because a model may well decide "Thomas" is just "Tomás" and say nothing. + */ +function variantsInText( + scene: SceneToAnalyze, + known: readonly Entity[], + mentions: readonly Mention[], +): Issue[] { + const knownTokens = known.flatMap((entity) => + [entity.canonicalName, ...entity.aliases].flatMap((name) => + nameTokens(name).map((token) => ({ entity, token })), + ), + ); + const exact = new Set(knownTokens.map((item) => item.token)); + const issues: Issue[] = []; + const words = new Set(scene.text.match(/\p{Lu}\p{L}+/gu) ?? []); + for (const word of words) { + if (exact.has(nameKey(word))) { + continue; + } + const match = knownTokens.find( + (item) => item.entity.type === "person" && looksLikeVariant(word, item.token), + ); + const first = match && mentions.find((mention) => mention.entityId === match.entity.id); + if (match !== undefined && first !== undefined) { + issues.push({ + kind: "name_variant", + a: { sceneId: first.sceneId, quote: first.quote }, + b: { sceneId: scene.id, quote: word }, + reason: `«${word}» y «${first.quote}» podrían ser el mismo nombre.`, + }); + } + } + return issues; +} + +/** Names that look like one misspelled as the other, which only the author can settle. */ +function nameVariantIssues( + state: AnalysisState, + before: readonly Entity[], + after: readonly Entity[], + mentions: readonly Mention[], +): Issue[] { + const issues: Issue[] = []; + const allMentions = [...state.mentions, ...mentions]; + const added = after.filter((entity) => !before.some((old) => old.id === entity.id)); + for (const entity of added) { + for (const other of after) { + if (other.id === entity.id || other.type !== entity.type) { + continue; + } + const pair = [entity.canonicalName, ...entity.aliases].flatMap((name) => + [other.canonicalName, ...other.aliases] + .filter((otherName) => looksLikeVariant(name, otherName)) + .map((otherName) => [name, otherName] as const), + )[0]; + if (pair === undefined) { + continue; + } + const a = allMentions.find((m) => m.entityId === other.id && m.quote === pair[1]); + const b = allMentions.find((m) => m.entityId === entity.id && m.quote === pair[0]); + if (a !== undefined && b !== undefined) { + issues.push({ + kind: "name_variant", + a: { sceneId: a.sceneId, quote: a.quote }, + b: { sceneId: b.sceneId, quote: b.quote }, + reason: `«${pair[0]}» y «${pair[1]}» podrían ser el mismo nombre.`, + }); + break; + } + } + } + return issues; +} + +function factLine( + fact: Fact, + tag: string, + sceneNumber: number, + entities: readonly Entity[], +): string { + const name = entities.find((entity) => entity.id === fact.entityId)?.canonicalName ?? "?"; + const dialogue = fact.inDialogue ? " (dialogue)" : ""; + const when = fact.storyTime === null ? "" : ` [when: ${fact.storyTime}]`; + return `${tag} scene ${sceneNumber}${dialogue}${when} | ${name} | ${fact.attribute}: ${fact.value} | "${fact.sentence}"`; +} + +/** Compared per entity: a trait of one person says nothing about another. */ +const PER_ENTITY: readonly ComparedCategory[] = [ + "physical", + "age", + "marital_status", + "family", + "job", + "alive_or_dead", +]; + +/** Compared across entities: the same trip or the same pair of people can be told from either side. */ +const ACROSS_ENTITIES: readonly ComparedCategory[] = [ + "relative_age", + "date_of_event", + "travel_time", +]; + +interface FactGroup { + readonly category: ComparedCategory | "topic"; + readonly facts: readonly Fact[]; +} + +/** What can be done, as opposed to what someone is: it can't happen after dying. */ +const DEEDS: ReadonlySet = new Set([ + "action", + "location", + "thought", + "possession", + "other", +]); + +const isSubset = (small: readonly Fact[], large: readonly Fact[]) => + small.every((fact) => large.includes(fact)); + +/** + * Facts sharing a word, whatever category and person the model gave them. A model may file a start date + * under what a character did, and this still sets it next to the other date. People's names don't count + * as topics, or every fact about the protagonist would be one group. + */ +function topicGroups( + known: readonly Fact[], + newFacts: readonly Fact[], + entities: readonly Entity[], +): FactGroup[] { + const all = [...known, ...newFacts]; + const people = new Set( + entities + .filter((entity) => entity.type === "person") + .flatMap((entity) => [entity.canonicalName, ...entity.aliases]) + .flatMap(nameWords), + ); + const wordsOf = new Map( + all.map((fact) => [fact, topicWords(`${fact.quote} ${fact.value}`, people)]), + ); + const topics = new Set(newFacts.flatMap((fact) => [...(wordsOf.get(fact) ?? [])])); + const groups: Fact[][] = []; + for (const topic of topics) { + const facts = all.filter((fact) => wordsOf.get(fact)?.has(topic)); + const scenes = new Set(facts.map((fact) => fact.sceneId)); + // Inside one scene, it's usually one sentence split into several facts. + if (scenes.size >= 2 && facts.some((fact) => newFacts.includes(fact))) { + groups.push(facts); + } + } + return groups + .sort((a, b) => a.length - b.length) + .slice(0, MAX_TOPIC_GROUPS) + .map((facts) => ({ category: "topic", facts })); +} + +/** + * Which facts to weigh together. Only facts of one kind, about one thing, and only groups with something + * new in them. Actions and locations aren't compared with each other, as they change all the time: + * that's also why a character lying about where they've been can't turn into a false alarm. + */ +function factGroups( + known: readonly Fact[], + newFacts: readonly Fact[], + entities: readonly Entity[], +): FactGroup[] { + const all = [...known, ...newFacts]; + const newIds = new Set(newFacts.map((fact) => fact.id)); + const groups: FactGroup[] = []; + const add = (category: ComparedCategory, facts: readonly Fact[]) => { + if (facts.length >= 2 && facts.some((fact) => newIds.has(fact.id))) { + groups.push({ category, facts: facts.slice(-MAX_FACTS_IN_GROUP) }); + } + }; + for (const category of ACROSS_ENTITIES) { + add( + category, + all.filter((fact) => fact.category === category), + ); + } + for (const category of PER_ENTITY) { + const entityIds = new Set( + newFacts.filter((fact) => fact.category === category).map((fact) => fact.entityId), + ); + for (const entityId of entityIds) { + add( + category, + all.filter((fact) => fact.category === category && fact.entityId === entityId), + ); + } + } + // The dead acting. A death can be filed under anything, and so can what they do later, which can + // even be filed under someone else: "Marta got a call: her aunt asked her". + const isDeath = (fact: Fact) => fact.category === "alive_or_dead" || speaksOfDeath(fact.quote); + const entityById = new Map(entities.map((entity) => [entity.id, entity])); + const deadIds = new Set( + all + .filter(isDeath) + .filter((fact) => entityById.get(fact.entityId)?.type === "person") + .map((fact) => fact.entityId), + ); + for (const entityId of deadIds) { + const names = [ + entityById.get(entityId)?.canonicalName ?? "", + ...(entityById.get(entityId)?.aliases ?? []), + ].filter((name) => name !== ""); + const deaths = all.filter((fact) => isDeath(fact) && fact.entityId === entityId); + const deeds = newFacts.filter( + (fact) => + !deaths.includes(fact) && + DEEDS.has(fact.category) && + (fact.entityId === entityId || names.some((name) => fact.quote.includes(name))), + ); + if (deeds.length > 0) { + add("alive_or_dead", [...deaths, ...deeds]); + } + } + const unique = groups.filter( + (group, index) => + !groups.some( + (other, otherIndex) => + index !== otherIndex && + isSubset(group.facts, other.facts) && + // Of two equal groups, one stays. + (other.facts.length > group.facts.length || otherIndex < index), + ), + ); + const topics = topicGroups(known, newFacts, entities).filter( + (group, index, list) => + !unique.some((other) => isSubset(group.facts, other.facts)) && + !list.some( + (other, otherIndex) => + index !== otherIndex && + isSubset(group.facts, other.facts) && + (other.facts.length > group.facts.length || otherIndex < index), + ), + ); + return [ + ...unique, + ...topics.map((group) => ({ ...group, facts: group.facts.slice(-MAX_FACTS_IN_GROUP) })), + ]; +} + +/** Step 3: new facts against the registry, without rereading the novel, in small groups. */ +async function compare( + state: AnalysisState, + newFacts: readonly Fact[], + entities: readonly Entity[], + deps: AnalysisDeps, +): Promise { + const sceneNumber = (id: string) => state.sceneOrder.indexOf(id) + 1; + const newIds = new Set(newFacts.map((fact) => fact.id)); + const issues: Issue[] = []; + const seen = new Set(); + + for (const group of factGroups(state.facts, newFacts, entities)) { + const ordered = [...group.facts].sort( + (a, b) => sceneNumber(a.sceneId) - sceneNumber(b.sceneId), + ); + const tagged = new Map(); + const lines = ordered.map((fact, index) => { + const tag = `F${index + 1}`; + tagged.set(tag, fact); + const line = factLine(fact, tag, sceneNumber(fact.sceneId), entities); + return newIds.has(fact.id) ? `${line} NEW` : line; + }); + const findings: readonly Finding[] = await ask( + deps, + `compare:${group.category}`, + { + instructions: comparisonInstructions(deps.language, group.category), + input: `FACTS:\n${lines.join("\n")}`, + structured: { name: "comparison", schema: COMPARISON_SCHEMA }, + }, + validateFindings, + ); + for (const finding of findings) { + if (finding.verdict !== "contradiction") { + continue; + } + // Some models answer "F1, F2" for one conflict with two facts. + for (const newTag of finding.new_fact.match(/F\d+/g) ?? []) { + for (const otherTag of finding.other_fact.match(/F\d+/g) ?? []) { + const a = tagged.get(otherTag); + const b = tagged.get(newTag); + if ( + a === undefined || + b === undefined || + // Two old facts were weighed when they were new. Stored here, this wouldn't come back + // if the scene they cite were analysed again, as it doesn't cite the one being analysed. + (!newIds.has(a.id) && !newIds.has(b.id)) || + // One sentence doesn't contradict itself: these are two readings of it. + (a.sceneId === b.sceneId && a.sentence === b.sentence) + ) { + continue; + } + // By sentence: a model can find one conflict again through two facts of the same sentence. + const key = [a.sentence, b.sentence].sort().join("|"); + if (!seen.has(key)) { + seen.add(key); + issues.push({ + kind: "contradiction", + a: { sceneId: a.sceneId, quote: a.quote }, + b: { sceneId: b.sceneId, quote: b.quote }, + reason: finding.reason, + }); + } + } + } + } + } + return issues; +} + +/** A line of dialogue opened with a hyphen. A rule, so it's found every time without any model. */ +function dialogueDashIssues(scene: SceneToAnalyze): Issue[] { + return scene.text.split("\n").flatMap((line): Issue[] => { + const match = line.match(/^\s*(-\s?)\S/); + if (match === null) { + return []; + } + const sentence = line.trim().split(/[,.;:!?]/)[0] ?? line.trim(); + return [ + { + kind: "correction", + a: { sceneId: scene.id, quote: sentence }, + fix: sentence.replace(/^-\s?/, "—"), + reason: "Los diálogos se abren con raya, no con guion.", + }, + ]; + }); +} + +/** + * Spanish misspellings mostly sound right, and letters alone can't tell them from another word: "halla" + * is as near to the right "haya" as to the wrong "había". By sound, "haya" is the same word, and so is + * "a ver" for "haber". + */ +function asItSounds(text: string, language: string): string { + return language === "es" + ? text + .replace(/(? withoutAccents(text.toLocaleLowerCase()).split(/\s+/); + const a = words(before); + const b = words(after); + while (a.length > 0 && a[0] === b[0]) { + a.shift(); + b.shift(); + } + while (a.length > 0 && a.at(-1) === b.at(-1)) { + a.pop(); + b.pop(); + } + // Nothing to compare with means a word was added or taken away, which isn't a typo. + if (a.length === 0 || b.length === 0) { + return a.length === b.length; + } + const left = asItSounds(a.join(" "), language); + const right = asItSounds(b.join(" "), language); + return editDistance(left, right) / Math.max(left.length, right.length, 1) <= MAX_TYPO_DISTANCE; +} + +function isSmallCorrection(correction: Correction, sceneText: string, language: string): boolean { + return ( + correction.quote !== correction.fix && + correction.quote !== "" && + sceneText.includes(correction.quote) && + looksLikeTypo(correction.quote, correction.fix, language) + ); +} + +/** Step 5. Anything that isn't a small, literal fix is thrown away. */ +async function proofread(scene: SceneToAnalyze, deps: AnalysisDeps): Promise { + const corrections = await ask( + deps, + "proofread", + { + instructions: proofreadInstructions(deps.language), + input: sceneBlock(scene), + structured: { name: "proofreading", schema: PROOFREAD_SCHEMA }, + }, + validateCorrections, + ); + return corrections + .filter((correction) => isSmallCorrection(correction, scene.text, deps.language)) + .map((correction) => ({ + kind: "correction", + a: { sceneId: scene.id, quote: correction.quote }, + fix: correction.fix, + reason: correction.reason, + })); +} + +/** + * Analyses one scene, in the steps of SPEC §5, against what's known of the rest. Reanalysing a scene + * first takes out everything that came from it. If the model can't be reached this throws, and the + * state stays as it was, so that the scene can simply wait in the queue. + */ +export async function analyzeScene( + current: AnalysisState, + scene: SceneToAnalyze, + deps: AnalysisDeps, +): Promise { + const newId = deps.newId ?? (() => crypto.randomUUID()); + const state = withoutScene(current, scene.id); + + const { extraction, discarded } = await extract(state, scene, deps); + const { entities, nameToId } = await resolve(state, scene, extraction, deps, newId); + const mentions = mentionsIn(scene, entities); + + const unplacedFacts: string[] = []; + const facts: Fact[] = extraction.facts.flatMap((fact) => { + const entityId = + idForName(fact.entity, nameToId, entities) ?? onlyEntityNamedIn(fact.quote, entities); + if (entityId === undefined) { + unplacedFacts.push(fact.quote); + return []; + } + return [ + { + id: newId(), + sceneId: scene.id, + entityId, + category: fact.category, + attribute: fact.attribute, + value: fact.value, + quote: fact.quote, + sentence: sentenceAround(scene.text, fact.quote), + storyTime: fact.story_time, + inDialogue: isInDialogue(scene.text, fact.quote), + }, + ]; + }); + const relations: Relation[] = extraction.relations.flatMap((relation) => { + const fromEntityId = idForName(relation.from, nameToId, entities); + const toEntityId = idForName(relation.to, nameToId, entities); + return fromEntityId === undefined || toEntityId === undefined + ? [] + : [ + { + id: newId(), + sceneId: scene.id, + fromEntityId, + toEntityId, + label: relation.label, + quote: relation.quote, + }, + ]; + }); + + const contradictions = await compare(state, facts, entities, deps); + const corrections = await proofread(scene, deps); + // The same pair can be found both ways: from the text and from a new entity. + const variants = [ + ...variantsInText(scene, state.entities, state.mentions), + ...nameVariantIssues(state, state.entities, entities, mentions), + ].filter( + (issue, index, all) => + all.findIndex((other) => "b" in other && "b" in issue && other.b.quote === issue.b.quote) === + index, + ); + + return { + state: { + ...state, + entities, + mentions: [...state.mentions, ...mentions], + facts: [...state.facts, ...facts], + relations: [...state.relations, ...relations], + issues: [ + ...state.issues, + ...contradictions, + ...variants, + ...dialogueDashIssues(scene), + ...corrections, + ], + }, + discardedQuotes: discarded, + unplacedFacts, + }; +} diff --git a/packages/core/src/analysis/prompts.ts b/packages/core/src/analysis/prompts.ts new file mode 100644 index 0000000..61516bc --- /dev/null +++ b/packages/core/src/analysis/prompts.ts @@ -0,0 +1,129 @@ +/** + * The instructions for each step. Any change here must go through the evaluation in `fixtures/`, and + * be compared with the previous result. + * + * They're in English, and ask for values in the novel's language, so that one set serves every novel. + */ + +const LANGUAGE_NAMES: Readonly> = { es: "Spanish", en: "English" }; + +const languageName = (language: string) => LANGUAGE_NAMES[language] ?? language; + +export function extractionInstructions(language: string): string { + return `You read one scene of a novel and list what it states, in a structured way. You never add anything the scene doesn't say. + +Go through the scene sentence by sentence, and don't skip any sentence that says something about a person, a place, a date or an event. That includes what a character thinks or notices about someone: if someone thinks that Ana has grey eyes, that's a fact about Ana's eyes. + +ENTITIES +List every person, every location, and each story-specific thing or event that matters to the plot (type "lore": a letter, a fishing ban, a ceremony). Ordinary objects and furniture aren't entities. +- "name" is the fullest proper name the scene gives. Use a name from KNOWN ENTITIES when it's clearly the same one. +- "aliases" has other names for the same one, such as a first name, or a job used as a name ("la doctora"). Copy them from the scene. Never use pronouns, articles or family words like "su hermano" as aliases. +- Two people who share a first name are DIFFERENT entities unless the text says they're the same. In "Ana, la maestra" and "Ana Ruiz, la jueza" there are two people: "Ana" with the alias "la maestra", and "Ana Ruiz" with the alias "la jueza". Never mix up their aliases. +- A person who appears only by their job ("la doctora") is listed under that job. + +FACTS +List every fact a continuity checker could need, including small actions such as arriving late somewhere: each sentence that says something should give at least one fact. +- "entity" is who or what the fact is about. Work it out when the subject is omitted or is a pronoun: in Spanish a verb like "llegó tarde" has a subject you must find in the context. +- "category" is the closest of these: + physical: body and appearance, such as eye colour or height. + age: how old someone is at some moment. + relative_age: who is older or younger than whom, as in "three years older than him" or "her older brother". + marital_status: single, married, widowed. + family: who is whose parent, sibling or child. + job: what someone works as. + alive_or_dead: someone dying, being dead or being buried. + date_of_event: the date or day when something happens or starts. + travel_time: how long getting from one place to another takes. This includes someone leaving one place and being in another by a stated time, as in "she set off after breakfast and reached the border by noon". + location: where someone or something is. + action: anything else someone does, however small. + possession: what someone owns, has sold or has lost. + thought: what someone believes, wants or feels. + other: anything else. +- "attribute" names the fact in a few words, "value" says it, naming whoever a pronoun stands for. +- "quote" is the shortest exact text from the scene that proves the fact, copied character by character, with its original accents and capitals. Never paraphrase or join separate pieces. If you can't quote it, leave the fact out. +- "story_time" says when it happens in the story, if the scene tells (such as "when she was twelve", "two years later"). Otherwise null. + +RELATIONS +List the relationships the scene states between two entities, with an exact "quote" as above. + +Write attributes, values and labels in ${languageName(language)}. Answer only with the JSON object.`; +} + +export function identityInstructions(): string { + return `You work out who is who in a novel. You get one scene, the entities known so far, and some names from the scene. For each name, decide whether it refers to a known entity. + +- Answer with the known entity's id only if the text gives real grounds: a shared job, a place, or an action that ties them together. An epithet like "la doctora" can refer to a known person if the story points to it. +- Two people can share a first name and still be different people. If a name fits more than one known entity and nothing tells them apart, answer null. +- Answer null for anyone new, and whenever you aren't sure. A wrong merge is worse than a missing one. + +Answer only with the JSON object, with one answer per name.`; +} + +/** The rule for each kind of fact the comparison looks at. */ +export const COMPARISON_RULES = { + physical: + "These facts are about one body. Eye colour, height and the like don't change by themselves: a trait stated in two different ways is a contradiction, unless one of the facts says what changed it, such as a new scar or dyed hair. Something that only looks different for a moment, or a different trait, isn't related.", + age: "Someone's age changes over time. Being twelve in a memory and forty now is no error, and neither is any other pair of ages from different moments. Report only ages that can't both hold at the same moment.", + relative_age: + "These facts say who is older between two people. If one says that A is older than B, and another says that B is older than A, that's a contradiction. Read them carefully: 'her older cousin' means the cousin is the older one.", + marital_status: + "Marital status changes over time. Being single and later married is an evolution, never a contradiction. Report a contradiction only if both are stated for the same moment.", + family: + "These facts are about family ties, which don't change. The same person given two different parents is a contradiction. Different relatives aren't.", + job: "Jobs change over time, which is an evolution. Report a contradiction only if both are stated for the same moment.", + alive_or_dead: + "Someone who has died or been buried can't act, talk or phone later in the story. If an action happens after the death in reading order, that's a contradiction, unless the facts say it's a memory, a dream or a recording.", + date_of_event: + "If the same event, such as a wedding or a trial, is given two different dates or days, that's a contradiction, even if a character says one of them. Different events can have different dates.", + travel_time: + "These facts are about how long journeys take. If the same route takes a long time in one fact and is done much faster in another, that's a contradiction, whoever is travelling. For example, a month by sea against setting off on Monday and arriving on Tuesday. Different routes aren't comparable.", +} as const; + +export type ComparedCategory = keyof typeof COMPARISON_RULES; + +/** For facts that only share a topic, which may have been filed under any category. */ +const TOPIC_RULE = `These facts use the same word, but may be about different things: most pairs are unrelated. +It's a contradiction if both can't be true of the story, as with: +- The same event, such as a wedding or a trial, given two different dates or days, even if a character says one of them. +- The same route taking very different times, whoever travels. For example, a month by sea against setting off on Monday and arriving on Tuesday. +- Who is older between two people, stated in opposite ways. "Her older cousin" means the cousin is the older one. +- A body trait stated in two different ways, with nothing in the facts explaining it. +It isn't a contradiction if: +- It can change over time: where someone is, what they own, their job, whether they're married, how old they are. +- A character says something about their own life in dialogue. They may be lying or mistaken, which is part of the story. +- They're about different people's things. One person selling a house says nothing about another person's house.`; + +export function comparisonInstructions( + language: string, + category: ComparedCategory | "topic", +): string { + return `You check a novel for continuity errors. You get a few facts, from numbered scenes in reading order. Facts marked NEW were just added. + +${category === "topic" ? TOPIC_RULE : COMPARISON_RULES[category]} + +Report each pair with at least one NEW fact that is related in one of these two ways: +- "contradiction": both can't be true of the story. +- "evolution": it changed over time and both are true. +Don't report anything else: if they're unrelated, or about different things, leave them out. An empty list is a normal answer. Use the fact ids exactly as given. + +Write each "reason" in ${languageName(language)}, in one short sentence. Answer only with the JSON object.`; +} + +/** Second reading of a scene, of what the first one left out. */ +export function missedSentencesBlock(sentences: readonly string[]): string { + return `A first reading found no facts in these sentences of the scene. List what each of them states, if anything. If one of them states nothing, leave it out.\n${sentences + .map((sentence) => `- ${sentence}`) + .join("\n")}`; +} + +export function proofreadInstructions(language: string): string { + return `You proofread one scene of a novel written in ${languageName(language)}. Report only typos, spelling mistakes and wrong forms of a word, such as a verb form that should be another one with a similar sound. + +Never suggest changes of style, wording, rhythm or punctuation choices. Fiction often breaks rules on purpose: when in doubt, report nothing. Dialogue marks are handled elsewhere. An empty list is the normal answer. + +- "quote" is the wrong text, copied character by character from the scene, with just enough words around it to place it. +- "fix" is that same text corrected, and nothing else changed. +- "reason" says what's wrong in ${languageName(language)}, in one short sentence. + +Answer only with the JSON object.`; +} diff --git a/packages/core/src/analysis/queue.ts b/packages/core/src/analysis/queue.ts new file mode 100644 index 0000000..a31e554 --- /dev/null +++ b/packages/core/src/analysis/queue.ts @@ -0,0 +1,249 @@ +import { readScene } from "../export/manuscript"; +import { plainText } from "../prose/text"; +import { StructuredOutputError } from "../provider/structured"; +import type { Storage } from "../storage/storage"; +import { type AnalysisDeps, analyzeScene } from "./pipeline"; +import { withoutScene } from "./state"; + +/** + * Analyses what changed, when the author stops, and only then (SPEC §5). Its state lives in the + * database: a scene is queued for as long as its text differs from the text last analysed, so that + * being offline, a rate limit, or closing the app can only delay the work, never lose it. + */ + +export type QueueStatus = + /** Everything that could be analysed was. */ + | "done" + /** Offline or rate limited. The rest stays queued and is tried again by itself. */ + | "waiting" + /** Refused, as with a bad key or no credit. It stays queued, and only the author can sort it out. */ + | "blocked" + | "cancelled"; + +export interface QueueRun { + readonly status: QueueStatus; + readonly analyzed: readonly string[]; + /** The model's answer for these wasn't usable. They aren't tried again until their text changes. */ + readonly skipped: readonly string[]; + /** Scenes still queued, not counting the skipped ones. */ + readonly pending: number; + readonly message: string | null; +} + +export interface AnalysisQueueOptions { + readonly storage: Storage; + readonly projectId: string; + /** The provider and model the author chose. The queue never picks another one (rule 10). */ + readonly analysis: Omit; + /** How long without typing counts as a pause. */ + readonly pauseMs?: number; + /** The first wait before trying again after `waiting`. It doubles each time, up to an hour. */ + readonly retryMs?: number; + readonly onRun?: (run: QueueRun) => void; + /** Something other than the model went wrong, as a failing database. */ + readonly onError?: (error: unknown) => void; +} + +const DEFAULT_PAUSE_MS = 60_000; +const DEFAULT_RETRY_MS = 5 * 60_000; +const LONGEST_RETRY_MS = 60 * 60_000; + +export async function hashText(text: string): Promise { + const digest = await crypto.subtle.digest("SHA-256", new TextEncoder().encode(text)); + return [...new Uint8Array(digest)].map((byte) => byte.toString(16).padStart(2, "0")).join(""); +} + +export class AnalysisQueue { + private readonly changed = new Set(); + /** Scene and hash pairs the model couldn't answer for. In memory: restarting the app retries them. */ + private readonly unusable = new Set(); + /** Until the first run has looked at every scene: a crash can leave saved text with an old hash. */ + private checkedAll = false; + private timer: ReturnType | null = null; + private running: Promise | null = null; + private following: Promise | null = null; + private controller = new AbortController(); + private runAgain = false; + private stopped = false; + private retryMs: number; + + constructor(private readonly options: AnalysisQueueOptions) { + this.retryMs = options.retryMs ?? DEFAULT_RETRY_MS; + } + + /** Called as the author writes. It only restarts a timer, so typing never reaches a model. */ + sceneEdited(sceneId: string): void { + this.changed.add(sceneId); + this.resetRetry(); + this.schedule(this.options.pauseMs ?? DEFAULT_PAUSE_MS); + } + + sceneClosed(sceneId: string): void { + this.changed.add(sceneId); + this.resetRetry(); + this.schedule(0); + } + + /** On opening a project, to pick up whatever is still queued from last time. */ + start(): void { + this.schedule(0); + } + + /** Resolves once whatever a timer started is over, including what's decided to do next. */ + async settled(): Promise { + await this.following; + } + + /** Cancels what's in progress. A scene cut short stays queued, with its old data untouched. */ + async stop(): Promise { + this.stopped = true; + this.clearTimer(); + this.controller.abort(); + await this.running?.catch(() => {}); + } + + /** Runs now, or joins the run in progress. */ + run(): Promise { + if (this.running !== null) { + return this.running; + } + this.clearTimer(); + this.controller = new AbortController(); + this.running = this.work(this.controller.signal).finally(() => { + this.running = null; + }); + return this.running; + } + + private resetRetry(): void { + this.retryMs = this.options.retryMs ?? DEFAULT_RETRY_MS; + } + + private clearTimer(): void { + if (this.timer !== null) { + clearTimeout(this.timer); + this.timer = null; + } + } + + private schedule(delayMs: number): void { + if (this.stopped) { + return; + } + if (this.running !== null) { + this.runAgain = true; + return; + } + this.clearTimer(); + this.timer = setTimeout(() => { + this.timer = null; + this.following = this.runAndFollowUp(); + }, delayMs); + } + + private async runAndFollowUp(): Promise { + try { + const run = await this.run(); + this.options.onRun?.(run); + if (this.runAgain) { + this.runAgain = false; + this.schedule(this.options.pauseMs ?? DEFAULT_PAUSE_MS); + } else if (run.status === "waiting") { + this.schedule(this.retryMs); + this.retryMs = Math.min(this.retryMs * 2, LONGEST_RETRY_MS); + } else if (run.status === "done") { + this.resetRetry(); + } + } catch (error) { + this.options.onError?.(error); + } + } + + private async sceneText(sceneId: string): Promise { + return plainText(await readScene(this.options.storage, sceneId)); + } + + /** Brings the stored hashes up to date, reading only the scenes that may have changed. */ + private async refreshHashes(): Promise { + const { storage, projectId } = this.options; + let sceneIds = [...this.changed]; + this.changed.clear(); + if (!this.checkedAll) { + const chapters = await storage.listChapters(projectId); + const lists = await Promise.all(chapters.map((chapter) => storage.listScenes(chapter.id))); + sceneIds = lists.flat().map((scene) => scene.id); + } + for (const sceneId of sceneIds) { + await storage.setSceneContentHash(sceneId, await hashText(await this.sceneText(sceneId))); + } + this.checkedAll = true; + } + + private async work(signal: AbortSignal): Promise { + const { storage, projectId, analysis } = this.options; + await this.refreshHashes(); + + const analyzed: string[] = []; + const skipped: string[] = []; + let status: QueueStatus = "done"; + let message: string | null = null; + + for (const scene of await storage.listScenesToAnalyze(projectId)) { + if (signal.aborted) { + status = "cancelled"; + break; + } + const text = await this.sceneText(scene.id); + const hash = await hashText(text); + await storage.setSceneContentHash(scene.id, hash); + const pair = `${scene.id}:${hash}`; + if (this.unusable.has(pair)) { + skipped.push(scene.id); + continue; + } + const current = await storage.loadAnalysis(projectId); + + // An empty scene has nothing to ask about, and whatever it used to say is gone. + if (text.trim() === "") { + await storage.saveSceneAnalysis(projectId, scene.id, hash, withoutScene(current, scene.id)); + analyzed.push(scene.id); + continue; + } + + try { + const { state } = await analyzeScene( + current, + { id: scene.id, title: scene.title, text }, + { ...analysis, signal }, + ); + await storage.saveSceneAnalysis(projectId, scene.id, hash, state); + analyzed.push(scene.id); + } catch (error) { + if (!(error instanceof StructuredOutputError)) { + throw error; + } + if (error.failure === "bad_answer") { + this.unusable.add(pair); + skipped.push(scene.id); + continue; + } + message = error.message; + if (signal.aborted) { + status = "cancelled"; + } else { + status = error.failure === "refused" ? "blocked" : "waiting"; + } + break; + } + } + + const left = await storage.listScenesToAnalyze(projectId); + return { + status, + analyzed, + skipped, + pending: left.filter((scene) => !skipped.includes(scene.id)).length, + message, + }; + } +} diff --git a/packages/core/src/analysis/state.ts b/packages/core/src/analysis/state.ts new file mode 100644 index 0000000..5c1b23d --- /dev/null +++ b/packages/core/src/analysis/state.ts @@ -0,0 +1,119 @@ +import type { EntityType, FactCategory } from "./contract"; + +/** + * What the analysis knows about a novel: the registry of SPEC §5, against which new facts are + * compared without rereading the book. Rebuilding a scene replaces what came from that scene. + */ + +export interface Entity { + readonly id: string; + readonly type: EntityType; + readonly canonicalName: string; + readonly aliases: readonly string[]; +} + +export interface Mention { + readonly sceneId: string; + readonly entityId: string; + /** The name as written, which appears literally in the scene. */ + readonly quote: string; +} + +export interface Fact { + readonly id: string; + readonly sceneId: string; + readonly entityId: string; + readonly category: FactCategory; + readonly attribute: string; + readonly value: string; + readonly quote: string; + /** The sentence the quote comes from. A model may split one sentence into several facts. */ + readonly sentence: string; + readonly storyTime: string | null; + /** Said by a character, who may be mistaken or lying, rather than stated by the narrator. */ + readonly inDialogue: boolean; +} + +export interface Relation { + readonly id: string; + readonly sceneId: string; + readonly fromEntityId: string; + readonly toEntityId: string; + readonly label: string; + readonly quote: string; +} + +export interface QuoteRef { + readonly sceneId: string; + readonly quote: string; +} + +export type Issue = + | { + readonly kind: "contradiction"; + readonly a: QuoteRef; + readonly b: QuoteRef; + readonly reason: string; + } + /** Two names that may be one misspelled. The author decides: they're never merged silently. */ + | { + readonly kind: "name_variant"; + readonly a: QuoteRef; + readonly b: QuoteRef; + readonly reason: string; + } + | { + readonly kind: "correction"; + readonly a: QuoteRef; + readonly fix: string; + readonly reason: string; + }; + +export interface AnalysisState { + readonly entities: readonly Entity[]; + readonly mentions: readonly Mention[]; + readonly facts: readonly Fact[]; + readonly relations: readonly Relation[]; + readonly issues: readonly Issue[]; + /** Scenes in reading order, so that a comparison knows what came first. */ + readonly sceneOrder: readonly string[]; +} + +export const emptyState = (sceneOrder: readonly string[] = []): AnalysisState => ({ + entities: [], + mentions: [], + facts: [], + relations: [], + issues: [], + sceneOrder, +}); + +/** + * The entities some scene still supports. The others stay stored, so that a name coming back keeps its + * id and whatever the author said about it (rule 4), but they aren't in the novel now. + */ +export function liveEntities(state: AnalysisState): readonly Entity[] { + const supported = new Set([ + ...state.mentions.map((mention) => mention.entityId), + ...state.facts.map((fact) => fact.entityId), + ]); + return state.entities.filter((entity) => supported.has(entity.id)); +} + +const citesScene = (issue: Issue, sceneId: string) => + issue.a.sceneId === sceneId || ("b" in issue && issue.b.sceneId === sceneId); + +/** + * Takes out everything that came from a scene, for reanalysing it (SPEC §4). What it leaves is exactly + * what the rest of the novel supports: an issue citing this scene goes too, and it comes back if the + * text still justifies it. + */ +export function withoutScene(state: AnalysisState, sceneId: string): AnalysisState { + return { + ...state, + mentions: state.mentions.filter((mention) => mention.sceneId !== sceneId), + facts: state.facts.filter((fact) => fact.sceneId !== sceneId), + relations: state.relations.filter((relation) => relation.sceneId !== sceneId), + issues: state.issues.filter((issue) => !citesScene(issue, sceneId)), + }; +} diff --git a/packages/core/src/analysis/topics.ts b/packages/core/src/analysis/topics.ts new file mode 100644 index 0000000..289e8a0 --- /dev/null +++ b/packages/core/src/analysis/topics.ts @@ -0,0 +1,127 @@ +import { isPartOfName, nameKey, withoutAccents } from "./names"; + +/** + * Finding what to compare and whom to keep apart by reading the text, without any model: the same + * answer every time, whichever model produced the facts, and whichever category it filed them under. + */ + +/** Too common to say what a sentence is about, in the languages the app supports. Times too. */ +const NOT_A_TOPIC = new Set( + `algo alguien alguna alguno aquel aquella aquello aquellos ante antes aqui asi aunque ayer cada casi + como con contra cual cuando desde despues donde durante ella ello ellos entonces entre era eran esa + esas ese eso esos esta estaba estaban estas este esto estos fue fueron habia habian hace hacia hasta + hora horas manana mas mientras mismo misma mucho muy nada nadie noche nunca otra otras otro otros + para pero poco porque pues puede quien quiza segun sido siempre sobre solo suya suyo tambien tanto + tarde tenia tiene toda todas todo todos tras unas unos + ano anos dia dias mes meses semana semanas + lunes martes miercoles jueves viernes sabado domingo + about after again also been before being could from have into just more much never only other over + said some than that their them then there these they this those very were what when where which + while will with would year years month months week weeks today tonight yesterday tomorrow + monday tuesday wednesday thursday friday saturday sunday`.split(/\s+/), +); + +const SHORTEST_TOPIC = 4; + +const normalize = (word: string) => withoutAccents(word.toLocaleLowerCase()).replace(/s$/, ""); + +/** The words of a text that can say what it's about, leaving out any in `excluded`. */ +export function topicWords(text: string, excluded: ReadonlySet = new Set()): Set { + const words = text.match(/\p{L}+/gu) ?? []; + const topics = new Set(); + for (const word of words) { + const plain = withoutAccents(word.toLocaleLowerCase()); + const topic = normalize(word); + if ( + plain.length >= SHORTEST_TOPIC && + !NOT_A_TOPIC.has(plain) && + !NOT_A_TOPIC.has(topic) && + !excluded.has(topic) + ) { + topics.add(topic); + } + } + return topics; +} + +/** The words of a name, in the form `topicWords` compares them. */ +export const nameWords = (name: string): string[] => [...topicWords(name)]; + +/** + * Whether a text talks of someone dying or being dead. It only widens what gets compared, so a false + * positive costs a question, while a miss loses the contradiction. + */ +export function speaksOfDeath(text: string): boolean { + return /\b(enterr|entierro|muri[oe]|muert|fallec|difunt|funeral|velatorio|buried|burial|died|dead|deceased|funeral)/u.test( + withoutAccents(text.toLocaleLowerCase()), + ); +} + +/** The sentences of a text, as written. */ +export function sentencesOf(text: string): string[] { + return (text.match(/[^.!?…\n]+[.!?…]*/gu) ?? []) + .map((sentence) => sentence.trim()) + .filter((sentence) => sentence !== ""); +} + +/** The sentence a quote comes from, or the quote itself if it spans more than one. */ +export function sentenceAround(text: string, quote: string): string { + return sentencesOf(text).find((sentence) => sentence.includes(quote)) ?? quote; +} + +/** Sentences that no quote touches, so that nothing was extracted from them. */ +export function sentencesWithoutFacts(text: string, quotes: readonly string[]): string[] { + return sentencesOf(text).filter( + (sentence) => + (sentence.match(/\p{L}+/gu) ?? []).length >= 3 && + !quotes.some((quote) => sentence.includes(quote) || quote.includes(sentence)), + ); +} + +export interface Apposition { + /** As written: "Luis". */ + readonly name: string; + /** What the text says that person is: "el cartero". */ + readonly epithet: string; + /** Both, as they appear together: "Luis, el cartero". */ + readonly text: string; +} + +/** Capitalized only for opening a sentence: "En Madrid, la capital" is about Madrid. */ +const SENTENCE_OPENERS = + /^(?:A|Al|Ante|Con|Cuando|De|Del|Desde|En|Entre|Hasta|Para|Pero|Por|Que|Si|Sin|Sobre|Tras|Y|And|At|But|By|For|From|In|On|When|With)\s+/u; + +/** A name, then what that person is: "Luis, el cartero". */ +export function appositionsIn(text: string): Apposition[] { + const found = text.matchAll(/(\p{Lu}\p{L}*(?:\s+\p{Lu}\p{L}+)*),\s+((?:el|la|the)\s+\p{Ll}+)/gu); + return [...found].flatMap((match) => { + const written = match[1] ?? ""; + const name = written.replace(SENTENCE_OPENERS, ""); + const epithet = match[2] ?? ""; + return name === "" || SENTENCE_OPENERS.test(`${name} `) + ? [] + : [{ name, epithet, text: `${name}, ${epithet}` }]; + }); +} + +/** + * Pairs the text itself keeps apart: one name inside the other, each with its own description, as in + * "Luis, el cartero" and "Luis Ferrer, el alcalde". The first of each pair has the shorter name. If + * it's really one person with two roles, the author can merge them; a wrong merge would be silent. + */ +export function namesakesIn(text: string): (readonly [Apposition, Apposition])[] { + const found = appositionsIn(text); + const pairs: (readonly [Apposition, Apposition])[] = []; + for (const [index, a] of found.entries()) { + for (const b of found.slice(index + 1)) { + if ( + nameKey(a.name) !== nameKey(b.name) && + nameKey(a.epithet) !== nameKey(b.epithet) && + isPartOfName(a.name, b.name) + ) { + pairs.push(a.name.length <= b.name.length ? [a, b] : [b, a]); + } + } + } + return pairs; +} diff --git a/packages/core/src/export/manuscript.ts b/packages/core/src/export/manuscript.ts index 962029e..eebdc9e 100644 --- a/packages/core/src/export/manuscript.ts +++ b/packages/core/src/export/manuscript.ts @@ -15,7 +15,8 @@ export interface Manuscript { readonly chapters: readonly ManuscriptChapter[]; } -async function readScene(storage: Storage, sceneId: string): Promise { +/** A scene's prose as stored: its base and every update since. */ +export async function readScene(storage: Storage, sceneId: string): Promise { const prose = await storage.loadSceneProse(sceneId); const doc = new Y.Doc(); try { diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 486fa1a..5b1e86f 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,3 +1,17 @@ +export type { AnalysisDeps, SceneAnalysis, SceneToAnalyze } from "./analysis/pipeline"; +export { analyzeScene } from "./analysis/pipeline"; +export type { AnalysisQueueOptions, QueueRun, QueueStatus } from "./analysis/queue"; +export { AnalysisQueue, hashText } from "./analysis/queue"; +export type { + AnalysisState, + Entity, + Fact, + Issue, + Mention, + QuoteRef, + Relation, +} from "./analysis/state"; +export { emptyState, liveEntities, withoutScene } from "./analysis/state"; export { toDocx } from "./export/docx"; export type { Manuscript, ManuscriptChapter } from "./export/manuscript"; export { buildManuscript } from "./export/manuscript"; @@ -15,6 +29,12 @@ export { plainText, readParagraphs, } from "./prose/text"; +export { buildOllamaBody, createOllamaProvider } from "./provider/ollama"; +export { + buildOpenRouterBody, + createOpenRouterProvider, + parseOpenRouterModels, +} from "./provider/openrouter"; export type { AiProvider, BillingKind, @@ -27,8 +47,18 @@ export type { ModelInfo, ProviderError, StructuredOutputMode, + Usage, } from "./provider/provider"; export { collectDraft, unsupportedRequest } from "./provider/provider"; +export type { StructuredFailure, StructuredOptions, Validated } from "./provider/structured"; +export { generateStructured, StructuredOutputError } from "./provider/structured"; +export type { + AiTransport, + ProviderRoute, + TransportRequest, + TransportResponse, +} from "./provider/transport"; +export { readAll, readLines, readSseData } from "./provider/transport"; export type { SqlDriver, SqlResult, SqlRow, SqlStatement, SqlValue } from "./storage/driver"; export type { Migration, MigrationResult } from "./storage/migrate"; export { DatabaseTooNewError, migrate } from "./storage/migrate"; diff --git a/packages/core/src/provider/ollama.ts b/packages/core/src/provider/ollama.ts new file mode 100644 index 0000000..fe3b8eb --- /dev/null +++ b/packages/core/src/provider/ollama.ts @@ -0,0 +1,169 @@ +import type { AiProvider, GenerateEvent, GenerateRequest, ModelInfo, Usage } from "./provider"; +import { unsupportedRequest } from "./provider"; +import { type AiTransport, readAll, readLines } from "./transport"; + +/** Enough for a long scene plus the facts it's compared with. Ollama's default can cut prompts. */ +const CONTEXT_TOKENS = 16384; + +const CAPABILITIES = { structuredOutput: "json_schema", webSearch: false, tools: false } as const; + +type Json = Record; + +function asObject(value: unknown): Json { + return typeof value === "object" && value !== null ? (value as Json) : {}; +} + +function parseJson(text: string): Json | null { + try { + return asObject(JSON.parse(text)); + } catch { + return null; + } +} + +export function buildOllamaBody(request: GenerateRequest): Json { + return { + model: request.model, + stream: true, + messages: [ + { role: "system", content: request.instructions }, + { role: "user", content: request.input }, + ], + // Some models reason unseen by default, which delays any answer by several seconds. + ...(request.reasoning === "off" ? { think: false } : {}), + ...(request.structured === undefined ? {} : { format: request.structured.schema }), + options: { + num_ctx: CONTEXT_TOKENS, + ...(request.temperature === undefined ? {} : { temperature: request.temperature }), + ...(request.maxOutputTokens === undefined ? {} : { num_predict: request.maxOutputTokens }), + }, + }; +} + +export function createOllamaProvider(transport: AiTransport): AiProvider { + const listModels = async (): Promise => { + const response = await transport.send({ route: "ollama", method: "GET", path: "/api/tags" }); + const text = await readAll(response.body); + if (response.status !== 200) { + throw new Error(`The model list returned ${response.status}`); + } + const models = parseJson(text)?.models; + if (!Array.isArray(models)) { + return []; + } + return models.flatMap((entry) => { + const name = asObject(entry).name; + return typeof name === "string" + ? [{ id: name, displayName: name, capabilities: CAPABILITIES }] + : []; + }); + }; + + return { + id: "ollama", + displayName: "Ollama", + billing: "local", + listModels, + + async *generate(request: GenerateRequest): AsyncGenerator { + const unsupported = unsupportedRequest(CAPABILITIES, request); + if (unsupported !== null) { + yield { + type: "end", + outcome: { + status: "failed", + error: { code: "unsupported_capability", message: unsupported }, + }, + }; + return; + } + let usage: Usage | undefined; + try { + // Whatever the transport does with the signal, a cancelled request never reports success. + request.signal?.throwIfAborted(); + const response = await transport.send({ + route: "ollama", + method: "POST", + path: "/api/chat", + body: buildOllamaBody(request), + ...(request.signal === undefined ? {} : { signal: request.signal }), + }); + if (response.status !== 200) { + const text = await readAll(response.body); + const message = parseJson(text)?.error; + yield { + type: "end", + outcome: { + status: "failed", + error: { + code: response.status === 404 ? "unknown_model" : String(response.status), + message: typeof message === "string" ? message : text.slice(0, 300), + httpStatus: response.status, + }, + }, + }; + return; + } + + let done = false; + let reason: string | null = null; + for await (const line of readLines(response.body)) { + request.signal?.throwIfAborted(); + if (line.trim() === "") { + continue; + } + const chunk = parseJson(line); + if (chunk === null) { + continue; + } + // A 200 can still carry a failure, in a line with an error. + if (typeof chunk.error === "string") { + yield { + type: "end", + outcome: { status: "failed", error: { code: null, message: chunk.error } }, + }; + return; + } + const content = asObject(chunk.message).content; + if (typeof content === "string" && content !== "") { + yield { type: "text", delta: content }; + } + if (chunk.done === true) { + done = true; + reason = typeof chunk.done_reason === "string" ? chunk.done_reason : null; + usage = { + inputTokens: + typeof chunk.prompt_eval_count === "number" ? chunk.prompt_eval_count : null, + outputTokens: typeof chunk.eval_count === "number" ? chunk.eval_count : null, + cost: 0, + }; + } + } + const withUsage = usage === undefined ? {} : { usage }; + if (!done) { + yield { + type: "end", + outcome: { status: "interrupted", message: "The stream ended early" }, + ...withUsage, + }; + } else if (reason === "length") { + yield { + type: "end", + outcome: { status: "incomplete", reason: "length" }, + ...withUsage, + }; + } else { + yield { type: "end", outcome: { status: "completed" }, ...withUsage }; + } + } catch (error) { + yield { + type: "end", + outcome: { + status: "interrupted", + message: error instanceof Error ? error.message : String(error), + }, + }; + } + }, + }; +} diff --git a/packages/core/src/provider/openrouter.ts b/packages/core/src/provider/openrouter.ts new file mode 100644 index 0000000..dc11514 --- /dev/null +++ b/packages/core/src/provider/openrouter.ts @@ -0,0 +1,259 @@ +import { + type AiProvider, + type GenerateEvent, + type GenerateRequest, + type ModelInfo, + type ProviderError, + type Usage, + unsupportedRequest, +} from "./provider"; +import { type AiTransport, readAll, readSseData } from "./transport"; + +/** Bounds each request, and leaves room for reasoning, which is spent before any answer. */ +const DEFAULT_MAX_TOKENS = 8000; + +type Json = Record; + +function asObject(value: unknown): Json { + return typeof value === "object" && value !== null ? (value as Json) : {}; +} + +function parseJson(text: string): Json | null { + try { + return asObject(JSON.parse(text)); + } catch { + return null; + } +} + +function toError( + body: Json | null, + fallback: string, + httpStatus: number | undefined, + requestId: string | null, +): ProviderError { + const error = asObject(body?.error); + const code = error.code; + return { + code: code === undefined || code === null ? null : String(code), + message: typeof error.message === "string" ? error.message : fallback, + ...(httpStatus === undefined ? {} : { httpStatus }), + ...(requestId === null ? {} : { requestId }), + }; +} + +export function parseOpenRouterModels(body: unknown): ModelInfo[] { + const data = asObject(body).data; + if (!Array.isArray(data)) { + return []; + } + return data.flatMap((entry) => { + const model = asObject(entry); + if (typeof model.id !== "string") { + return []; + } + const parameters = Array.isArray(model.supported_parameters) ? model.supported_parameters : []; + const pricing = asObject(model.pricing); + return [ + { + id: model.id, + displayName: typeof model.name === "string" ? model.name : model.id, + capabilities: { + structuredOutput: parameters.includes("structured_outputs") ? "json_schema" : "none", + webSearch: parameters.includes("web_search_options"), + tools: parameters.includes("tools"), + }, + free: + model.id.endsWith(":free") && + Number(pricing.prompt ?? 1) === 0 && + Number(pricing.completion ?? 1) === 0, + } satisfies ModelInfo, + ]; + }); +} + +export function buildOpenRouterBody(request: GenerateRequest): Json { + return { + model: request.model, + stream: true, + max_tokens: request.maxOutputTokens ?? DEFAULT_MAX_TOKENS, + messages: [ + { role: "system", content: request.instructions }, + { role: "user", content: request.input }, + ], + ...(request.temperature === undefined ? {} : { temperature: request.temperature }), + ...(request.reasoning === "off" ? { reasoning: { enabled: false } } : {}), + ...(request.structured === undefined + ? {} + : { + response_format: { + type: "json_schema", + json_schema: { + name: request.structured.name, + strict: true, + schema: request.structured.schema, + }, + }, + // Without this, routing could pick an endpoint that ignores the schema, silently. + provider: { require_parameters: true }, + }), + }; +} + +const toUsage = (usage: Json): Usage => ({ + inputTokens: typeof usage.prompt_tokens === "number" ? usage.prompt_tokens : null, + outputTokens: typeof usage.completion_tokens === "number" ? usage.completion_tokens : null, + cost: typeof usage.cost === "number" ? usage.cost : null, +}); + +export function createOpenRouterProvider(transport: AiTransport): AiProvider { + let models: ModelInfo[] | undefined; + + const listModels = async () => { + if (models === undefined) { + const response = await transport.send({ + route: "openrouter", + method: "GET", + path: "/api/v1/models", + }); + const text = await readAll(response.body); + if (response.status !== 200) { + throw new Error(`The model list returned ${response.status}`); + } + models = parseOpenRouterModels(parseJson(text)); + } + return models; + }; + + return { + id: "openrouter", + displayName: "OpenRouter", + billing: "openrouter_credits", + listModels, + + async *generate(request: GenerateRequest): AsyncGenerator { + let usage: Usage | undefined; + const end = (outcome: GenerateEvent & { type: "end" }): GenerateEvent => ({ + ...outcome, + ...(usage === undefined ? {} : { usage }), + }); + try { + const model = (await listModels()).find((candidate) => candidate.id === request.model); + if (model === undefined) { + yield { + type: "end", + outcome: { + status: "failed", + error: { code: "unknown_model", message: `No model "${request.model}"` }, + }, + }; + return; + } + const unsupported = unsupportedRequest(model.capabilities, request); + if (unsupported !== null) { + yield { + type: "end", + outcome: { + status: "failed", + error: { code: "unsupported_capability", message: unsupported }, + }, + }; + return; + } + + // Whatever the transport does with the signal, a cancelled request never reports success. + request.signal?.throwIfAborted(); + const response = await transport.send({ + route: "openrouter", + method: "POST", + path: "/api/v1/chat/completions", + body: buildOpenRouterBody(request), + ...(request.signal === undefined ? {} : { signal: request.signal }), + }); + if (response.status !== 200) { + const text = await readAll(response.body); + yield { + type: "end", + outcome: { + status: "failed", + error: { + ...toError( + parseJson(text), + text.slice(0, 300), + response.status, + response.requestId, + ), + ...(response.retryAfterSeconds === null + ? {} + : { retryAfterSeconds: response.retryAfterSeconds }), + }, + }, + }; + return; + } + + let finish: string | null = null; + let done = false; + for await (const data of readSseData(response.body)) { + request.signal?.throwIfAborted(); + if (data.trim() === "[DONE]") { + done = true; + break; + } + const chunk = parseJson(data); + if (chunk === null) { + continue; + } + if (chunk.usage !== undefined && chunk.usage !== null) { + usage = toUsage(asObject(chunk.usage)); + } + // A 200 can still carry a failure, in a chunk with a top-level error. + if (chunk.error !== undefined) { + yield end({ + type: "end", + outcome: { + status: "failed", + error: toError( + chunk, + "The stream reported an error", + undefined, + response.requestId, + ), + }, + }); + return; + } + const choices = Array.isArray(chunk.choices) ? chunk.choices : []; + const choice = asObject(choices[0]); + const delta = asObject(choice.delta); + if (typeof delta.content === "string" && delta.content !== "") { + yield { type: "text", delta: delta.content }; + } + if (typeof choice.finish_reason === "string") { + finish = choice.finish_reason; + } + } + + if (!done || finish === null) { + yield end({ + type: "end", + outcome: { status: "interrupted", message: "The stream ended early" }, + }); + } else if (finish === "stop") { + yield end({ type: "end", outcome: { status: "completed" } }); + } else { + // Length, a content filter or anything new: only a clean stop counts as a whole answer. + yield end({ type: "end", outcome: { status: "incomplete", reason: finish } }); + } + } catch (error) { + yield end({ + type: "end", + outcome: { + status: "interrupted", + message: error instanceof Error ? error.message : String(error), + }, + }); + } + }, + }; +} diff --git a/packages/core/src/provider/provider.ts b/packages/core/src/provider/provider.ts index 5c9511c..c994ac2 100644 --- a/packages/core/src/provider/provider.ts +++ b/packages/core/src/provider/provider.ts @@ -23,6 +23,8 @@ export interface ModelInfo { readonly id: string; readonly displayName: string; readonly capabilities: ModelCapabilities; + /** True only if the provider states that using it costs nothing. */ + readonly free?: boolean; } export interface GenerateRequest { @@ -35,9 +37,20 @@ export interface GenerateRequest { }; /** Each search costs money, and one request can run several, so the cap is mandatory. */ readonly webSearch?: { readonly maxSearches: number }; + readonly temperature?: number; + readonly maxOutputTokens?: number; + /** "off" skips the reasoning some models do before answering, which costs time and tokens. */ + readonly reasoning?: "default" | "off"; readonly signal?: AbortSignal; } +export interface Usage { + readonly inputTokens: number | null; + readonly outputTokens: number | null; + /** In dollars, when the provider says. */ + readonly cost: number | null; +} + export interface Citation { readonly url: string; readonly title: string | null; @@ -49,6 +62,7 @@ export interface ProviderError { /** Kept for support, as the ChatGPT route asks for. */ readonly requestId?: string; readonly httpStatus?: number; + readonly retryAfterSeconds?: number; } /** Only `completed` is success. Any other ending leaves, at most, a partial draft. */ @@ -61,7 +75,7 @@ export type GenerateOutcome = export type GenerateEvent = | { readonly type: "text"; readonly delta: string } | { readonly type: "citation"; readonly citation: Citation } - | { readonly type: "end"; readonly outcome: GenerateOutcome }; + | { readonly type: "end"; readonly outcome: GenerateOutcome; readonly usage?: Usage }; export interface AiProvider { readonly id: string; @@ -99,6 +113,7 @@ export interface Draft { readonly outcome: GenerateOutcome; /** True only after a `completed` ending. Anything else can't be treated as a finished result. */ readonly completed: boolean; + readonly usage?: Usage; } /** @@ -112,6 +127,7 @@ export async function collectDraft( let text = ""; const citations: Citation[] = []; let outcome: GenerateOutcome | undefined; + let usage: Usage | undefined; try { for await (const event of events) { if (event.type === "text") { @@ -121,6 +137,7 @@ export async function collectDraft( citations.push(event.citation); } else { outcome = event.outcome; + usage = event.usage; break; } } @@ -134,5 +151,11 @@ export async function collectDraft( status: "interrupted" as const, message: "The stream ended without an ending event", }; - return { text, citations, outcome: final, completed: final.status === "completed" }; + return { + text, + citations, + outcome: final, + completed: final.status === "completed", + ...(usage === undefined ? {} : { usage }), + }; } diff --git a/packages/core/src/provider/structured.ts b/packages/core/src/provider/structured.ts new file mode 100644 index 0000000..c3d73ee --- /dev/null +++ b/packages/core/src/provider/structured.ts @@ -0,0 +1,130 @@ +import { type AiProvider, collectDraft, type GenerateRequest, type Usage } from "./provider"; + +/** A value, or why it doesn't fit. */ +export type Validated = + | { readonly ok: true; readonly value: T } + | { readonly ok: false; readonly problem: string }; + +/** + * What to do about a failure. `later`: nothing's wrong with the work, as with no connection or a rate + * limit. `refused`: it won't work until the setup changes, as with a bad key or no credit. + * `bad_answer`: the model answered, but not with anything usable, even after the retry. + */ +export type StructuredFailure = "later" | "refused" | "bad_answer"; + +export class StructuredOutputError extends Error { + override readonly name = "StructuredOutputError"; + + constructor( + message: string, + readonly failure: StructuredFailure, + ) { + super(message); + } +} + +export interface StructuredOptions { + readonly sleep?: (ms: number, signal?: AbortSignal) => Promise; + readonly onUsage?: (usage: Usage) => void; +} + +const MAX_TRANSIENT_RETRIES = 4; +const FIRST_WAIT_MS = 5000; +const LONGEST_WAIT_MS = 60_000; + +const TRANSIENT = new Set(["408", "429", "500", "502", "503", "504"]); + +/** Ends early if cancelled, so that closing the app doesn't wait out a rate limit. */ +function abortableSleep(ms: number, signal?: AbortSignal): Promise { + return new Promise((resolve) => { + if (signal?.aborted) { + resolve(); + return; + } + const timer = setTimeout(resolve, ms); + signal?.addEventListener( + "abort", + () => { + clearTimeout(timer); + resolve(); + }, + { once: true }, + ); + }); +} + +function parse(text: string): unknown { + // Some models wrap their JSON in a code fence, even when told not to. + const unfenced = text + .trim() + .replace(/^```(?:json)?\s*/i, "") + .replace(/\s*```$/, ""); + return JSON.parse(unfenced); +} + +/** + * Gets output that passes `validate`. SPEC §5: if it doesn't fit, one retry that says what was wrong. + * Rate limits and server errors are waited out a few times, and don't use up that retry. + */ +export async function generateStructured( + provider: AiProvider, + request: GenerateRequest, + validate: (value: unknown) => Validated, + options: StructuredOptions = {}, +): Promise { + const sleep = options.sleep ?? abortableSleep; + let input = request.input; + let repaired = false; + let transientFailures = 0; + + for (;;) { + const draft = await collectDraft(provider.generate({ ...request, input })); + if (draft.usage !== undefined) { + options.onUsage?.(draft.usage); + } + + let problem: string; + if (draft.outcome.status === "failed") { + const { error } = draft.outcome; + const transient = error.code !== null && TRANSIENT.has(error.code); + const wait = error.retryAfterSeconds + ? error.retryAfterSeconds * 1000 + : Math.min(FIRST_WAIT_MS * 2 ** transientFailures, LONGEST_WAIT_MS); + // Asked to wait for longer than that, as with a daily limit, the work waits instead of a request. + if (transient && transientFailures < MAX_TRANSIENT_RETRIES && wait <= LONGEST_WAIT_MS) { + transientFailures += 1; + await sleep(wait, request.signal); + continue; + } + throw new StructuredOutputError( + `${error.code ?? "error"}: ${error.message}`, + transient || error.code === null ? "later" : "refused", + ); + } + if (draft.outcome.status === "interrupted") { + throw new StructuredOutputError(draft.outcome.message, "later"); + } + if (draft.outcome.status === "incomplete") { + problem = "The answer was cut off before it ended. Give a shorter answer."; + } else { + try { + const checked = validate(parse(draft.text)); + if (checked.ok) { + return checked.value; + } + problem = checked.problem; + } catch { + problem = "The answer wasn't valid JSON."; + } + } + + if (repaired) { + throw new StructuredOutputError( + `The answer didn't fit after a retry: ${problem}`, + "bad_answer", + ); + } + repaired = true; + input = `${request.input}\n\nYour previous answer was rejected: ${problem}\nAnswer again with only the JSON object.`; + } +} diff --git a/packages/core/src/provider/transport.ts b/packages/core/src/provider/transport.ts new file mode 100644 index 0000000..0f68741 --- /dev/null +++ b/packages/core/src/provider/transport.ts @@ -0,0 +1,73 @@ +/** + * How adapters reach the network. Whoever implements this attaches the credentials, so the core and + * the WebView never see them (rule 9). The desktop shell does it in Rust; the evaluation harness, in + * Node. + */ + +export type ProviderRoute = "openrouter" | "ollama"; + +export interface TransportRequest { + readonly route: ProviderRoute; + readonly method: "GET" | "POST"; + readonly path: string; + readonly body?: unknown; + readonly signal?: AbortSignal; +} + +export interface TransportResponse { + readonly status: number; + readonly requestId: string | null; + /** From the `Retry-After` header, if there was one. */ + readonly retryAfterSeconds: number | null; + readonly body: AsyncIterable; +} + +export interface AiTransport { + /** Rejects if the service can't be reached at all, as when offline or if Ollama isn't running. */ + send(request: TransportRequest): Promise; +} + +export async function readAll(body: AsyncIterable): Promise { + const decoder = new TextDecoder(); + let text = ""; + for await (const chunk of body) { + text += decoder.decode(chunk, { stream: true }); + } + return text + decoder.decode(); +} + +/** Splits into lines, whatever the chunks. A character split between chunks stays in one piece. */ +export async function* readLines(body: AsyncIterable): AsyncGenerator { + const decoder = new TextDecoder(); + let pending = ""; + for await (const chunk of body) { + pending += decoder.decode(chunk, { stream: true }); + const lines = pending.split("\n"); + pending = lines.pop() ?? ""; + for (const line of lines) { + yield line.endsWith("\r") ? line.slice(0, -1) : line; + } + } + pending += decoder.decode(); + if (pending !== "") { + yield pending; + } +} + +/** The `data:` payloads of a server-sent event stream. Comments and other fields are skipped. */ +export async function* readSseData(body: AsyncIterable): AsyncGenerator { + let data: string[] = []; + for await (const line of readLines(body)) { + if (line === "") { + if (data.length > 0) { + yield data.join("\n"); + data = []; + } + } else if (line.startsWith("data:")) { + data.push(line.slice(5).replace(/^ /, "")); + } + } + if (data.length > 0) { + yield data.join("\n"); + } +} diff --git a/packages/core/src/storage/migrations.ts b/packages/core/src/storage/migrations.ts index 6257fe2..9cc859b 100644 --- a/packages/core/src/storage/migrations.ts +++ b/packages/core/src/storage/migrations.ts @@ -62,4 +62,66 @@ export const MIGRATIONS: readonly Migration[] = [ "CREATE INDEX scene_snapshots_by_scene ON scene_snapshots (scene_id, created_at)", ], }, + { + version: 3, + name: "analysis", + // See docs/adr/0011. Quotes rather than Yjs anchors for now: M4 is what needs to place them. + statements: [ + "ALTER TABLE scenes ADD COLUMN content_hash TEXT", + "ALTER TABLE scenes ADD COLUMN analyzed_hash TEXT", + `CREATE TABLE entities ( + id TEXT PRIMARY KEY, + project_id TEXT NOT NULL REFERENCES projects (id) ON DELETE RESTRICT, + type TEXT NOT NULL, + canonical_name TEXT NOT NULL + ) STRICT`, + "CREATE INDEX entities_by_project ON entities (project_id)", + `CREATE TABLE entity_aliases ( + entity_id TEXT NOT NULL REFERENCES entities (id) ON DELETE RESTRICT, + alias TEXT NOT NULL, + source TEXT NOT NULL, + PRIMARY KEY (entity_id, alias) + ) STRICT`, + `CREATE TABLE mentions ( + scene_id TEXT NOT NULL REFERENCES scenes (id) ON DELETE RESTRICT, + entity_id TEXT NOT NULL REFERENCES entities (id) ON DELETE RESTRICT, + quote TEXT NOT NULL, + PRIMARY KEY (scene_id, entity_id, quote) + ) STRICT`, + `CREATE TABLE facts ( + id TEXT PRIMARY KEY, + scene_id TEXT NOT NULL REFERENCES scenes (id) ON DELETE RESTRICT, + entity_id TEXT NOT NULL REFERENCES entities (id) ON DELETE RESTRICT, + category TEXT NOT NULL, + attribute TEXT NOT NULL, + value TEXT NOT NULL, + quote TEXT NOT NULL, + sentence TEXT NOT NULL, + story_time TEXT, + in_dialogue INTEGER NOT NULL CHECK (in_dialogue IN (0, 1)) + ) STRICT`, + "CREATE INDEX facts_by_scene ON facts (scene_id)", + `CREATE TABLE relations ( + id TEXT PRIMARY KEY, + scene_id TEXT NOT NULL REFERENCES scenes (id) ON DELETE RESTRICT, + from_entity TEXT NOT NULL REFERENCES entities (id) ON DELETE RESTRICT, + to_entity TEXT NOT NULL REFERENCES entities (id) ON DELETE RESTRICT, + label TEXT NOT NULL, + quote TEXT NOT NULL + ) STRICT`, + "CREATE INDEX relations_by_scene ON relations (scene_id)", + `CREATE TABLE issues ( + id TEXT PRIMARY KEY, + kind TEXT NOT NULL, + scene_a TEXT NOT NULL REFERENCES scenes (id) ON DELETE RESTRICT, + quote_a TEXT NOT NULL, + scene_b TEXT REFERENCES scenes (id) ON DELETE RESTRICT, + quote_b TEXT, + fix TEXT, + reason TEXT NOT NULL + ) STRICT`, + "CREATE INDEX issues_by_scene_a ON issues (scene_a)", + "CREATE INDEX issues_by_scene_b ON issues (scene_b)", + ], + }, ]; diff --git a/packages/core/src/storage/sql-storage.ts b/packages/core/src/storage/sql-storage.ts index 1a4d8a6..55d3f0f 100644 --- a/packages/core/src/storage/sql-storage.ts +++ b/packages/core/src/storage/sql-storage.ts @@ -1,4 +1,14 @@ -import type { SqlDriver, SqlRow, SqlValue } from "./driver"; +import type { EntityType, FactCategory } from "../analysis/contract"; +import type { + AnalysisState, + Entity, + Fact, + Issue, + Mention, + QuoteRef, + Relation, +} from "../analysis/state"; +import type { SqlDriver, SqlRow, SqlStatement, SqlValue } from "./driver"; import { type MigrationResult, migrate } from "./migrate"; import { type Chapter, @@ -89,6 +99,51 @@ const toSnapshotInfo = (row: SqlRow): SnapshotInfo => ({ wordCount: integer(row, "word_count"), }); +const toMention = (row: SqlRow): Mention => ({ + sceneId: text(row, "scene_id"), + entityId: text(row, "entity_id"), + quote: text(row, "quote"), +}); + +const toFact = (row: SqlRow): Fact => ({ + id: text(row, "id"), + sceneId: text(row, "scene_id"), + entityId: text(row, "entity_id"), + category: text(row, "category") as FactCategory, + attribute: text(row, "attribute"), + value: text(row, "value"), + quote: text(row, "quote"), + sentence: text(row, "sentence"), + storyTime: textOrNull(row, "story_time"), + inDialogue: integer(row, "in_dialogue") === 1, +}); + +const toRelation = (row: SqlRow): Relation => ({ + id: text(row, "id"), + sceneId: text(row, "scene_id"), + fromEntityId: text(row, "from_entity"), + toEntityId: text(row, "to_entity"), + label: text(row, "label"), + quote: text(row, "quote"), +}); + +function toIssue(row: SqlRow): Issue { + const a: QuoteRef = { sceneId: text(row, "scene_a"), quote: text(row, "quote_a") }; + const reason = text(row, "reason"); + const kind = text(row, "kind"); + if (kind === "correction") { + return { kind, a, fix: text(row, "fix"), reason }; + } + if (kind !== "contradiction" && kind !== "name_variant") { + throw new Error(`Unknown issue kind "${kind}"`); + } + return { kind, a, b: { sceneId: text(row, "scene_b"), quote: text(row, "quote_b") }, reason }; +} + +/** The project's scenes that aren't in the trash, by themselves or through their chapter. */ +const LIVE_SCENES = `SELECT s.id FROM scenes s JOIN chapters c ON c.id = s.chapter_id + WHERE c.project_id = ? AND s.deleted_at IS NULL AND c.deleted_at IS NULL`; + /** The columns of a snapshot without its data, which can be large. */ const SNAPSHOT_INFO = "id, scene_id, created_at, label, word_count"; @@ -329,6 +384,167 @@ export function createSqlStorage(driver: SqlDriver, options: SqlStorageOptions = const [row] = await driver.query("SELECT ydoc FROM scene_snapshots WHERE id = ?", [id]); return row === undefined ? null : blob(row, "ydoc"); }, + + async setSceneContentHash(sceneId: string, hash: string) { + await driver.execute("UPDATE scenes SET content_hash = ? WHERE id = ?", [hash, sceneId]); + }, + + async listScenesToAnalyze(projectId: string) { + const rows = await driver.query( + `SELECT s.id, s.chapter_id, s.position, s.title, s.status FROM scenes s + JOIN chapters c ON c.id = s.chapter_id + WHERE c.project_id = ? AND s.deleted_at IS NULL AND c.deleted_at IS NULL + AND s.content_hash IS NOT NULL AND s.content_hash IS NOT s.analyzed_hash + ORDER BY c.position, c.id, s.position, s.id`, + [projectId], + ); + return rows.map(toScene); + }, + + async loadAnalysis(projectId: string) { + const order = await driver.query( + `SELECT s.id FROM scenes s JOIN chapters c ON c.id = s.chapter_id + WHERE c.project_id = ? AND s.deleted_at IS NULL AND c.deleted_at IS NULL + ORDER BY c.position, c.id, s.position, s.id`, + [projectId], + ); + const entityRows = await driver.query( + "SELECT id, type, canonical_name FROM entities WHERE project_id = ? ORDER BY rowid", + [projectId], + ); + const aliasRows = await driver.query( + `SELECT a.entity_id, a.alias FROM entity_aliases a JOIN entities e ON e.id = a.entity_id + WHERE e.project_id = ? ORDER BY a.rowid`, + [projectId], + ); + const inLiveScene = `scene_id IN (${LIVE_SCENES})`; + const mentions = await driver.query( + `SELECT * FROM mentions WHERE ${inLiveScene} ORDER BY rowid`, + [projectId], + ); + const facts = await driver.query(`SELECT * FROM facts WHERE ${inLiveScene} ORDER BY rowid`, [ + projectId, + ]); + const relations = await driver.query( + `SELECT * FROM relations WHERE ${inLiveScene} ORDER BY rowid`, + [projectId], + ); + const issues = await driver.query( + `SELECT * FROM issues WHERE scene_a IN (${LIVE_SCENES}) + AND (scene_b IS NULL OR scene_b IN (${LIVE_SCENES})) ORDER BY rowid`, + [projectId, projectId], + ); + const entities: Entity[] = entityRows.map((row) => ({ + id: text(row, "id"), + type: text(row, "type") as EntityType, + canonicalName: text(row, "canonical_name"), + aliases: aliasRows + .filter((alias) => alias.entity_id === row.id) + .map((alias) => text(alias, "alias")), + })); + return { + entities, + mentions: mentions.map(toMention), + facts: facts.map(toFact), + relations: relations.map(toRelation), + issues: issues.map(toIssue), + sceneOrder: order.map((row) => text(row, "id")), + }; + }, + + async saveSceneAnalysis( + projectId: string, + sceneId: string, + analyzedHash: string, + state: AnalysisState, + ) { + const statements: SqlStatement[] = []; + for (const entity of state.entities) { + statements.push({ + sql: `INSERT INTO entities (id, project_id, type, canonical_name) VALUES (?, ?, ?, ?) + ON CONFLICT (id) DO NOTHING`, + params: [entity.id, projectId, entity.type, entity.canonicalName], + }); + for (const alias of entity.aliases) { + statements.push({ + sql: `INSERT INTO entity_aliases (entity_id, alias, source) VALUES (?, ?, 'analysis') + ON CONFLICT DO NOTHING`, + params: [entity.id, alias], + }); + } + } + statements.push( + { sql: "DELETE FROM mentions WHERE scene_id = ?", params: [sceneId] }, + { sql: "DELETE FROM facts WHERE scene_id = ?", params: [sceneId] }, + { sql: "DELETE FROM relations WHERE scene_id = ?", params: [sceneId] }, + { sql: "DELETE FROM issues WHERE scene_a = ? OR scene_b = ?", params: [sceneId, sceneId] }, + ); + for (const mention of state.mentions.filter((item) => item.sceneId === sceneId)) { + statements.push({ + sql: `INSERT INTO mentions (scene_id, entity_id, quote) VALUES (?, ?, ?) + ON CONFLICT DO NOTHING`, + params: [sceneId, mention.entityId, mention.quote], + }); + } + for (const fact of state.facts.filter((item) => item.sceneId === sceneId)) { + statements.push({ + sql: `INSERT INTO facts + (id, scene_id, entity_id, category, attribute, value, quote, sentence, story_time, + in_dialogue) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + params: [ + fact.id, + sceneId, + fact.entityId, + fact.category, + fact.attribute, + fact.value, + fact.quote, + fact.sentence, + fact.storyTime, + fact.inDialogue ? 1 : 0, + ], + }); + } + for (const relation of state.relations.filter((item) => item.sceneId === sceneId)) { + statements.push({ + sql: `INSERT INTO relations (id, scene_id, from_entity, to_entity, label, quote) + VALUES (?, ?, ?, ?, ?, ?)`, + params: [ + relation.id, + sceneId, + relation.fromEntityId, + relation.toEntityId, + relation.label, + relation.quote, + ], + }); + } + const citesScene = (issue: Issue) => + issue.a.sceneId === sceneId || ("b" in issue && issue.b.sceneId === sceneId); + for (const issue of state.issues.filter(citesScene)) { + const b = "b" in issue ? issue.b : null; + statements.push({ + sql: `INSERT INTO issues (id, kind, scene_a, quote_a, scene_b, quote_b, fix, reason) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + params: [ + newId(), + issue.kind, + issue.a.sceneId, + issue.a.quote, + b?.sceneId ?? null, + b?.quote ?? null, + "fix" in issue ? issue.fix : null, + issue.reason, + ], + }); + } + statements.push({ + sql: "UPDATE scenes SET analyzed_hash = ? WHERE id = ?", + params: [analyzedHash, sceneId], + }); + await driver.transaction(statements); + }, }; } diff --git a/packages/core/src/storage/storage.ts b/packages/core/src/storage/storage.ts index 4a9e373..598166a 100644 --- a/packages/core/src/storage/storage.ts +++ b/packages/core/src/storage/storage.ts @@ -1,3 +1,5 @@ +import type { AnalysisState } from "../analysis/state"; + export interface Project { readonly id: string; readonly title: string; @@ -117,4 +119,22 @@ export interface Storage { /** Newest first. */ listSnapshots(sceneId: string): Promise; loadSnapshot(id: string): Promise; + + /** What a scene's text hashes to now. It waits for analysis while this differs from the last one. */ + setSceneContentHash(sceneId: string, hash: string): Promise; + /** Scenes with text not yet analysed, in reading order, without the ones in the trash. */ + listScenesToAnalyze(projectId: string): Promise; + /** What's known of the novel, with what came from scenes in the trash left out. */ + loadAnalysis(projectId: string): Promise; + /** + * Stores what analysing a scene produced, all or nothing: its data replaces the scene's old data + * (SPEC §4), and `analyzedHash` says which text it describes. If the author kept writing meanwhile, + * the scene stays in the queue. An entity already stored keeps its name: only its aliases can grow. + */ + saveSceneAnalysis( + projectId: string, + sceneId: string, + analyzedHash: string, + state: AnalysisState, + ): Promise; } diff --git a/packages/core/test/analysis/adapters.test.ts b/packages/core/test/analysis/adapters.test.ts new file mode 100644 index 0000000..568c6ef --- /dev/null +++ b/packages/core/test/analysis/adapters.test.ts @@ -0,0 +1,410 @@ +import { describe, expect, it } from "vitest"; +import { + type AiTransport, + buildOllamaBody, + buildOpenRouterBody, + collectDraft, + createOllamaProvider, + createOpenRouterProvider, + type GenerateEvent, + parseOpenRouterModels, + type TransportRequest, +} from "../../src/index"; +import { describeProviderContract } from "../provider-contract"; + +const encode = (text: string) => new TextEncoder().encode(text); + +async function* chunks(...parts: readonly string[]) { + for (const part of parts) { + yield encode(part); + } +} + +function response(status: number, ...parts: readonly string[]) { + return { status, requestId: "req-1", retryAfterSeconds: null, body: chunks(...parts) }; +} + +/** Answers from a list of handlers by path, and remembers what it was asked. */ +function fakeTransport(handlers: Readonly ReturnType>>) { + const sent: TransportRequest[] = []; + const transport: AiTransport = { + async send(request) { + sent.push(request); + const handler = handlers[request.path]; + if (handler === undefined) { + throw new Error(`No handler for ${request.path}`); + } + return handler(); + }, + }; + return { transport, sent }; +} + +const MODELS = JSON.stringify({ + data: [ + { + id: "full/model:free", + name: "Full", + supported_parameters: ["structured_outputs", "tools", "web_search_options"], + pricing: { prompt: "0", completion: "0" }, + }, + { + id: "bare/model", + name: "Bare", + supported_parameters: [], + pricing: { prompt: "0.000001", completion: "0.000002" }, + }, + ], +}); + +const STREAM_OK = [ + ": OPENROUTER PROCESSING\n\n", + 'data: {"choices":[{"delta":{"content":"Ho"}}]}\n\n', + 'data: {"choices":[{"delta":{"content":"la"}}]}\n\n', + 'data: {"choices":[{"delta":{},"finish_reason":"stop"}],"usage":{"prompt_tokens":3,"completion_tokens":2,"cost":0}}\n\n', + "data: [DONE]\n\n", +]; + +const request = { model: "full/model:free", instructions: "i", input: "x" }; + +const list = async (events: AsyncIterable) => { + const out: GenerateEvent[] = []; + for await (const event of events) { + out.push(event); + } + return out; +}; + +describeProviderContract("OpenRouter, with a fake network", () => { + const { transport } = fakeTransport({ + "/api/v1/models": () => response(200, MODELS), + "/api/v1/chat/completions": () => response(200, ...STREAM_OK), + }); + return createOpenRouterProvider(transport); +}); + +describeProviderContract("Ollama, with a fake network", () => { + // Unlike OpenRouter, the adapter doesn't look models up first: the server is who refuses one. + const transport: AiTransport = { + send: async (sent) => { + if (sent.path === "/api/tags") { + return response(200, JSON.stringify({ models: [{ name: "m:1" }] })); + } + return (sent.body as { model: string }).model === "m:1" + ? response( + 200, + '{"message":{"content":"x"},"done":false}\n{"done":true,"done_reason":"stop"}\n', + ) + : response(404, '{"error":"model not found"}'); + }, + }; + return createOllamaProvider(transport); +}); + +describe("OpenRouter models", () => { + it("reads capabilities per model", () => { + const [full, bare] = parseOpenRouterModels(JSON.parse(MODELS)); + + expect(full?.capabilities).toEqual({ + structuredOutput: "json_schema", + webSearch: true, + tools: true, + }); + expect(bare?.capabilities.structuredOutput).toBe("none"); + }); + + it.each([ + ["a/b:free", "0", "0", true], + ["a/b:free", "0.1", "0", false], + ["a/b:free", "0", "0.1", false], + ["a/b", "0", "0", false], + ["a/b:free", undefined, undefined, false], + ])("%s at %s and %s: free is %s", (id, prompt, completion, free) => { + const [model] = parseOpenRouterModels({ data: [{ id, pricing: { prompt, completion } }] }); + + expect(model?.free).toBe(free); + }); + + it("copes with a malformed answer", () => { + expect(parseOpenRouterModels({})).toEqual([]); + expect(parseOpenRouterModels({ data: [{ name: "no id" }] })).toEqual([]); + }); +}); + +describe("OpenRouter requests", () => { + it("asks for the schema and refuses endpoints that would ignore it", () => { + const body = buildOpenRouterBody({ + ...request, + structured: { name: "n", schema: { type: "object" } }, + }); + + expect(body.response_format).toMatchObject({ type: "json_schema" }); + expect(body.provider).toEqual({ require_parameters: true }); + }); + + it("never asks for a paid web search by itself", () => { + const body = JSON.stringify(buildOpenRouterBody(request)); + + expect(body).not.toContain("web_search"); + expect(body).not.toContain("plugins"); + expect(body).not.toContain("tools"); + }); + + it("bounds the answer, and can turn reasoning off", () => { + const body = buildOpenRouterBody({ ...request, reasoning: "off", temperature: 0 }); + + expect(body.max_tokens).toBeGreaterThan(0); + expect(body.reasoning).toEqual({ enabled: false }); + expect(body.temperature).toBe(0); + expect(buildOpenRouterBody(request).reasoning).toBeUndefined(); + }); +}); + +describe("OpenRouter streaming", () => { + const provider = (...parts: readonly string[]) => + createOpenRouterProvider( + fakeTransport({ + "/api/v1/models": () => response(200, MODELS), + "/api/v1/chat/completions": () => response(200, ...parts), + }).transport, + ); + + it("ignores comments, and reports usage with the cost", async () => { + const draft = await collectDraft(provider(...STREAM_OK).generate(request)); + + expect(draft).toMatchObject({ text: "Hola", completed: true }); + expect(draft.usage).toEqual({ inputTokens: 3, outputTokens: 2, cost: 0 }); + }); + + it("treats an error chunk as a failure, even under a 200", async () => { + const draft = await collectDraft( + provider( + 'data: {"choices":[{"delta":{"content":"par"}}]}\n\n', + 'data: {"error":{"code":429,"message":"Rate limit"},"choices":[{"delta":{},"finish_reason":"error"}]}\n\n', + ).generate(request), + ); + + expect(draft.completed).toBe(false); + expect(draft.text).toBe("par"); + expect(draft.outcome).toMatchObject({ status: "failed", error: { code: "429" } }); + }); + + it("treats running out of tokens as incomplete, not as success", async () => { + const draft = await collectDraft( + provider( + 'data: {"choices":[{"delta":{},"finish_reason":"length"}]}\n\n', + "data: [DONE]\n\n", + ).generate(request), + ); + + expect(draft.outcome).toEqual({ status: "incomplete", reason: "length" }); + }); + + it("doesn't take an answer stopped by a content filter as a whole one", async () => { + const draft = await collectDraft( + provider( + 'data: {"choices":[{"delta":{"content":"a"},"finish_reason":"content_filter"}]}\n\n', + "data: [DONE]\n\n", + ).generate(request), + ); + + expect(draft.outcome).toEqual({ status: "incomplete", reason: "content_filter" }); + }); + + it("stops reading when cancelled halfway, and doesn't report success", async () => { + const controller = new AbortController(); + const { transport } = fakeTransport({ "/api/v1/models": () => response(200, MODELS) }); + const original = transport.send; + async function* cancelledHalfway() { + yield encode('data: {"choices":[{"delta":{"content":"a"}}]}\n\n'); + controller.abort(); + yield encode('data: {"choices":[{"delta":{"content":"b"}}]}\n\n'); + yield encode('data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n'); + } + const cancelling: AiTransport = { + send: (r) => + r.path === "/api/v1/chat/completions" + ? Promise.resolve({ ...response(200), body: cancelledHalfway() }) + : original(r), + }; + + const draft = await collectDraft( + createOpenRouterProvider(cancelling).generate({ ...request, signal: controller.signal }), + ); + + expect(draft.text).toBe("a"); + expect(draft.outcome.status).toBe("interrupted"); + }); + + it("treats a cut stream as interrupted", async () => { + const draft = await collectDraft( + provider('data: {"choices":[{"delta":{"content":"a medias"}}]}\n\n').generate(request), + ); + + expect(draft.outcome.status).toBe("interrupted"); + expect(draft.text).toBe("a medias"); + }); + + it("keeps a character split between chunks", async () => { + const bytes = encode('data: {"choices":[{"delta":{"content":"ñ"}}]}\n\n'); + const at = bytes.indexOf(0xc3) + 1; + async function* split() { + yield bytes.slice(0, at); + yield bytes.slice(at); + yield encode('data: {"choices":[{"delta":{},"finish_reason":"stop"}]}\n\ndata: [DONE]\n\n'); + } + const { transport } = fakeTransport({ "/api/v1/models": () => response(200, MODELS) }); + const original = transport.send; + const split429: AiTransport = { + send: (r) => + r.path === "/api/v1/chat/completions" + ? Promise.resolve({ ...response(200), body: split() }) + : original(r), + }; + + const draft = await collectDraft(createOpenRouterProvider(split429).generate(request)); + + expect(draft.text).toBe("ñ"); + }); + + it("reports a refusal with its code, its request ID and how long to wait", async () => { + const { transport } = fakeTransport({ "/api/v1/models": () => response(200, MODELS) }); + const original = transport.send; + const limited: AiTransport = { + send: (r) => + r.path === "/api/v1/chat/completions" + ? Promise.resolve({ + ...response(429, '{"error":{"code":429,"message":"Rate limit exceeded"}}'), + retryAfterSeconds: 30, + }) + : original(r), + }; + + const draft = await collectDraft(createOpenRouterProvider(limited).generate(request)); + + expect(draft.outcome).toEqual({ + status: "failed", + error: { + code: "429", + message: "Rate limit exceeded", + httpStatus: 429, + requestId: "req-1", + retryAfterSeconds: 30, + }, + }); + }); + + it("sends nothing for what the model can't do, so a refusal costs nothing", async () => { + const { transport, sent } = fakeTransport({ "/api/v1/models": () => response(200, MODELS) }); + + const events = await list( + createOpenRouterProvider(transport).generate({ + ...request, + model: "bare/model", + structured: { name: "n", schema: {} }, + }), + ); + + expect(events.at(-1)).toMatchObject({ + outcome: { status: "failed", error: { code: "unsupported_capability" } }, + }); + expect(sent.map((r) => r.path)).toEqual(["/api/v1/models"]); + }); + + it("reports no connection as interrupted, so that the work can wait", async () => { + const offline: AiTransport = { + send: async () => { + throw new Error("fetch failed"); + }, + }; + + const draft = await collectDraft(createOpenRouterProvider(offline).generate(request)); + + expect(draft.outcome).toMatchObject({ status: "interrupted", message: "fetch failed" }); + }); +}); + +describe("Ollama", () => { + const generate = (...lines: readonly string[]) => + createOllamaProvider( + fakeTransport({ "/api/chat": () => response(200, ...lines) }).transport, + ).generate({ model: "m", instructions: "i", input: "x" }); + + it("sends the schema, and reserves room for the prompt", () => { + const body = buildOllamaBody({ + model: "m", + instructions: "i", + input: "x", + structured: { name: "n", schema: { type: "object" } }, + }); + + expect(body.format).toEqual({ type: "object" }); + expect(body.options).toMatchObject({ num_ctx: expect.any(Number) }); + }); + + it("only turns reasoning off when asked", () => { + const base = { model: "m", instructions: "i", input: "x" }; + + expect(buildOllamaBody(base).think).toBeUndefined(); + expect(buildOllamaBody({ ...base, reasoning: "off" }).think).toBe(false); + }); + + it("completes on the last line, and reports no cost", async () => { + const draft = await collectDraft( + generate( + '{"message":{"content":"Ho"},"done":false}\n', + '{"message":{"content":"la"},"done":false}\n', + '{"message":{"content":""},"done":true,"done_reason":"stop","prompt_eval_count":5,"eval_count":2}\n', + ), + ); + + expect(draft).toMatchObject({ text: "Hola", completed: true }); + expect(draft.usage).toEqual({ inputTokens: 5, outputTokens: 2, cost: 0 }); + }); + + it("treats an error line as a failure, keeping the partial text", async () => { + const draft = await collectDraft( + generate('{"message":{"content":"par"},"done":false}\n', '{"error":"out of memory"}\n'), + ); + + expect(draft.text).toBe("par"); + expect(draft.outcome).toMatchObject({ status: "failed", error: { message: "out of memory" } }); + }); + + it("treats a cut stream as interrupted, and running out of tokens as incomplete", async () => { + const cut = await collectDraft(generate('{"message":{"content":"a"},"done":false}\n')); + const long = await collectDraft(generate('{"done":true,"done_reason":"length"}\n')); + + expect(cut.outcome.status).toBe("interrupted"); + expect(long.outcome.status).toBe("incomplete"); + }); + + it("names a missing model", async () => { + const { transport } = fakeTransport({ + "/api/chat": () => response(404, '{"error":"model not found"}'), + }); + + const draft = await collectDraft( + createOllamaProvider(transport).generate({ model: "m", instructions: "i", input: "x" }), + ); + + expect(draft.outcome).toMatchObject({ + status: "failed", + error: { code: "unknown_model", message: "model not found" }, + }); + }); + + it("reports the server not running as interrupted, so that the work can wait", async () => { + const offline: AiTransport = { + send: async () => { + throw new Error("connect ECONNREFUSED"); + }, + }; + + const draft = await collectDraft( + createOllamaProvider(offline).generate({ model: "m", instructions: "i", input: "x" }), + ); + + expect(draft.outcome.status).toBe("interrupted"); + }); +}); diff --git a/packages/core/test/analysis/eval.test.ts b/packages/core/test/analysis/eval.test.ts new file mode 100644 index 0000000..be72563 --- /dev/null +++ b/packages/core/test/analysis/eval.test.ts @@ -0,0 +1,498 @@ +import { mkdtempSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { withCache } from "../../eval/cache"; +import { + assertAllowance, + assertFreeModel, + assertNoCost, + estimateRequests, + SpendingRefused, +} from "../../eval/guard"; +import { type Expected, mergedPeople, sameQuote, scoreAnalysis } from "../../eval/score"; +import { + type AiProvider, + type AnalysisState, + collectDraft, + emptyState, + type Fact, + type GenerateEvent, +} from "../../src/index"; + +const EXPECTED: Expected = { + entities: [ + { id: "marta", names: ["Marta Rey", "Marta"] }, + { id: "tomas", names: ["Tomás"] }, + { id: "luis_cartero", names: ["Luis", "el cartero"] }, + { id: "luis_alcalde", names: ["Luis Ferrer", "el alcalde"] }, + ], + must_resolve_reference: [ + { scene: "s1", quote: "Llegó tarde", entity: "marta" }, + { scene: "s2", quote: "lo vio", entity: "tomas" }, + ], + must_flag: [ + { + id: "ojos", + category: "coherence", + a: { scene: "s1", quote: "ojos verdes" }, + b: { scene: "s2", quote: "ojos marrones" }, + }, + { + id: "errata", + category: "correction", + a: { scene: "s2", quote: "halla leído" }, + expected_fix: "haya leído", + }, + ], + must_not_flag: [ + { + id: "n_dos_luis", + namesakes: ["luis_cartero", "luis_alcalde"], + a: { scene: "s1", quote: "Luis, el cartero" }, + b: { scene: "s1", quote: "Luis Ferrer, el alcalde" }, + }, + { + id: "soltera", + a: { scene: "s1", quote: "soltera" }, + b: { scene: "s2", quote: "casada" }, + }, + ], +}; + +const TEXTS = new Map([ + [ + "s1", + "Llegó tarde. Tenía los ojos verdes y estaba soltera. Me la dio Luis, el cartero. Luis Ferrer, el alcalde, entró.", + ], + ["s2", "Sus ojos marrones. Ya casada. Inés lo vio. Cuando halla leído la carta."], +]); + +function fact(sceneId: string, entityId: string, quote: string, value = "v"): Fact { + return { + id: `${sceneId}-${quote}`, + sceneId, + entityId, + category: "other", + attribute: "a", + value, + quote, + sentence: quote, + storyTime: null, + inDialogue: false, + }; +} + +const perfect: AnalysisState = { + ...emptyState(["s1", "s2"]), + entities: [ + { id: "m", type: "person", canonicalName: "Marta Rey", aliases: ["Marta"] }, + { id: "t", type: "person", canonicalName: "Tomás", aliases: [] }, + { id: "c", type: "person", canonicalName: "Luis", aliases: ["el cartero"] }, + { id: "f", type: "person", canonicalName: "Luis Ferrer", aliases: ["el alcalde"] }, + ], + facts: [ + fact("s1", "m", "Llegó tarde"), + fact("s2", "t", "lo vio"), + fact("s1", "c", "Luis, el cartero"), + fact("s1", "f", "Luis Ferrer, el alcalde"), + ], + issues: [ + { + kind: "contradiction", + a: { sceneId: "s1", quote: "ojos verdes" }, + b: { sceneId: "s2", quote: "ojos marrones" }, + reason: "r", + }, + { + kind: "correction", + a: { sceneId: "s2", quote: "halla leído" }, + fix: "haya leído", + reason: "r", + }, + ], +}; + +const score = (state: AnalysisState, discarded = 0) => + scoreAnalysis(state, EXPECTED, TEXTS, discarded); + +// The thresholds of the real fixture would ask for more than this small example has. +const metrics = (state: AnalysisState) => { + const { mustFlag, falseAlarms, inventedQuotes, twoLuisMerged, references } = score(state); + return { + found: mustFlag.found, + falseAlarms, + inventedQuotes, + twoLuisMerged, + refs: references.found, + }; +}; + +describe("score", () => { + it("finds everything in a perfect analysis", () => { + expect(metrics(perfect)).toEqual({ + found: ["ojos", "errata"], + falseAlarms: [], + inventedQuotes: 0, + twoLuisMerged: false, + refs: 2, + }); + }); + + it("finds nothing in an empty analysis, and that doesn't pass", () => { + const result = score(emptyState()); + + expect(result.mustFlag.found).toEqual([]); + expect(result.passed).toBe(false); + }); + + it("matches a pair whichever way round it comes", () => { + const swapped: AnalysisState = { + ...perfect, + issues: [ + { + kind: "contradiction", + a: { sceneId: "s2", quote: "ojos marrones" }, + b: { sceneId: "s1", quote: "ojos verdes" }, + reason: "r", + }, + ], + }; + + expect(score(swapped).mustFlag.found).toContain("ojos"); + }); + + it("doesn't count a contradiction found in the wrong scene", () => { + const wrong: AnalysisState = { + ...perfect, + issues: [ + { + kind: "contradiction", + a: { sceneId: "s2", quote: "ojos verdes" }, + b: { sceneId: "s2", quote: "ojos marrones" }, + reason: "r", + }, + ], + }; + + expect(score(wrong).mustFlag.found).not.toContain("ojos"); + }); + + it("doesn't count half a pair", () => { + const half: AnalysisState = { + ...perfect, + issues: [ + { + kind: "contradiction", + a: { sceneId: "s1", quote: "ojos verdes" }, + b: { sceneId: "s2", quote: "Ya casada" }, + reason: "r", + }, + ], + }; + + expect(score(half).mustFlag.found).not.toContain("ojos"); + }); + + it("counts an evolution reported as an error as a false alarm", () => { + const alarm: AnalysisState = { + ...perfect, + issues: [ + ...perfect.issues, + { + kind: "contradiction", + a: { sceneId: "s1", quote: "soltera" }, + b: { sceneId: "s2", quote: "casada" }, + reason: "r", + }, + ], + }; + + expect(score(alarm).falseAlarms).toEqual(["soltera"]); + }); + + it("doesn't count a correction with a wrong fix: it's applied with one click", async () => { + const wrongFix: AnalysisState = { + ...perfect, + issues: [ + { + kind: "correction", + a: { sceneId: "s2", quote: "halla leído" }, + fix: "había leído", + reason: "r", + }, + ], + }; + + expect(score(wrongFix).mustFlag.found).not.toContain("errata"); + }); + + it("takes two quotes of one sentence as one place, even with its final stop", async () => { + const texts = new Map([["s1", "Primera frase. Marta oyó la llamada, y luego la voz."]]); + const ref = { scene: "s1", quote: "oyó la llamada" }; + + expect(sameQuote({ sceneId: "s1", quote: "luego la voz." }, ref, texts)).toBe(true); + expect(sameQuote({ sceneId: "s1", quote: "Primera frase." }, ref, texts)).toBe(false); + }); + + it("doesn't take a correction as the contradiction it resembles, nor the other way", () => { + const mixedUp: AnalysisState = { + ...perfect, + issues: [ + { + kind: "contradiction", + a: { sceneId: "s2", quote: "halla leído" }, + b: { sceneId: "s2", quote: "x" }, + reason: "r", + }, + ], + }; + + expect(score(mixedUp).mustFlag.found).not.toContain("errata"); + }); + + it("catches a quote kept that isn't in the scene", () => { + const invented: AnalysisState = { + ...perfect, + facts: [...perfect.facts, fact("s1", "m", "algo que no dice")], + }; + + expect(score(invented).inventedQuotes).toBe(1); + expect(score(invented).passed).toBe(false); + }); + + it("reports the discarded quotes apart, since those were caught", () => { + expect(score(perfect, 3).discardedQuotes).toBe(3); + expect(score(perfect, 3).inventedQuotes).toBe(0); + }); + + it("counts a reference only if it's about the right person", () => { + const wrong: AnalysisState = { + ...perfect, + facts: perfect.facts.map((f) => (f.quote === "Llegó tarde" ? { ...f, entityId: "t" } : f)), + }; + + expect(score(wrong).references.found).toBe(1); + expect(score(wrong).references.missed).toEqual(["Llegó tarde"]); + }); + + it("counts a pronoun resolved in the fact's value", () => { + const inValue: AnalysisState = { + ...perfect, + facts: perfect.facts.map((f) => + f.quote === "lo vio" ? { ...f, entityId: "m", value: "vio a Tomás" } : f, + ), + }; + + expect(score(inValue).references.found).toBe(2); + }); + + it("counts issues that are neither expected nor forbidden", () => { + const extra: AnalysisState = { + ...perfect, + issues: [ + ...perfect.issues, + { kind: "correction", a: { sceneId: "s1", quote: "Llegó" }, fix: "x", reason: "r" }, + ], + }; + + expect(score(extra).extraIssues).toBe(1); + }); +}); + +describe("the two people with one name", () => { + it("are fine when kept apart", () => { + expect(mergedPeople(perfect, EXPECTED)).toBe(false); + }); + + it("are merged if a name of each ends up in one entity", () => { + const merged: AnalysisState = { + ...perfect, + entities: [ + { id: "f", type: "person", canonicalName: "Luis Ferrer", aliases: ["el cartero"] }, + ], + facts: [], + }; + + expect(mergedPeople(merged, EXPECTED)).toBe(true); + }); + + it("are merged if each one's quote is attributed to the same entity, even with no alias", () => { + // This is how the first real run went wrong, while a check of the names alone said it was fine. + const merged: AnalysisState = { + ...perfect, + entities: [{ id: "f", type: "person", canonicalName: "Luis Ferrer", aliases: [] }], + facts: [fact("s1", "f", "Luis, el cartero"), fact("s1", "f", "Luis Ferrer, el alcalde")], + }; + + expect(mergedPeople(merged, EXPECTED)).toBe(true); + }); +}); + +describe("sameQuote", () => { + const ref = { scene: "s1", quote: "ojos verdes" }; + + it.each([ + [{ sceneId: "s1", quote: "ojos verdes" }, true], + [{ sceneId: "s1", quote: "los ojos verdes y" }, true], + [{ sceneId: "s1", quote: "verdes" }, true], + [{ sceneId: "s2", quote: "ojos verdes" }, false], + [{ sceneId: "s1", quote: "" }, false], + [{ sceneId: "s1", quote: "Llegó tarde" }, false], + ])("%j: %s", (actual, expected) => { + expect(sameQuote(actual, ref, TEXTS)).toBe(expected); + }); + + it("accepts another part of the same sentence, only if it knows the text", () => { + const sameSentence = { sceneId: "s1", quote: "estaba soltera" }; + + expect(sameQuote(sameSentence, ref, TEXTS)).toBe(true); + expect(sameQuote(sameSentence, ref)).toBe(false); + }); +}); + +describe("spending guard", () => { + const models = [ + { id: "a:free", displayName: "A", capabilities: CAPS(), free: true }, + { id: "b", displayName: "B", capabilities: CAPS(), free: false }, + { id: "c:free", displayName: "C", capabilities: CAPS() }, + ]; + + function CAPS() { + return { structuredOutput: "json_schema", webSearch: false, tools: false } as const; + } + + it("lets a free model through", () => { + expect(() => assertFreeModel(models, "a:free")).not.toThrow(); + }); + + it.each(["b", "c:free", "unknown"])("stops %s before any request", (id) => { + expect(() => assertFreeModel(models, id)).toThrow(SpendingRefused); + }); + + it("stops at the first answer with any cost", () => { + expect(() => assertNoCost({ inputTokens: 1, outputTokens: 1, cost: 0.0001 })).toThrow( + SpendingRefused, + ); + expect(() => assertNoCost({ inputTokens: 1, outputTokens: 1, cost: 0 })).not.toThrow(); + expect(() => assertNoCost({ inputTokens: 1, outputTokens: 1, cost: null })).not.toThrow(); + }); + + it("won't start a run that doesn't fit today's allowance", () => { + const allowance = { used: 45, limit: 50, remaining: 5 }; + + expect(() => assertAllowance(allowance, 6)).toThrow(SpendingRefused); + expect(() => assertAllowance(allowance, 5)).not.toThrow(); + expect(() => assertAllowance(null, 100)).not.toThrow(); + }); + + it("estimates requests by scene, on the safe side", () => { + expect(estimateRequests(8)).toBeGreaterThanOrEqual(8 * 3); + expect(estimateRequests(0)).toBe(0); + }); +}); + +describe("cache", () => { + const directories: string[] = []; + + afterEach(() => { + for (const directory of directories.splice(0)) { + rmSync(directory, { recursive: true, force: true }); + } + }); + + function counting(outcomes: readonly ("completed" | "failed")[]) { + let calls = 0; + const provider: AiProvider = { + id: "fake", + displayName: "Fake", + billing: "local", + listModels: async () => [], + async *generate(): AsyncGenerator { + const outcome = outcomes[Math.min(calls, outcomes.length - 1)]; + calls += 1; + if (outcome === "completed") { + yield { type: "text", delta: `respuesta ${calls}` }; + yield { type: "end", outcome: { status: "completed" } }; + } else { + yield { + type: "end", + outcome: { status: "failed", error: { code: "429", message: "limit" } }, + }; + } + }, + }; + return { provider, calls: () => calls }; + } + + const fresh = () => { + const directory = mkdtempSync(join(tmpdir(), "ink-cache-")); + directories.push(directory); + return directory; + }; + + const request = { model: "m", instructions: "i", input: "x" }; + + it("never asks the same thing twice, so the daily allowance lasts", async () => { + const { provider, calls } = counting(["completed"]); + const cached = withCache(provider, fresh()); + + const first = await collectDraft(cached.generate(request)); + const second = await collectDraft(cached.generate(request)); + + expect(calls()).toBe(1); + expect(second.text).toBe(first.text); + expect(second.completed).toBe(true); + expect(cached.stats).toEqual({ hits: 1, misses: 1 }); + }); + + it("asks again when anything in the request changes", async () => { + const { provider, calls } = counting(["completed"]); + const cached = withCache(provider, fresh()); + + await collectDraft(cached.generate(request)); + await collectDraft(cached.generate({ ...request, input: "otra" })); + await collectDraft(cached.generate({ ...request, model: "n" })); + await collectDraft(cached.generate({ ...request, reasoning: "off" })); + + expect(calls()).toBe(4); + }); + + it("doesn't remember a failure: it must be tried again", async () => { + const { provider, calls } = counting(["failed", "completed"]); + const directory = fresh(); + const cached = withCache(provider, directory); + + const failed = await collectDraft(cached.generate(request)); + expect(failed.completed).toBe(false); + expect(readdirSync(directory)).toEqual([]); + + const retried = await collectDraft(cached.generate(request)); + expect(retried.completed).toBe(true); + expect(calls()).toBe(2); + }); + + it("in cache-only mode never asks, so that scoring again can't use up the allowance", async () => { + const { provider, calls } = counting(["completed"]); + const directory = fresh(); + await collectDraft(withCache(provider, directory).generate(request)); + + const scoring = withCache(provider, directory, true); + const known = await collectDraft(scoring.generate(request)); + const unknown = await collectDraft(scoring.generate({ ...request, input: "otra" })); + + expect(known.completed).toBe(true); + expect(unknown.outcome).toMatchObject({ status: "failed", error: { code: "not_cached" } }); + expect(calls()).toBe(1); + }); + + it("is kept on disk, for the next run", async () => { + const { provider, calls } = counting(["completed"]); + const directory = fresh(); + + await collectDraft(withCache(provider, directory).generate(request)); + await collectDraft(withCache(provider, directory).generate(request)); + + expect(calls()).toBe(1); + }); +}); diff --git a/packages/core/test/analysis/names.test.ts b/packages/core/test/analysis/names.test.ts new file mode 100644 index 0000000..e2d8034 --- /dev/null +++ b/packages/core/test/analysis/names.test.ts @@ -0,0 +1,75 @@ +import { describe, expect, it } from "vitest"; +import { + isEpithet, + isPartOfName, + looksLikeVariant, + nameTokens, + withoutAccents, +} from "../../src/analysis/names"; + +describe("nameTokens", () => { + it.each([ + ["Marta Rey", ["marta", "rey"]], + ["la tía Remedios", ["remedios"]], + ["el faro de la Punta", ["faro", "punta"]], + ["la doctora", ["doctora"]], + ["su", []], + ])("%s gives %j", (name, tokens) => { + expect(nameTokens(name)).toEqual(tokens); + }); +}); + +describe("isPartOfName", () => { + it.each([ + ["Marta", "Marta Rey", true], + ["Marta Rey", "Marta", true], + ["tía Remedios", "Remedios", true], + ["faro", "el faro de la Punta", true], + ["Marta", "Inés Cabrera", false], + ["Rey", "Reyes", false], + ["la", "Marta", false], + ["", "Marta", false], + ])("%s and %s: %s", (a, b, expected) => { + expect(isPartOfName(a, b)).toBe(expected); + }); +}); + +describe("isEpithet", () => { + it.each([ + ["la doctora", true], + ["el cartero", true], + ["Marta", false], + ["Luis Ferrer", false], + ["La Punta", false], + ])("%s: %s", (name, expected) => { + expect(isEpithet(name)).toBe(expected); + }); +}); + +describe("looksLikeVariant", () => { + it.each([ + ["Tomás", "Thomas", true], + ["Tomas", "Tomás", true], + ["Marta", "Martha", true], + ["Puerto Viejo", "Puerto Viego", true], + ])("flags %s and %s", (a, b, expected) => { + expect(looksLikeVariant(a, b)).toBe(expected); + }); + + it.each([ + ["Marta", "Marta"], + ["marta", "Marta"], + ["Ana", "Eva"], + ["Luis", "Inés"], + ["Tomás", "Julián"], + ["Rey", "Ray"], + ])("doesn't flag %s and %s", (a, b) => { + expect(looksLikeVariant(a, b)).toBe(false); + }); +}); + +describe("withoutAccents", () => { + it("only takes the accents off", () => { + expect(withoutAccents("Tomás Núñez")).toBe("Tomas Nunez"); + }); +}); diff --git a/packages/core/test/analysis/pipeline.test.ts b/packages/core/test/analysis/pipeline.test.ts new file mode 100644 index 0000000..28d14a8 --- /dev/null +++ b/packages/core/test/analysis/pipeline.test.ts @@ -0,0 +1,1394 @@ +import { describe, expect, it } from "vitest"; +import { + type AiProvider, + type AnalysisDeps, + type AnalysisState, + analyzeScene, + emptyState, + type GenerateEvent, + type GenerateRequest, + liveEntities, + StructuredOutputError, + withoutScene, +} from "../../src/index"; + +type Extracted = { + entities: { name: string; type: string; aliases: string[] }[]; + facts: { + entity: string; + category: string; + attribute: string; + value: string; + quote: string; + story_time: string | null; + }[]; + relations: { from: string; to: string; label: string; quote: string }[]; +}; + +const person = (name: string, ...aliases: string[]) => ({ name, type: "person", aliases }); + +const fact = (entity: string, category: string, value: string, quote: string) => ({ + entity, + category, + attribute: "a", + value, + quote, + story_time: null, +}); + +const NOTHING: Extracted = { entities: [], facts: [], relations: [] }; + +interface Script { + /** First reading of each scene, in order. */ + readonly extract?: readonly Extracted[]; + /** Second reading, of the sentences the first left out. */ + readonly missed?: readonly Extracted[]; + readonly identities?: readonly { name: string; entity_id: string | null }[]; + /** Given the question, what the comparison finds. */ + readonly compare?: (input: string) => readonly { + new_fact: string; + other_fact: string; + verdict: string; + reason: string; + }[]; + readonly corrections?: readonly { quote: string; fix: string; reason: string }[]; +} + +/** Answers each step by the schema it asks for, and remembers everything it was asked. */ +function fakeModel(script: Script) { + const asked: GenerateRequest[] = []; + const extractions = [...(script.extract ?? [])]; + const secondReadings = [...(script.missed ?? [])]; + const provider: AiProvider = { + id: "fake", + displayName: "Fake", + billing: "local", + listModels: async () => [], + async *generate(request): AsyncGenerator { + asked.push(request); + const answer = () => { + switch (request.structured?.name) { + case "scene_extraction": + return request.input.includes("A first reading found no facts") + ? (secondReadings.shift() ?? NOTHING) + : (extractions.shift() ?? NOTHING); + case "identity_answers": + return { answers: script.identities ?? [] }; + case "comparison": + return { findings: script.compare?.(request.input) ?? [] }; + case "proofreading": + return { corrections: script.corrections ?? [] }; + default: + throw new Error(`Unexpected request ${request.structured?.name}`); + } + }; + yield { type: "text", delta: JSON.stringify(answer()) }; + yield { type: "end", outcome: { status: "completed" } }; + }, + }; + return { provider, asked }; +} + +function depsFor(script: Script): AnalysisDeps & { asked: GenerateRequest[] } { + const { provider, asked } = fakeModel(script); + let next = 0; + return { + provider, + asked, + model: "m", + language: "es", + newId: () => { + next += 1; + return `id${next}`; + }, + }; +} + +const steps = (asked: readonly GenerateRequest[]) => asked.map((r) => r.structured?.name); + +const scene = (id: string, text: string) => ({ id, title: id, text }); + +const start = emptyState(["s1", "s2", "s3"]); + +/** Which scene and quote a comparison's fact tag stands for. */ +const tagOf = (input: string, text: string) => + input + .split("\n") + .find((line) => line.includes(text)) + ?.split(" ")[0] ?? ""; + +/** The comparison shows whole sentences, so each of these scenes says it in its only sentence. */ + +describe("step 1, the rule on quotes (rule 3)", () => { + it("keeps a literal quote and throws away any other, keeping count", async () => { + const text = "Marta bajó del autobús."; + const { state, discardedQuotes } = await analyzeScene( + start, + scene("s1", text), + depsFor({ + extract: [ + { + entities: [person("Marta")], + facts: [ + fact("Marta", "action", "bajó", "bajó del autobús"), + fact("Marta", "action", "subió", "subió al tren"), + fact("Marta", "action", "sin cita", ""), + ], + relations: [{ from: "Marta", to: "Marta", label: "x", quote: "no aparece" }], + }, + ], + }), + ); + + expect(state.facts.map((item) => item.quote)).toEqual(["bajó del autobús"]); + expect(state.relations).toEqual([]); + expect(discardedQuotes).toEqual(["subió al tren", "", "no aparece"]); + }); + + it("requires the exact characters, accents and capitals included", async () => { + const { state } = await analyzeScene( + start, + scene("s1", "Marta bajó."), + depsFor({ + extract: [ + { + entities: [person("Marta")], + facts: [ + fact("Marta", "action", "v", "marta bajó"), + fact("Marta", "action", "v", "Marta bajo"), + ], + relations: [], + }, + ], + }), + ); + + expect(state.facts).toEqual([]); + }); + + it("gives every kept fact a source, with its scene", async () => { + const { state } = await analyzeScene( + start, + scene("s1", "Marta bajó."), + depsFor({ + extract: [ + { + entities: [person("Marta")], + facts: [fact("Marta", "action", "v", "Marta bajó")], + relations: [], + }, + ], + }), + ); + + expect(state.facts[0]).toMatchObject({ sceneId: "s1", quote: "Marta bajó" }); + }); + + it("drops a fact about a name that isn't in the scene, and says so", async () => { + const { state, unplacedFacts } = await analyzeScene( + start, + scene("s1", "Marta bajó. Llovía sin parar."), + depsFor({ + extract: [ + { + entities: [person("Marta")], + facts: [fact("Alguien de fuera", "action", "v", "Llovía sin parar")], + relations: [], + }, + ], + }), + ); + + expect(state.facts).toEqual([]); + expect(unplacedFacts).toEqual(["Llovía sin parar"]); + }); + + it("gives a fact filed under a made-up name to the one person its quote names", async () => { + const { state, unplacedFacts } = await analyzeScene( + start, + scene("s1", "Saludaron a Marta en la plaza."), + depsFor({ + extract: [ + { + entities: [person("Marta")], + facts: [fact("saludo", "other", "v", "Saludaron a Marta")], + relations: [], + }, + ], + }), + ); + + expect(state.facts[0]?.entityId).toBe(state.entities[0]?.id); + expect(unplacedFacts).toEqual([]); + }); + + it("doesn't guess if its quote names two people, or only part of a word", async () => { + const { state, unplacedFacts } = await analyzeScene( + start, + scene("s1", "Marta y Tomás salieron. Martalena llegó."), + depsFor({ + extract: [ + { + entities: [person("Marta"), person("Tomás")], + facts: [ + fact("salida", "other", "v", "Marta y Tomás salieron"), + fact("llegada", "other", "v", "Martalena llegó"), + ], + relations: [], + }, + ], + }), + ); + + expect(state.facts).toEqual([]); + expect(unplacedFacts).toEqual(["Marta y Tomás salieron", "Martalena llegó"]); + }); + + it("keeps a fact about something the model forgot to list, if the scene names it", async () => { + // A real answer that used to lose a contradiction: a good fact, but no "veda" in the list. + const { state, unplacedFacts } = await analyzeScene( + start, + scene("s1", "El jueves empezaba la veda."), + depsFor({ + extract: [ + { + entities: [], + facts: [fact("veda", "date_of_event", "jueves", "El jueves empezaba la veda")], + relations: [], + }, + ], + }), + ); + + expect(state.facts).toHaveLength(1); + expect(state.entities.map((entity) => entity.canonicalName)).toEqual(["veda"]); + expect(unplacedFacts).toEqual([]); + }); + + it("joins something forgotten to what's already known by that name", async () => { + const { state: first } = await analyzeScene( + start, + scene("s1", "Empieza la veda el martes."), + depsFor({ + extract: [ + { + entities: [{ name: "la veda", type: "lore", aliases: [] }], + facts: [fact("la veda", "date_of_event", "martes", "Empieza la veda el martes")], + relations: [], + }, + ], + }), + ); + + const { state } = await analyzeScene( + first, + scene("s2", "El jueves empezaba la veda."), + depsFor({ + extract: [ + { + entities: [], + facts: [fact("veda", "date_of_event", "jueves", "El jueves empezaba la veda")], + relations: [], + }, + ], + }), + ); + + expect(state.entities).toHaveLength(1); + expect(new Set(state.facts.map((item) => item.entityId)).size).toBe(1); + }); + + it("keeps the whole sentence, as a model may split one into several facts", async () => { + const text = "Marta salió al amanecer y esa misma tarde estaba en la capital. Fin de todo."; + const { state } = await analyzeScene( + start, + scene("s1", text), + depsFor({ + extract: [ + { + entities: [person("Marta")], + facts: [fact("Marta", "location", "capital", "estaba en la capital")], + relations: [], + }, + ], + }), + ); + + expect(state.facts[0]?.sentence).toBe( + "Marta salió al amanecer y esa misma tarde estaba en la capital.", + ); + }); + + it("notices a fact stated in a line of dialogue", async () => { + const text = "Tomás calló.\n\n—Lo vendí."; + const { state } = await analyzeScene( + start, + scene("s1", text), + depsFor({ + extract: [ + { + entities: [person("Tomás")], + facts: [ + fact("Tomás", "action", "calló", "Tomás calló"), + fact("Tomás", "action", "vendió", "Lo vendí"), + ], + relations: [], + }, + ], + }), + ); + + expect(state.facts.map((item) => item.inDialogue)).toEqual([false, true]); + }); +}); + +describe("step 1, a second reading of what the first left out", () => { + const text = "Marta bajó del autobús. El jueves empezaba la veda."; + const first: Extracted = { + entities: [person("Marta")], + facts: [fact("Marta", "action", "bajó", "Marta bajó del autobús")], + relations: [], + }; + + it("asks about the sentences without facts, and adds what it finds", async () => { + const deps = depsFor({ + extract: [first], + missed: [ + { + entities: [{ name: "la veda", type: "lore", aliases: [] }], + facts: [fact("la veda", "date_of_event", "el jueves", "El jueves empezaba la veda")], + relations: [], + }, + ], + }); + + const { state } = await analyzeScene(start, scene("s1", text), deps); + + expect(deps.asked[1]?.input).toContain("- El jueves empezaba la veda."); + expect(state.facts.map((item) => item.quote)).toEqual([ + "Marta bajó del autobús", + "El jueves empezaba la veda", + ]); + }); + + it("doesn't spend a request if every sentence gave something", async () => { + const deps = depsFor({ + extract: [ + { + ...first, + facts: [...first.facts, fact("Marta", "date_of_event", "x", "El jueves empezaba")], + }, + ], + }); + + await analyzeScene(start, scene("s1", text), deps); + + expect(steps(deps.asked)).toEqual(["scene_extraction", "proofreading"]); + }); + + it("checks the second reading's quotes too", async () => { + const { state, discardedQuotes } = await analyzeScene( + start, + scene("s1", text), + depsFor({ + extract: [first], + missed: [ + { + entities: [], + facts: [fact("Marta", "action", "v", "un invento")], + relations: [], + }, + ], + }), + ); + + expect(state.facts).toHaveLength(1); + expect(discardedQuotes).toEqual(["un invento"]); + }); + + it("joins an entity both readings named, instead of having two", async () => { + const { state } = await analyzeScene( + start, + scene("s1", text), + depsFor({ + extract: [first], + missed: [ + { + entities: [person("Marta", "Marta bajó")], + facts: [], + relations: [], + }, + ], + }), + ); + + expect(state.entities).toHaveLength(1); + }); +}); + +describe("step 1, what can be a name", () => { + it.each(["su hermano", "su tía", "her brother", "la", "de"])("not «%s»", async (alias) => { + const text = `Tomás y ${alias} hablaron.`; + const { state } = await analyzeScene( + start, + scene("s1", text), + depsFor({ + extract: [ + { + entities: [person("Tomás", alias)], + facts: [fact("Tomás", "action", "habló", "hablaron")], + relations: [], + }, + ], + }), + ); + + expect(state.entities[0]?.aliases).not.toContain(alias); + }); + + it("not one the scene doesn't use", async () => { + const { state } = await analyzeScene( + start, + scene("s1", "Tomás habló largo rato."), + depsFor({ + extract: [ + { + entities: [person("Tomás", "el marinero")], + facts: [fact("Tomás", "action", "habló", "Tomás habló")], + relations: [], + }, + ], + }), + ); + + expect(state.entities[0]?.aliases).not.toContain("el marinero"); + }); + + it("not a misspelling, which the author must see instead", async () => { + const { state } = await analyzeScene( + start, + scene("s1", "Tomás habló. Thomas calló allí."), + depsFor({ + extract: [ + { + entities: [person("Tomás", "Thomas")], + facts: [ + fact("Tomás", "action", "habló", "Tomás habló"), + fact("Tomás", "action", "calló", "Thomas calló allí"), + ], + relations: [], + }, + ], + }), + ); + + expect(state.entities[0]?.aliases).not.toContain("Thomas"); + }); +}); + +describe("step 1, people the text keeps apart", () => { + const text = "—Me la dio Luis, el cartero.\n\nLuis Ferrer, el alcalde, entró sin llamar."; + + const luisIn = (state: AnalysisState, quote: string) => { + const id = state.facts.find((item) => item.quote === quote)?.entityId; + return state.entities.find((entity) => entity.id === id)?.canonicalName; + }; + + it("separates them when the model folds one into the other, with their facts", async () => { + const { state } = await analyzeScene( + start, + scene("s1", text), + depsFor({ + extract: [ + { + entities: [person("Luis Ferrer", "Luis", "el alcalde")], + facts: [ + fact("Luis Ferrer", "action", "dio", "Me la dio Luis, el cartero"), + fact("Luis Ferrer", "action", "entró", "entró sin llamar"), + ], + relations: [], + }, + ], + }), + ); + + const names = state.entities.map((entity) => entity.canonicalName); + expect(names).toContain("Luis"); + expect(names).toContain("Luis Ferrer"); + expect(luisIn(state, "Me la dio Luis, el cartero")).toBe("Luis"); + expect(luisIn(state, "entró sin llamar")).toBe("Luis Ferrer"); + expect(state.entities.find((e) => e.canonicalName === "Luis")?.aliases).toContain("el cartero"); + expect(state.entities.find((e) => e.canonicalName === "Luis Ferrer")?.aliases).not.toContain( + "Luis", + ); + }); + + it("also the other way round, if the model used the shorter name", async () => { + const { state } = await analyzeScene( + start, + scene("s1", text), + depsFor({ + extract: [ + { + entities: [person("Luis", "el cartero", "Luis Ferrer", "el alcalde")], + facts: [fact("Luis", "action", "entró", "entró sin llamar")], + relations: [], + }, + ], + }), + ); + + expect(luisIn(state, "entró sin llamar")).toBe("Luis Ferrer"); + }); + + it("leaves them alone if the model got it right", async () => { + const { state } = await analyzeScene( + start, + scene("s1", text), + depsFor({ + extract: [ + { + entities: [person("Luis", "el cartero"), person("Luis Ferrer", "el alcalde")], + facts: [ + fact("Luis", "action", "dio", "Me la dio Luis, el cartero"), + fact("Luis Ferrer", "action", "entró", "entró sin llamar"), + ], + relations: [], + }, + ], + }), + ); + + expect(state.entities).toHaveLength(2); + expect(luisIn(state, "Me la dio Luis, el cartero")).toBe("Luis"); + }); + + it("doesn't touch anyone else in the scene", async () => { + const { state } = await analyzeScene( + start, + scene("s1", `Marta oyó. ${text}`), + depsFor({ + extract: [ + { + entities: [person("Marta"), person("Luis Ferrer", "Luis")], + facts: [fact("Marta", "action", "oyó", "Marta oyó")], + relations: [], + }, + ], + }), + ); + + expect(luisIn(state, "Marta oyó")).toBe("Marta"); + }); +}); + +describe("step 2, identities", () => { + const known: AnalysisState = { + ...start, + entities: [ + { id: "marta", type: "person", canonicalName: "Marta Rey", aliases: ["Marta"] }, + { id: "ines", type: "person", canonicalName: "Inés Cabrera", aliases: [] }, + { id: "cartero", type: "person", canonicalName: "Luis", aliases: ["el cartero"] }, + { id: "alcalde", type: "person", canonicalName: "Luis Ferrer", aliases: ["el alcalde"] }, + ], + mentions: [ + { sceneId: "s1", entityId: "marta", quote: "Marta" }, + { sceneId: "s1", entityId: "ines", quote: "Inés Cabrera" }, + ], + }; + + const about = (name: string, quote: string) => ({ + entities: [person(name)], + facts: [fact(name, "action", "v", quote)], + relations: [], + }); + + it("settles an exact name without asking the model", async () => { + const deps = depsFor({ extract: [about("Marta", "Marta salió")] }); + + const { state } = await analyzeScene(known, scene("s2", "Marta salió."), deps); + + expect(state.facts[0]?.entityId).toBe("marta"); + expect(state.entities).toHaveLength(known.entities.length); + expect(steps(deps.asked)).not.toContain("identity_answers"); + }); + + it("settles part of a name without asking the model", async () => { + const deps = depsFor({ extract: [about("Inés", "Inés salió")] }); + + const { state } = await analyzeScene(known, scene("s2", "Inés salió."), deps); + + expect(state.facts[0]?.entityId).toBe("ines"); + expect(state.entities.find((e) => e.id === "ines")?.aliases).toContain("Inés"); + }); + + it("asks the model about an epithet, and follows what it says", async () => { + const deps = depsFor({ + extract: [about("la doctora", "la doctora salió")], + identities: [{ name: "la doctora", entity_id: "marta" }], + }); + + const { state } = await analyzeScene(known, scene("s2", "Luego la doctora salió."), deps); + + expect(state.facts[0]?.entityId).toBe("marta"); + expect(state.entities.find((e) => e.id === "marta")?.aliases).toContain("la doctora"); + }); + + it("creates someone new when the model isn't sure", async () => { + const { state } = await analyzeScene( + known, + scene("s2", "Luego la doctora salió."), + depsFor({ + extract: [about("la doctora", "la doctora salió")], + identities: [{ name: "la doctora", entity_id: null }], + }), + ); + + expect(state.entities).toHaveLength(known.entities.length + 1); + }); + + it("ignores an answer naming an entity that doesn't exist", async () => { + const { state } = await analyzeScene( + known, + scene("s2", "Luego la doctora salió."), + depsFor({ + extract: [about("la doctora", "la doctora salió")], + identities: [{ name: "la doctora", entity_id: "inventado" }], + }), + ); + + expect(state.entities.map((e) => e.id)).not.toContain("inventado"); + expect(state.entities).toHaveLength(known.entities.length + 1); + }); + + it("doesn't take a known name for someone else in the next scene", async () => { + const { state } = await analyzeScene( + known, + scene("s2", "Luis Ferrer firmó."), + depsFor({ extract: [about("Luis Ferrer", "Luis Ferrer firmó")] }), + ); + + expect(state.facts[0]?.entityId).toBe("alcalde"); + }); + + it("never tells the model about someone no scene supports any more", async () => { + const deps = depsFor({ extract: [about("Marta", "Marta salió")] }); + + await analyzeScene(known, scene("s2", "Marta salió."), deps); + + expect(deps.asked[0]?.input).toContain("Marta Rey"); + expect(deps.asked[0]?.input).not.toContain("Luis Ferrer"); + }); + + it("keeps the id of a name that comes back, so that notes on it would survive (rule 4)", async () => { + expect(liveEntities(known).map((e) => e.id)).not.toContain("alcalde"); + + const { state } = await analyzeScene( + known, + scene("s2", "Luis Ferrer firmó."), + depsFor({ extract: [about("Luis Ferrer", "Luis Ferrer firmó")] }), + ); + + expect(state.entities.find((e) => e.canonicalName === "Luis Ferrer")?.id).toBe("alcalde"); + }); +}); + +describe("step 3, comparison", () => { + const eyes = (color: string, quote: string): Extracted => ({ + entities: [person("Marta")], + facts: [fact("Marta", "physical", color, quote)], + relations: [], + }); + + async function twoScenes( + first: Extracted, + second: Extracted, + compare: Script["compare"], + texts: readonly [string, string], + ) { + const deps = depsFor({ extract: [first, second], ...(compare ? { compare } : {}) }); + const one = await analyzeScene(start, scene("s1", texts[0]), deps); + const before = deps.asked.length; + const two = await analyzeScene(one.state, scene("s2", texts[1]), deps); + return { state: two.state, asked: deps.asked.slice(before) }; + } + + const contradiction = (input: string, a: string, b: string) => [ + { + new_fact: tagOf(input, b), + other_fact: tagOf(input, a), + verdict: "contradiction", + reason: "No pueden ser ciertas las dos.", + }, + ]; + + it("reports a contradiction between scenes, citing both", async () => { + const { state } = await twoScenes( + eyes("verdes", "ojos verdes"), + eyes("marrones", "ojos marrones"), + (input) => contradiction(input, "ojos verdes", "ojos marrones"), + ["Tenía los ojos verdes.", "Tenía los ojos marrones."], + ); + + expect(state.issues).toEqual([ + { + kind: "contradiction", + a: { sceneId: "s1", quote: "ojos verdes" }, + b: { sceneId: "s2", quote: "ojos marrones" }, + reason: "No pueden ser ciertas las dos.", + }, + ]); + }); + + it("marks only what's new, and says which scene each fact is from", async () => { + const { asked } = await twoScenes( + eyes("verdes", "ojos verdes"), + eyes("marrones", "ojos marrones"), + () => [], + ["Tenía los ojos verdes.", "Tenía los ojos marrones."], + ); + const question = asked.find((r) => r.structured?.name === "comparison")?.input ?? ""; + + expect(question).toMatch(/scene 1 .*ojos verdes\."$/m); + expect(question).toMatch(/scene 2 .*ojos marrones\." NEW$/m); + }); + + it("tells apart an evolution, which isn't an error", async () => { + const { state } = await twoScenes( + eyes("verdes", "ojos verdes"), + eyes("marrones", "ojos marrones"), + (input) => [ + { + new_fact: tagOf(input, "ojos marrones"), + other_fact: tagOf(input, "ojos verdes"), + verdict: "evolution", + reason: "Cambió.", + }, + ], + ["Tenía los ojos verdes.", "Tenía los ojos marrones."], + ); + + expect(state.issues).toEqual([]); + }); + + it("understands one finding naming two facts at once, as some models answer", async () => { + const { state } = await twoScenes( + { + entities: [person("Marta")], + facts: [ + fact("Marta", "physical", "verdes", "ojos verdes"), + fact("Marta", "physical", "verdes", "verdes como su madre"), + ], + relations: [], + }, + eyes("marrones", "ojos marrones"), + (input) => [ + { + new_fact: tagOf(input, "ojos marrones"), + other_fact: `${tagOf(input, "ojos verdes")}, ${tagOf(input, "verdes como su madre")}`, + verdict: "contradiction", + reason: "r", + }, + ], + ["Los ojos verdes. Eran verdes como su madre.", "Tenía los ojos marrones."], + ); + + expect(state.issues.filter((issue) => issue.kind === "contradiction")).toHaveLength(2); + }); + + it("ignores a finding between two old facts, already weighed when they were new", async () => { + const deps = depsFor({ + extract: [ + eyes("verdes", "ojos verdes"), + eyes("marrones", "ojos marrones"), + eyes("azules", "ojos azules"), + ], + compare: (input) => + input.includes("ojos azules") ? contradiction(input, "ojos verdes", "ojos marrones") : [], + }); + const one = await analyzeScene(start, scene("s1", "Los ojos verdes."), deps); + const two = await analyzeScene(one.state, scene("s2", "Los ojos marrones."), deps); + + const { state } = await analyzeScene(two.state, scene("s3", "Los ojos azules."), deps); + + expect(state.issues).toEqual([]); + }); + + it("doesn't take two readings of one sentence for a contradiction", async () => { + const deps = depsFor({ + extract: [ + { + entities: [person("Marta")], + facts: [ + fact("Marta", "physical", "marrones", "ojos marrones"), + fact("Marta", "physical", "oscuros", "más oscuros"), + ], + relations: [], + }, + ], + compare: (input) => contradiction(input, "más oscuros", "ojos marrones"), + }); + const text = "Sus ojos marrones eran más oscuros aquel día."; + + const { state } = await analyzeScene(start, scene("s1", text), deps); + + expect(state.issues).toEqual([]); + }); + + it("ignores a finding about facts that weren't in the question", async () => { + const { state } = await twoScenes( + eyes("verdes", "ojos verdes"), + eyes("marrones", "ojos marrones"), + () => [ + { new_fact: "F99", other_fact: "F1", verdict: "contradiction", reason: "r" }, + { new_fact: "F1", other_fact: "F1", verdict: "contradiction", reason: "r" }, + ], + ["Tenía los ojos verdes.", "Tenía los ojos marrones."], + ); + + expect(state.issues).toEqual([]); + }); + + it("shows the model the whole sentence of each fact", async () => { + const { asked } = await twoScenes( + { + entities: [person("Marta")], + facts: [fact("Marta", "travel_time", "una semana", "una semana de camino")], + relations: [], + }, + { + entities: [person("Marta")], + facts: [fact("Marta", "travel_time", "la capital", "estaba en la capital")], + relations: [], + }, + () => [], + ["Hasta allí hay una semana de camino.", "Salió al amanecer y estaba en la capital."], + ); + const question = asked.find((r) => r.structured?.name === "comparison")?.input ?? ""; + + expect(question).toContain("Salió al amanecer y estaba en la capital."); + }); + + it("reports one conflict once, even if found through two facts of one sentence", async () => { + const { state } = await twoScenes( + { + entities: [person("Marta")], + facts: [ + fact("Marta", "physical", "verdes", "ojos verdes"), + fact("Marta", "physical", "iguales", "lo único igual"), + ], + relations: [], + }, + eyes("marrones", "ojos marrones"), + (input) => [ + ...contradiction(input, "lo único igual", "ojos marrones"), + ...contradiction(input, "ojos verdes", "ojos marrones"), + ], + ["Tenía ojos verdes, lo único igual.", "Tenía los ojos marrones."], + ); + + expect(state.issues.filter((issue) => issue.kind === "contradiction")).toHaveLength(1); + }); + + it("reports one pair once, even if two groups find it", async () => { + const { state } = await twoScenes( + eyes("verdes", "ojos verdes"), + { + entities: [person("Marta")], + facts: [ + fact("Marta", "physical", "marrones", "ojos marrones"), + fact("Marta", "action", "miró", "miró sus ojos marrones"), + ], + relations: [], + }, + (input) => + input.includes("ojos verdes") && input.includes("ojos marrones") + ? contradiction(input, "ojos verdes", "ojos marrones") + : [], + ["Tenía los ojos verdes.", "Se miró: miró sus ojos marrones."], + ); + + expect(state.issues.filter((issue) => issue.kind === "contradiction")).toHaveLength(1); + }); + + it("doesn't ask when nothing could be compared", async () => { + const { asked } = await twoScenes( + eyes("verdes", "ojos verdes"), + { + entities: [person("Tomás")], + facts: [fact("Tomás", "action", "pescó", "Tomás pescó")], + relations: [], + }, + () => { + throw new Error("It shouldn't ask"); + }, + ["Tenía los ojos verdes.", "Tomás pescó hasta tarde."], + ); + + expect(steps(asked)).not.toContain("comparison"); + }); + + it("doesn't compare a trait of one person with another person's", async () => { + const { asked } = await twoScenes( + eyes("verdes", "tenía altura"), + { + entities: [person("Tomás")], + facts: [fact("Tomás", "physical", "bajo", "medía poco")], + relations: [], + }, + () => [], + ["Marta tenía altura.", "Tomás medía poco."], + ); + + expect(steps(asked)).not.toContain("comparison"); + }); + + describe("by shared words, whatever category the model chose", () => { + const dates: readonly [Extracted, Extracted] = [ + { + entities: [person("Tomás")], + facts: [fact("Tomás", "action", "dijo", "la veda empieza el martes")], + relations: [], + }, + { + entities: [{ name: "la veda", type: "lore", aliases: [] }], + facts: [fact("la veda", "date_of_event", "jueves", "El jueves empezaba la veda")], + relations: [], + }, + ]; + + it("sets two dates of one event side by side", async () => { + const { state } = await twoScenes( + dates[0], + dates[1], + (input) => contradiction(input, "la veda empieza el martes", "El jueves empezaba la veda"), + ["Tomás avisó: la veda empieza el martes.", "El jueves empezaba la veda."], + ); + + expect(state.issues.map((issue) => issue.kind)).toEqual(["contradiction"]); + }); + + it("doesn't count people's names as a shared topic", async () => { + const { asked } = await twoScenes( + { + entities: [person("Marta")], + facts: [fact("Marta", "action", "pescó", "Marta pescó")], + relations: [], + }, + { + entities: [person("Marta")], + facts: [fact("Marta", "action", "leyó", "Marta leyó")], + relations: [], + }, + () => [], + ["Marta pescó mucho rato.", "Marta leyó mucho rato."], + ); + + expect(steps(asked)).not.toContain("comparison"); + }); + + it("doesn't ask about one sentence split into several facts", async () => { + const deps = depsFor({ + extract: [ + { + entities: [person("Marta")], + facts: [ + fact("Marta", "action", "bajó", "bajó del autobús rojo"), + fact("Marta", "possession", "autobús", "del autobús"), + ], + relations: [], + }, + ], + }); + + await analyzeScene(start, scene("s1", "Marta bajó del autobús rojo."), deps); + + expect(steps(deps.asked)).not.toContain("comparison"); + }); + }); + + describe("the dead acting", () => { + const text = "Enterraron a la tía Remedios un martes. La tía Remedios llamó a Marta."; + + it("finds it even if the burial and the call are filed as mere actions", async () => { + const deps = depsFor({ + extract: [ + { + entities: [person("Remedios"), person("Marta")], + facts: [ + fact("Remedios", "action", "enterrada", "Enterraron a la tía Remedios"), + fact("Remedios", "action", "llamó", "La tía Remedios llamó"), + ], + relations: [], + }, + ], + compare: (input) => + contradiction(input, "Enterraron a la tía Remedios", "La tía Remedios llamó"), + }); + + const { state } = await analyzeScene(start, scene("s1", text), deps); + + expect(state.issues.filter((issue) => issue.kind === "contradiction")).toHaveLength(1); + }); + + it("finds it even if the later deed is filed under someone else", async () => { + const deps = depsFor({ + extract: [ + { + entities: [person("Remedios", "la tía Remedios"), person("Marta")], + facts: [ + fact("Remedios", "alive_or_dead", "enterrada", "Enterraron a la tía Remedios"), + fact("Marta", "action", "recibió", "La tía Remedios llamó a Marta"), + ], + relations: [], + }, + ], + compare: (input) => { + expect(input).toContain("La tía Remedios llamó a Marta"); + return []; + }, + }); + + await analyzeScene( + start, + scene("s1", "Enterraron a la tía Remedios. La tía Remedios llamó a Marta."), + deps, + ); + + expect(steps(deps.asked)).toContain("comparison"); + }); + + it("isn't asked about someone merely mentioned with death nearby", async () => { + const deps = depsFor({ + extract: [ + { + entities: [person("Marta")], + facts: [fact("Marta", "action", "leyó", "Marta leyó")], + relations: [], + }, + ], + }); + + await analyzeScene(start, scene("s1", "Marta leyó en la mesa."), deps); + + expect(steps(deps.asked)).not.toContain("comparison"); + }); + }); +}); + +describe("names that look misspelled", () => { + const first: Extracted = { + entities: [person("Tomás")], + facts: [fact("Tomás", "action", "pescó", "Tomás pescó")], + relations: [], + }; + + async function withSecondScene(second: Extracted, text: string) { + const deps = depsFor({ extract: [first, second] }); + const one = await analyzeScene(start, scene("s1", "Tomás pescó."), deps); + return (await analyzeScene(one.state, scene("s2", text), deps)).state; + } + + const variants = (state: AnalysisState) => + state.issues.filter((issue) => issue.kind === "name_variant"); + + it("flags one the model took for a new person, without merging them", async () => { + const state = await withSecondScene( + { + entities: [person("Thomas")], + facts: [fact("Thomas", "action", "leyó", "Thomas leyó")], + relations: [], + }, + "Thomas leyó la carta.", + ); + + expect(variants(state)).toEqual([ + expect.objectContaining({ + a: { sceneId: "s1", quote: "Tomás" }, + b: { sceneId: "s2", quote: "Thomas" }, + }), + ]); + }); + + it("flags it from the text even if the model quietly took it for the same person", async () => { + const state = await withSecondScene( + { + entities: [person("Tomás")], + facts: [fact("Tomás", "action", "leyó", "Thomas leyó")], + relations: [], + }, + "Thomas leyó la carta.", + ); + + expect(variants(state)).toHaveLength(1); + }); + + it("flags it once, when both checks find it", async () => { + const state = await withSecondScene( + { + entities: [person("Thomas")], + facts: [fact("Thomas", "action", "leyó", "Thomas leyó")], + relations: [], + }, + "Thomas leyó la carta.", + ); + + expect(variants(state)).toHaveLength(1); + }); + + it("doesn't flag a known name, nor one nothing like it", async () => { + const state = await withSecondScene( + { + entities: [person("Tomás"), person("Julián")], + facts: [fact("Julián", "action", "leyó", "Julián leyó")], + relations: [], + }, + "Tomás y Julián leyó la carta.", + ); + + expect(variants(state)).toEqual([]); + }); +}); + +describe("step 5, proofreading", () => { + const text = "Cuando halla leído la carta, ya no habrá vuelta atrás."; + const corrections = (...list: readonly { quote: string; fix: string; reason: string }[]) => + analyzeScene(start, scene("s1", text), depsFor({ corrections: list })); + + const fixes = (state: AnalysisState) => + state.issues.flatMap((issue) => (issue.kind === "correction" ? [issue] : [])); + + it("keeps a small, literal correction", async () => { + const { state } = await corrections({ + quote: "halla leído", + fix: "haya leído", + reason: "Es del verbo haber.", + }); + + expect(fixes(state)).toEqual([ + { + kind: "correction", + a: { sceneId: "s1", quote: "halla leído" }, + fix: "haya leído", + reason: "Es del verbo haber.", + }, + ]); + }); + + it.each([ + [ + "a rewrite of the author's sentence (rule 1)", + text, + "Tras leer la carta supo que no volvería.", + ], + ["a quote that isn't in the scene", "hayas leído", "hayas leídos"], + ["a change that changes nothing", "halla leído", "halla leído"], + ["an empty quote", "", "algo"], + ])("throws away %s", async (_, quote, fix) => { + const { state } = await corrections({ quote, fix, reason: "r" }); + + expect(fixes(state)).toEqual([]); + }); + + it.each([ + ["a mistyped word", "Cuando halla leído", "Cuando haya leído"], + ["just the mistyped word, with nothing around it", "halla", "haya"], + ["a word with another spelling", "la carta que echo", "la carta que hecho"], + ["two words that sound like one", "Haber si viene", "A ver si viene"], + ["only an accent", "Sabia que vendría", "Sabía que vendría"], + ["only capitals", "en puerto de mar", "en Puerto de mar"], + ])("accepts %s", async (_, quote, fix) => { + const { state } = await analyzeScene( + start, + scene("s1", `${quote} aquella tarde de lluvia.`), + depsFor({ corrections: [{ quote, fix, reason: "r" }] }), + ); + + expect(fixes(state)).toHaveLength(1); + }); + + it.each([ + ["another word (rule 1)", "Cerró la puerta del patio", "Abrió la puerta del patio"], + ["another tense, which changes the meaning", "Cuando halla leído", "Cuando había leído"], + ["a word added to the author's sentence", "fue a casa de Ana", "fue a su casa de Ana"], + ["a word taken from the author's sentence", "fue a su casa de Ana", "fue a casa de Ana"], + ])("throws away %s, however small", async (_, quote, fix) => { + const { state } = await analyzeScene( + start, + scene("s1", `${quote} aquella tarde de lluvia.`), + depsFor({ corrections: [{ quote, fix, reason: "r" }] }), + ); + + expect(fixes(state)).toEqual([]); + }); + + it("finds a hyphen opening a line of dialogue with no model at all", async () => { + const deps = depsFor({}); + + const { state } = await analyzeScene( + start, + scene("s1", "Tomás habló.\n\n- Nunca pensé verte, dijo.\n\n—Bien —dijo ella."), + deps, + ); + + expect(fixes(state)).toEqual([ + expect.objectContaining({ + a: { sceneId: "s1", quote: "- Nunca pensé verte" }, + fix: "—Nunca pensé verte", + }), + ]); + }); +}); + +describe("reanalysing a scene", () => { + const first: Extracted = { + entities: [person("Marta")], + facts: [fact("Marta", "physical", "verdes", "ojos verdes")], + relations: [], + }; + const second: Extracted = { + entities: [person("Marta")], + facts: [fact("Marta", "physical", "marrones", "ojos marrones")], + relations: [], + }; + + async function analysed() { + const deps = depsFor({ + extract: [first, second], + compare: (input) => [ + { + new_fact: tagOf(input, "ojos marrones"), + other_fact: tagOf(input, "ojos verdes"), + verdict: "contradiction", + reason: "r", + }, + ], + }); + const one = await analyzeScene(start, scene("s1", "Los ojos verdes."), deps); + const two = await analyzeScene(one.state, scene("s2", "Los ojos marrones."), deps); + return two.state; + } + + it("takes out what the old text said, and the issues citing it", async () => { + const state = await analysed(); + expect(state.issues).toHaveLength(1); + + const after = withoutScene(state, "s2"); + + expect(after.facts.map((item) => item.quote)).toEqual(["ojos verdes"]); + expect(after.mentions.every((mention) => mention.sceneId === "s1")).toBe(true); + expect(after.issues).toEqual([]); + }); + + it("replaces a scene's facts, so that a fixed contradiction goes away", async () => { + const state = await analysed(); + + const { state: fixed } = await analyzeScene( + state, + scene("s2", "Los ojos verdes, otra vez."), + depsFor({ + extract: [ + { + entities: [person("Marta")], + facts: [fact("Marta", "physical", "verdes", "ojos verdes")], + relations: [], + }, + ], + }), + ); + + expect(fixed.facts.filter((item) => item.sceneId === "s2").map((item) => item.quote)).toEqual([ + "ojos verdes", + ]); + expect(fixed.issues).toEqual([]); + }); + + it("can compare an earlier scene with a later one", async () => { + const state = await analysed(); + const deps = depsFor({ + extract: [first], + compare: (input) => { + expect(input).toMatch(/scene 1 .*ojos verdes\." NEW$/m); + expect(input).toMatch(/scene 2 .*ojos marrones\."$/m); + return [ + { + new_fact: tagOf(input, "ojos verdes"), + other_fact: tagOf(input, "ojos marrones"), + verdict: "contradiction", + reason: "r", + }, + ]; + }, + }); + + const { state: after } = await analyzeScene(state, scene("s1", "Los ojos verdes."), deps); + + expect(after.issues).toHaveLength(1); + }); + + it("leaves the state as it was if the model can't be reached", async () => { + const state = await analysed(); + const offline: AiProvider = { + id: "fake", + displayName: "Fake", + billing: "local", + listModels: async () => [], + async *generate(): AsyncGenerator { + yield { type: "end", outcome: { status: "interrupted", message: "offline" } }; + }, + }; + + await expect( + analyzeScene(state, scene("s2", "Otro texto."), { + provider: offline, + model: "m", + language: "es", + }), + ).rejects.toBeInstanceOf(StructuredOutputError); + expect(state.facts).toHaveLength(2); + expect(state.issues).toHaveLength(1); + }); +}); + +describe("every request", () => { + it("is deterministic, and asks for the schema of its step", async () => { + const deps = depsFor({ + extract: [ + { + entities: [person("Marta")], + facts: [fact("Marta", "action", "bajó", "Marta bajó")], + relations: [], + }, + ], + }); + + await analyzeScene(start, scene("s1", "Marta bajó."), deps); + + expect(deps.asked.length).toBeGreaterThan(0); + for (const request of deps.asked) { + expect(request.temperature).toBe(0); + expect(request.model).toBe("m"); + expect(request.structured?.schema).toBeTypeOf("object"); + } + }); + + it("never asks for a web search, which is paid and sends more than a topic (rule 5)", async () => { + const deps = depsFor({}); + + await analyzeScene(start, scene("s1", "Marta bajó del autobús."), deps); + + for (const request of deps.asked) { + expect(request.webSearch).toBeUndefined(); + } + }); + + it("passes on what the caller set, reasoning and cancelling included", async () => { + const controller = new AbortController(); + const deps = { ...depsFor({}), reasoning: "off" as const, signal: controller.signal }; + + await analyzeScene(start, scene("s1", "Marta bajó del autobús."), deps); + + expect(deps.asked[0]?.reasoning).toBe("off"); + expect(deps.asked[0]?.signal).toBe(controller.signal); + }); +}); diff --git a/packages/core/test/analysis/queue.test.ts b/packages/core/test/analysis/queue.test.ts new file mode 100644 index 0000000..189927c --- /dev/null +++ b/packages/core/test/analysis/queue.test.ts @@ -0,0 +1,525 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import * as Y from "yjs"; +import { + type AiProvider, + AnalysisQueue, + type AnalysisState, + type GenerateEvent, + type GenerateOutcome, + type GenerateRequest, + hashText, + openSqlStorage, + type QueueRun, + type Storage, +} from "../../src/index"; +import { createNodeSqliteDriver } from "../support/node-sqlite-driver"; +import { writeParagraphs } from "../support/prose"; + +/** + * A model that finds one fact per sentence of the first paragraph, enough to see what got analysed. + * `failWith` makes it fail, for as long as it's set. + */ +function fakeModel() { + const state: { failWith: GenerateOutcome | null; badAnswers: boolean; asked: GenerateRequest[] } = + { + failWith: null, + badAnswers: false, + asked: [], + }; + const provider: AiProvider = { + id: "fake", + displayName: "Fake", + billing: "local", + listModels: async () => [], + async *generate(request): AsyncGenerator { + state.asked.push(request); + if (request.signal?.aborted) { + yield { type: "end", outcome: { status: "interrupted", message: "cancelled" } }; + return; + } + if (state.failWith !== null) { + yield { type: "end", outcome: state.failWith }; + return; + } + if (state.badAnswers) { + yield { type: "text", delta: "no es json" }; + yield { type: "end", outcome: { status: "completed" } }; + return; + } + const text = request.input.split("SCENE ")[1]?.split(":\n")[1] ?? ""; + const quote = text.split(".")[0] ?? ""; + const answer = + request.structured?.name === "scene_extraction" + ? { + entities: [{ name: "Marta", type: "person", aliases: [] }], + facts: [ + { + entity: "Marta", + category: "action", + attribute: "a", + value: "v", + quote, + story_time: null, + }, + ], + relations: [], + } + : request.structured?.name === "proofreading" + ? { corrections: [] } + : request.structured?.name === "comparison" + ? { findings: [] } + : { answers: [] }; + yield { type: "text", delta: JSON.stringify(answer) }; + yield { type: "end", outcome: { status: "completed" } }; + }, + }; + return { provider, state }; +} + +describe("analysis queue", () => { + let driver: ReturnType; + let storage: Storage; + let projectId: string; + let chapterId: string; + let model: ReturnType; + + beforeEach(async () => { + driver = createNodeSqliteDriver(); + ({ storage } = await openSqlStorage(driver)); + projectId = (await storage.createProject({ title: "T", language: "es", era: "e" })).id; + chapterId = (await storage.createChapter(projectId, "C")).id; + model = fakeModel(); + }); + + afterEach(() => { + vi.useRealTimers(); + driver.close(); + }); + + /** Only timers are faked: hashing and the database still take real, if very little, time. */ + const fakeTimers = () => vi.useFakeTimers({ toFake: ["setTimeout", "clearTimeout"] }); + + /** Moves the clock on, and waits for whatever that set off. */ + async function advance(queue: AnalysisQueue, ms: number) { + await vi.advanceTimersByTimeAsync(ms); + await queue.settled(); + } + + const makeQueue = (overrides: Partial[0]> = {}) => + new AnalysisQueue({ + storage, + projectId, + analysis: { provider: model.provider, model: "m", language: "es" }, + ...overrides, + }); + + async function write(sceneId: string, text: string) { + const doc = new Y.Doc(); + const prose = await storage.loadSceneProse(sceneId); + if (prose.base !== null) Y.applyUpdate(doc, prose.base); + for (const update of prose.updates) Y.applyUpdate(doc, update.data); + doc.getXmlFragment("prose").delete(0, doc.getXmlFragment("prose").length); + writeParagraphs(doc, [text]); + await storage.appendSceneUpdate(sceneId, Y.encodeStateAsUpdate(doc)); + doc.destroy(); + } + + async function newScene(title: string, text: string) { + const scene = await storage.createScene(chapterId, title); + await write(scene.id, text); + return scene.id; + } + + const factsOf = (state: AnalysisState) => state.facts.map((fact) => fact.quote); + + it("analyses everything not yet analysed, in reading order", async () => { + const one = await newScene("1", "Marta llegó."); + const two = await newScene("2", "Marta salió."); + + const run = await makeQueue().run(); + + expect(run).toMatchObject({ status: "done", analyzed: [one, two], skipped: [], pending: 0 }); + expect(factsOf(await storage.loadAnalysis(projectId))).toEqual(["Marta llegó", "Marta salió"]); + }); + + it("doesn't ask again about text it already analysed, even after a restart", async () => { + await newScene("1", "Marta llegó."); + await makeQueue().run(); + const asked = model.state.asked.length; + + const run = await makeQueue().run(); + + expect(run.analyzed).toEqual([]); + expect(model.state.asked).toHaveLength(asked); + }); + + it("analyses a scene again only if its text changed", async () => { + const one = await newScene("1", "Marta llegó."); + const two = await newScene("2", "Marta salió."); + const queue = makeQueue(); + await queue.run(); + + await write(two, "Marta volvió."); + queue.sceneEdited(two); + const run = await queue.run(); + + expect(run.analyzed).toEqual([two]); + expect(run.analyzed).not.toContain(one); + expect(factsOf(await storage.loadAnalysis(projectId))).toEqual(["Marta llegó", "Marta volvió"]); + }); + + it("finds on opening what a crash left with an old hash", async () => { + const one = await newScene("1", "Marta llegó."); + await makeQueue().run(); + + // Saved by the editor, and then the app died before anything else could happen. + await write(one, "Marta volvió."); + + const run = await makeQueue().run(); + + expect(run.analyzed).toEqual([one]); + }); + + it("clears what an emptied scene said, without asking any model", async () => { + const one = await newScene("1", "Marta llegó."); + const queue = makeQueue(); + await queue.run(); + const asked = model.state.asked.length; + + await write(one, ""); + queue.sceneEdited(one); + const run = await queue.run(); + + expect(run.analyzed).toEqual([one]); + expect(factsOf(await storage.loadAnalysis(projectId))).toEqual([]); + expect(model.state.asked).toHaveLength(asked); + }); + + it("describes the text it read, so that writing meanwhile keeps the scene queued", async () => { + const one = await newScene("1", "Marta llegó."); + const original = storage.saveSceneAnalysis.bind(storage); + // The author types while the model is thinking. + storage.saveSceneAnalysis = async (...args) => { + await write(one, "Marta volvió."); + await storage.setSceneContentHash(one, await hashText("Marta volvió.")); + await original(...args); + }; + + await makeQueue().run(); + storage.saveSceneAnalysis = original; + + expect((await storage.listScenesToAnalyze(projectId)).map((scene) => scene.id)).toEqual([one]); + const run = await makeQueue().run(); + expect(run.analyzed).toEqual([one]); + expect(factsOf(await storage.loadAnalysis(projectId))).toEqual(["Marta volvió"]); + }); + + describe("without a connection", () => { + const offline: GenerateOutcome = { status: "interrupted", message: "fetch failed" }; + + it("leaves every scene queued and loses nothing", async () => { + const one = await newScene("1", "Marta llegó."); + await newScene("2", "Marta salió."); + model.state.failWith = offline; + + const run = await makeQueue().run(); + + expect(run).toMatchObject({ status: "waiting", analyzed: [], pending: 2 }); + expect(run.message).toBe("fetch failed"); + expect((await storage.listScenesToAnalyze(projectId))[0]?.id).toBe(one); + }); + + it("stops at the first failure, instead of asking for every scene", async () => { + await newScene("1", "Marta llegó."); + await newScene("2", "Marta salió."); + model.state.failWith = offline; + + await makeQueue().run(); + + expect(model.state.asked).toHaveLength(1); + }); + + it("keeps what was already done, and picks up where it stopped", async () => { + const one = await newScene("1", "Marta llegó."); + const two = await newScene("2", "Marta salió."); + let calls = 0; + const dropsLater: AiProvider = { + ...model.provider, + generate: (request) => { + calls += 1; + model.state.failWith = calls > 2 ? offline : null; + return model.provider.generate(request); + }, + }; + + const first = await makeQueue({ + analysis: { provider: dropsLater, model: "m", language: "es" }, + }).run(); + + expect(first).toMatchObject({ status: "waiting", analyzed: [one], pending: 1 }); + expect(factsOf(await storage.loadAnalysis(projectId))).toEqual(["Marta llegó"]); + + model.state.failWith = null; + const second = await makeQueue().run(); + expect(second).toMatchObject({ status: "done", analyzed: [two], pending: 0 }); + }); + + it("leaves a scene's old data alone until a new analysis finishes", async () => { + const one = await newScene("1", "Marta llegó."); + const queue = makeQueue(); + await queue.run(); + + await write(one, "Marta volvió."); + queue.sceneEdited(one); + model.state.failWith = offline; + await queue.run(); + + expect(factsOf(await storage.loadAnalysis(projectId))).toEqual(["Marta llegó"]); + }); + + it("tries again by itself, waiting longer each time, until it works", async () => { + fakeTimers(); + await newScene("1", "Marta llegó."); + model.state.failWith = offline; + const runs: QueueRun[] = []; + const queue = makeQueue({ retryMs: 1000, onRun: (run) => runs.push(run) }); + + queue.start(); + await advance(queue, 1); + expect(runs.map((run) => run.status)).toEqual(["waiting"]); + + await advance(queue, 1000); + expect(runs).toHaveLength(2); + + // Twice as long now: nothing at the first interval. + await advance(queue, 1000); + expect(runs).toHaveLength(2); + await advance(queue, 1000); + expect(runs).toHaveLength(3); + + model.state.failWith = null; + await advance(queue, 4000); + expect(runs.at(-1)?.status).toBe("done"); + + // And then it stops: there's nothing left to try. + const count = runs.length; + await advance(queue, 100_000); + expect(runs).toHaveLength(count); + await queue.stop(); + }); + }); + + describe("when refused", () => { + const noCredit: GenerateOutcome = { + status: "failed", + error: { code: "402", message: "Insufficient credits", httpStatus: 402 }, + }; + + it("stops, keeps everything queued and says why", async () => { + await newScene("1", "Marta llegó."); + await newScene("2", "Marta salió."); + model.state.failWith = noCredit; + + const run = await makeQueue().run(); + + expect(run).toMatchObject({ status: "blocked", pending: 2 }); + expect(run.message).toContain("Insufficient credits"); + expect(model.state.asked).toHaveLength(1); + }); + + it("doesn't keep asking by itself: only the author can sort this out", async () => { + fakeTimers(); + await newScene("1", "Marta llegó."); + model.state.failWith = noCredit; + const queue = makeQueue({ retryMs: 1000 }); + + queue.start(); + await advance(queue, 1); + await advance(queue, 1_000_000); + + expect(model.state.asked).toHaveLength(1); + await queue.stop(); + }); + + it("never turns to another provider (rule 10)", async () => { + await newScene("1", "Marta llegó."); + model.state.failWith = noCredit; + const other = fakeModel(); + const queue = makeQueue(); + + await queue.run(); + + expect(other.state.asked).toEqual([]); + expect(model.state.asked.every((request) => request.model === "m")).toBe(true); + }); + }); + + describe("when the model's answer isn't usable", () => { + it("goes on with the other scenes, instead of getting stuck on one", async () => { + const one = await newScene("1", "Marta llegó."); + const two = await newScene("2", "Marta salió."); + let calls = 0; + const failsTwice: AiProvider = { + ...model.provider, + generate: (request) => { + calls += 1; + model.state.badAnswers = calls <= 2; + return model.provider.generate(request); + }, + }; + + const run = await makeQueue({ + analysis: { provider: failsTwice, model: "m", language: "es" }, + }).run(); + + expect(run).toMatchObject({ status: "done", analyzed: [two], skipped: [one], pending: 0 }); + }); + + it("doesn't spend requests on that text again, until it changes", async () => { + const one = await newScene("1", "Marta llegó."); + model.state.badAnswers = true; + const queue = makeQueue(); + await queue.run(); + const asked = model.state.asked.length; + + const again = await queue.run(); + expect(again.skipped).toEqual([one]); + expect(model.state.asked).toHaveLength(asked); + + model.state.badAnswers = false; + await write(one, "Marta volvió."); + queue.sceneEdited(one); + const changed = await queue.run(); + expect(changed.analyzed).toEqual([one]); + }); + }); + + describe("when it runs (SPEC §5)", () => { + beforeEach(() => { + fakeTimers(); + }); + + it("never at each key: typing only restarts the wait", async () => { + const one = await newScene("1", "Marta llegó."); + const queue = makeQueue({ pauseMs: 1000 }); + + for (let key = 0; key < 50; key += 1) { + queue.sceneEdited(one); + await advance(queue, 100); + } + expect(model.state.asked).toEqual([]); + + await advance(queue, 1000); + expect(model.state.asked.length).toBeGreaterThan(0); + await queue.stop(); + }); + + it("on closing a scene, without waiting for a pause", async () => { + const one = await newScene("1", "Marta llegó."); + const queue = makeQueue({ pauseMs: 1_000_000 }); + + queue.sceneClosed(one); + await advance(queue, 1); + + expect(model.state.asked.length).toBeGreaterThan(0); + await queue.stop(); + }); + + it("doesn't start two runs at once", async () => { + await newScene("1", "Marta llegó."); + const queue = makeQueue(); + + const [a, b] = await Promise.all([queue.run(), queue.run()]); + + expect(a).toBe(b); + await queue.stop(); + }); + + it("goes back for what was edited during a run", async () => { + const one = await newScene("1", "Marta llegó."); + const runs: QueueRun[] = []; + const queue = makeQueue({ pauseMs: 1000, onRun: (run) => runs.push(run) }); + const original = storage.saveSceneAnalysis.bind(storage); + storage.saveSceneAnalysis = async (...args) => { + storage.saveSceneAnalysis = original; + await write(one, "Marta volvió."); + queue.sceneEdited(one); + await original(...args); + }; + + queue.start(); + await advance(queue, 1); + await advance(queue, 1000); + + expect(runs).toHaveLength(2); + expect(factsOf(await storage.loadAnalysis(projectId))).toEqual(["Marta volvió"]); + await queue.stop(); + }); + + it("doesn't start anything after being stopped", async () => { + const one = await newScene("1", "Marta llegó."); + const queue = makeQueue({ pauseMs: 1000 }); + queue.sceneEdited(one); + + await queue.stop(); + await advance(queue, 10_000); + queue.sceneClosed(one); + await advance(queue, 10_000); + + expect(model.state.asked).toEqual([]); + }); + }); + + it("stopping cancels what's in progress, and that scene stays queued", async () => { + const one = await newScene("1", "Marta llegó."); + let queue: AnalysisQueue | null = null; + const stopsMidway: AiProvider = { + ...model.provider, + generate(request) { + void queue?.stop(); + return model.provider.generate(request); + }, + }; + queue = makeQueue({ analysis: { provider: stopsMidway, model: "m", language: "es" } }); + + const run = await queue.run(); + + expect(run).toMatchObject({ status: "cancelled", analyzed: [], pending: 1 }); + expect((await storage.listScenesToAnalyze(projectId)).map((scene) => scene.id)).toEqual([one]); + expect(factsOf(await storage.loadAnalysis(projectId))).toEqual([]); + }); + + it("reports what isn't the model's fault, without taking it for being offline", async () => { + await newScene("1", "Marta llegó."); + const broken: Storage = { + ...storage, + saveSceneAnalysis: async () => { + throw new Error("disk full"); + }, + }; + const errors: unknown[] = []; + fakeTimers(); + const queue = makeQueue({ storage: broken, onError: (error) => errors.push(error) }); + + queue.start(); + await advance(queue, 1); + + expect(errors).toEqual([expect.objectContaining({ message: "disk full" })]); + await queue.stop(); + }); + + it("leaves the trash out", async () => { + const one = await newScene("1", "Marta llegó."); + await storage.trashScene(one); + + const run = await makeQueue().run(); + + expect(run.analyzed).toEqual([]); + }); + + it("hashes the same text the same way, and any change differently", async () => { + expect(await hashText("Marta")).toBe(await hashText("Marta")); + expect(await hashText("Marta")).not.toBe(await hashText("Marta.")); + expect(await hashText("Marta")).toMatch(/^[0-9a-f]{64}$/); + }); +}); diff --git a/packages/core/test/analysis/storage.test.ts b/packages/core/test/analysis/storage.test.ts new file mode 100644 index 0000000..855c4e3 --- /dev/null +++ b/packages/core/test/analysis/storage.test.ts @@ -0,0 +1,328 @@ +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { + type AnalysisState, + emptyState, + type Fact, + type Issue, + MIGRATIONS, + migrate, + openSqlStorage, + type Storage, + withoutScene, +} from "../../src/index"; +import { createNodeSqliteDriver } from "../support/node-sqlite-driver"; + +describe("analysis storage", () => { + let driver: ReturnType; + let storage: Storage; + let projectId: string; + let chapterId: string; + let s1: string; + let s2: string; + + beforeEach(async () => { + driver = createNodeSqliteDriver(); + ({ storage } = await openSqlStorage(driver)); + projectId = (await storage.createProject({ title: "T", language: "es", era: "e" })).id; + chapterId = (await storage.createChapter(projectId, "C")).id; + s1 = (await storage.createScene(chapterId, "Uno")).id; + s2 = (await storage.createScene(chapterId, "Dos")).id; + }); + + afterEach(() => driver.close()); + + const fact = (id: string, sceneId: string, quote: string): Fact => ({ + id, + sceneId, + entityId: "marta", + category: "physical", + attribute: "ojos", + value: "v", + quote, + sentence: `Frase con ${quote}.`, + storyTime: null, + inDialogue: false, + }); + + const contradiction = (quoteA: string, quoteB: string): Issue => ({ + kind: "contradiction", + a: { sceneId: s1, quote: quoteA }, + b: { sceneId: s2, quote: quoteB }, + reason: "r", + }); + + const base = (): AnalysisState => ({ + ...emptyState([s1, s2]), + entities: [{ id: "marta", type: "person", canonicalName: "Marta Rey", aliases: ["Marta"] }], + mentions: [{ sceneId: s1, entityId: "marta", quote: "Marta" }], + facts: [fact("f1", s1, "ojos verdes")], + }); + + it("gives back what it was given, in every detail", async () => { + const state: AnalysisState = { + ...base(), + entities: [ + ...base().entities, + { id: "tomas", type: "person", canonicalName: "Tomás", aliases: [] }, + ], + facts: [ + { ...fact("f1", s1, "ojos verdes"), storyTime: "a los doce", inDialogue: true }, + { ...fact("f2", s1, "otra"), category: "travel_time" }, + ], + relations: [ + { + id: "r1", + sceneId: s1, + fromEntityId: "marta", + toEntityId: "tomas", + label: "hermana de", + quote: "su hermano", + }, + ], + issues: [ + { kind: "correction", a: { sceneId: s1, quote: "halla" }, fix: "haya", reason: "r" }, + { + kind: "name_variant", + a: { sceneId: s1, quote: "Tomás" }, + b: { sceneId: s1, quote: "Thomas" }, + reason: "r", + }, + ], + }; + + await storage.saveSceneAnalysis(projectId, s1, "h1", state); + + expect(await storage.loadAnalysis(projectId)).toEqual(state); + }); + + it("replaces a scene's data when it's analysed again (SPEC §4)", async () => { + await storage.saveSceneAnalysis(projectId, s1, "h1", base()); + + await storage.saveSceneAnalysis(projectId, s1, "h2", { + ...base(), + mentions: [], + facts: [fact("f9", s1, "ojos grises")], + }); + + const after = await storage.loadAnalysis(projectId); + expect(after.facts.map((item) => item.quote)).toEqual(["ojos grises"]); + expect(after.mentions).toEqual([]); + }); + + it("leaves other scenes' data alone", async () => { + await storage.saveSceneAnalysis(projectId, s1, "h1", base()); + const withBoth: AnalysisState = { + ...base(), + facts: [...base().facts, fact("f2", s2, "ojos marrones")], + }; + await storage.saveSceneAnalysis(projectId, s2, "h2", withBoth); + + await storage.saveSceneAnalysis(projectId, s2, "h3", withoutScene(withBoth, s2)); + + const after = await storage.loadAnalysis(projectId); + expect(after.facts.map((item) => item.quote)).toEqual(["ojos verdes"]); + }); + + it("takes out an issue when either of its scenes is analysed again", async () => { + const state: AnalysisState = { + ...base(), + facts: [...base().facts, fact("f2", s2, "ojos marrones")], + issues: [contradiction("ojos verdes", "ojos marrones")], + }; + await storage.saveSceneAnalysis(projectId, s1, "h1", state); + await storage.saveSceneAnalysis(projectId, s2, "h2", state); + expect((await storage.loadAnalysis(projectId)).issues).toHaveLength(1); + + // Scene 1 no longer says that, so the issue doesn't come back. + await storage.saveSceneAnalysis(projectId, s1, "h3", withoutScene(state, s1)); + + expect((await storage.loadAnalysis(projectId)).issues).toEqual([]); + }); + + it("stores an issue only once, from whichever scene found it", async () => { + const state: AnalysisState = { + ...base(), + issues: [contradiction("ojos verdes", "ojos marrones")], + }; + + await storage.saveSceneAnalysis(projectId, s1, "h1", state); + await storage.saveSceneAnalysis(projectId, s2, "h2", state); + + expect((await storage.loadAnalysis(projectId)).issues).toHaveLength(1); + }); + + it("is all or nothing: a failure leaves the old data and the scene queued", async () => { + await storage.setSceneContentHash(s1, "h1"); + await storage.saveSceneAnalysis(projectId, s1, "h1", base()); + await storage.setSceneContentHash(s1, "h2"); + + const impossible: AnalysisState = { + ...base(), + facts: [{ ...fact("f9", s1, "x"), entityId: "no-such-entity" }], + }; + + await expect(storage.saveSceneAnalysis(projectId, s1, "h2", impossible)).rejects.toThrow(); + + expect((await storage.loadAnalysis(projectId)).facts.map((item) => item.quote)).toEqual([ + "ojos verdes", + ]); + expect((await storage.listScenesToAnalyze(projectId)).map((scene) => scene.id)).toEqual([s1]); + }); + + describe("entities", () => { + it("keep their name once stored: a model can't rename what the author may have set", async () => { + await storage.saveSceneAnalysis(projectId, s1, "h1", base()); + + await storage.saveSceneAnalysis(projectId, s1, "h2", { + ...base(), + entities: [{ id: "marta", type: "person", canonicalName: "Otra", aliases: ["La otra"] }], + }); + + const [entity] = (await storage.loadAnalysis(projectId)).entities; + expect(entity?.canonicalName).toBe("Marta Rey"); + expect(entity?.aliases).toEqual(["Marta", "La otra"]); + }); + + it("stay when their scenes are analysed again, so that their id does too (rule 4)", async () => { + await storage.saveSceneAnalysis(projectId, s1, "h1", base()); + + await storage.saveSceneAnalysis(projectId, s1, "h2", withoutScene(base(), s1)); + + const after = await storage.loadAnalysis(projectId); + expect(after.entities.map((entity) => entity.id)).toEqual(["marta"]); + expect(after.facts).toEqual([]); + }); + + it("belong to one project", async () => { + await storage.saveSceneAnalysis(projectId, s1, "h1", base()); + const other = await storage.createProject({ title: "O", language: "es", era: "e" }); + + expect((await storage.loadAnalysis(other.id)).entities).toEqual([]); + }); + }); + + describe("the queue", () => { + const queued = async () => + (await storage.listScenesToAnalyze(projectId)).map((scene) => scene.id); + + it("is empty until a hash says there's text", async () => { + expect(await queued()).toEqual([]); + }); + + it("has a scene from the moment its text differs, until that text is analysed", async () => { + await storage.setSceneContentHash(s1, "h1"); + expect(await queued()).toEqual([s1]); + + await storage.saveSceneAnalysis(projectId, s1, "h1", base()); + expect(await queued()).toEqual([]); + + await storage.setSceneContentHash(s1, "h2"); + expect(await queued()).toEqual([s1]); + }); + + it("keeps a scene if the text changed while it was being analysed", async () => { + await storage.setSceneContentHash(s1, "h2"); + + await storage.saveSceneAnalysis(projectId, s1, "h1", base()); + + expect(await queued()).toEqual([s1]); + }); + + it("is in reading order, across chapters", async () => { + const later = await storage.createChapter(projectId, "Después"); + const s3 = (await storage.createScene(later.id, "Tres")).id; + for (const id of [s3, s2, s1]) { + await storage.setSceneContentHash(id, "h"); + } + + expect(await queued()).toEqual([s1, s2, s3]); + + await storage.moveChapter(later.id, "up"); + expect(await queued()).toEqual([s3, s1, s2]); + }); + + it("leaves out the trash", async () => { + await storage.setSceneContentHash(s1, "h"); + await storage.setSceneContentHash(s2, "h"); + + await storage.trashScene(s1); + expect(await queued()).toEqual([s2]); + + await storage.trashChapter(chapterId); + expect(await queued()).toEqual([]); + }); + }); + + describe("the trash", () => { + const state = (): AnalysisState => ({ + ...base(), + facts: [...base().facts, fact("f2", s2, "ojos marrones")], + issues: [contradiction("ojos verdes", "ojos marrones")], + }); + + beforeEach(async () => { + await storage.saveSceneAnalysis(projectId, s1, "h1", state()); + await storage.saveSceneAnalysis(projectId, s2, "h2", state()); + }); + + it("hides what came from a scene, and the issues citing it", async () => { + await storage.trashScene(s2); + + const after = await storage.loadAnalysis(projectId); + expect(after.sceneOrder).toEqual([s1]); + expect(after.facts.map((item) => item.quote)).toEqual(["ojos verdes"]); + expect(after.issues).toEqual([]); + }); + + it("gives it all back on restoring, with no new analysis", async () => { + await storage.trashScene(s2); + await storage.restoreScene(s2); + + const after = await storage.loadAnalysis(projectId); + expect(after.facts).toHaveLength(2); + expect(after.issues).toHaveLength(1); + }); + + it("works through a whole chapter too", async () => { + await storage.trashChapter(chapterId); + + const after = await storage.loadAnalysis(projectId); + expect(after.sceneOrder).toEqual([]); + expect(after.facts).toEqual([]); + expect(after.issues).toEqual([]); + + await storage.restoreChapter(chapterId); + expect((await storage.loadAnalysis(projectId)).facts).toHaveLength(2); + }); + }); + + it("says the scenes in reading order, so that a comparison knows what came first", async () => { + await storage.moveScene(s2, "up"); + + expect((await storage.loadAnalysis(projectId)).sceneOrder).toEqual([s2, s1]); + }); + + it("the migration keeps the prose of a database from before it", async () => { + const old = createNodeSqliteDriver(); + await migrate( + old, + MIGRATIONS.filter((migration) => migration.version <= 2), + ); + await old.execute("INSERT INTO projects VALUES ('p', 'T', 'es', 'e')"); + await old.execute( + "INSERT INTO chapters (id, project_id, position, title) VALUES ('c', 'p', 0, 'C')", + ); + await old.execute( + "INSERT INTO scenes (id, chapter_id, position, title) VALUES ('s', 'c', 0, 'S')", + ); + const prose = new Uint8Array([1, 2, 3]); + await old.execute("INSERT INTO scene_updates (scene_id, update_data) VALUES ('s', ?)", [prose]); + + const { storage: upgraded, migration } = await openSqlStorage(old); + + expect(migration.from).toBe(2); + expect((await upgraded.loadSceneProse("s")).updates[0]?.data).toEqual(prose); + expect(await upgraded.listScenesToAnalyze("p")).toEqual([]); + old.close(); + }); +}); diff --git a/packages/core/test/analysis/structured.test.ts b/packages/core/test/analysis/structured.test.ts new file mode 100644 index 0000000..606b209 --- /dev/null +++ b/packages/core/test/analysis/structured.test.ts @@ -0,0 +1,215 @@ +import { describe, expect, it, vi } from "vitest"; +import { + type AiProvider, + type GenerateEvent, + type GenerateOutcome, + generateStructured, + StructuredOutputError, + type Validated, +} from "../../src/index"; + +/** Plays back one answer per call, and remembers the inputs it got. */ +function scripted(...answers: readonly (string | GenerateOutcome)[]) { + const inputs: string[] = []; + let next = 0; + const provider: AiProvider = { + id: "fake", + displayName: "Fake", + billing: "local", + listModels: async () => [], + async *generate(request): AsyncGenerator { + inputs.push(request.input); + const answer = answers[next]; + next += 1; + if (answer === undefined) { + throw new Error("Asked more times than expected"); + } + if (typeof answer === "string") { + yield { type: "text", delta: answer }; + yield { + type: "end", + outcome: { status: "completed" }, + usage: { inputTokens: 1, outputTokens: 1, cost: 0 }, + }; + } else { + yield { type: "end", outcome: answer }; + } + }, + }; + return { provider, inputs, calls: () => next }; +} + +const isThing = (value: unknown): Validated<{ thing: string }> => + typeof value === "object" && value !== null && "thing" in value + ? { ok: true, value: value as { thing: string } } + : { ok: false, problem: 'It needs "thing".' }; + +const request = { model: "m", instructions: "i", input: "pregunta" }; +const options = { sleep: async () => {} }; + +const failed = (code: string, retryAfterSeconds?: number): GenerateOutcome => ({ + status: "failed", + error: { + code, + message: "m", + ...(retryAfterSeconds === undefined ? {} : { retryAfterSeconds }), + }, +}); + +describe("generateStructured", () => { + it("returns a valid answer the first time", async () => { + const { provider, calls } = scripted('{"thing":"a"}'); + + await expect(generateStructured(provider, request, isThing, options)).resolves.toEqual({ + thing: "a", + }); + expect(calls()).toBe(1); + }); + + it("accepts JSON wrapped in a code fence, which some models add anyway", async () => { + const { provider } = scripted('```json\n{"thing":"a"}\n```'); + + await expect(generateStructured(provider, request, isThing, options)).resolves.toEqual({ + thing: "a", + }); + }); + + it("gives one repair retry, saying what was wrong (SPEC §5)", async () => { + const { provider, inputs } = scripted('{"other":1}', '{"thing":"b"}'); + + await expect(generateStructured(provider, request, isThing, options)).resolves.toEqual({ + thing: "b", + }); + expect(inputs[1]).toContain("pregunta"); + expect(inputs[1]).toContain('It needs "thing".'); + }); + + it("gives up after that one retry, saying the answer was the problem", async () => { + const { provider, calls } = scripted("no es json", "tampoco"); + + const error = await generateStructured(provider, request, isThing, options).catch((e) => e); + + expect(error).toBeInstanceOf(StructuredOutputError); + expect(error.failure).toBe("bad_answer"); + expect(calls()).toBe(2); + }); + + it("treats an answer cut off by length as something to repair", async () => { + const { provider, inputs } = scripted( + { status: "incomplete", reason: "length" }, + '{"thing":"a"}', + ); + + await expect(generateStructured(provider, request, isThing, options)).resolves.toEqual({ + thing: "a", + }); + expect(inputs[1]).toContain("shorter"); + }); + + it("waits out a rate limit, without using up the repair retry", async () => { + const { provider } = scripted(failed("429"), failed("503"), '{"other":1}', '{"thing":"a"}'); + const sleep = vi.fn(async () => {}); + + await expect(generateStructured(provider, request, isThing, { sleep })).resolves.toEqual({ + thing: "a", + }); + expect(sleep).toHaveBeenCalledTimes(2); + }); + + it("honours Retry-After, and otherwise waits longer each time", async () => { + const { provider } = scripted(failed("429", 7), failed("429"), failed("429"), '{"thing":"a"}'); + const waits: number[] = []; + + await generateStructured(provider, request, isThing, { + sleep: async (ms) => { + waits.push(ms); + }, + }); + + expect(waits).toEqual([7000, 10_000, 20_000]); + }); + + it("stops retrying a rate limit eventually, leaving the work for later", async () => { + const { provider, calls } = scripted(...Array.from({ length: 5 }, () => failed("429"))); + + const error = await generateStructured(provider, request, isThing, options).catch((e) => e); + + expect(error).toBeInstanceOf(StructuredOutputError); + expect(error.failure).toBe("later"); + expect(calls()).toBe(5); + }); + + it("doesn't wait hours for a long Retry-After, as with a daily limit", async () => { + const { provider, calls } = scripted(failed("429", 3600)); + const sleep = vi.fn(async () => {}); + + const error = await generateStructured(provider, request, isThing, { sleep }).catch((e) => e); + + expect(error.failure).toBe("later"); + expect(sleep).not.toHaveBeenCalled(); + expect(calls()).toBe(1); + }); + + it("stops waiting when cancelled", async () => { + const controller = new AbortController(); + const { provider } = scripted(failed("429", 30), { + status: "interrupted", + message: "cancelled", + }); + const started = Date.now(); + + const pending = generateStructured( + provider, + { ...request, signal: controller.signal }, + isThing, + ).catch((e) => e); + await new Promise((resolve) => setTimeout(resolve, 10)); + controller.abort(); + const error = await pending; + + expect(error.failure).toBe("later"); + expect(Date.now() - started).toBeLessThan(5000); + }); + + it("doesn't retry a refusal that can't change, and says so", async () => { + const { provider, calls } = scripted(failed("unsupported_capability")); + + const error = await generateStructured(provider, request, isThing, options).catch((e) => e); + + expect(error.failure).toBe("refused"); + expect(calls()).toBe(1); + }); + + it("treats no connection as something to try later", async () => { + const { provider, calls } = scripted({ status: "interrupted", message: "offline" }); + + const error = await generateStructured(provider, request, isThing, options).catch((e) => e); + + expect(error.failure).toBe("later"); + expect(error.message).toBe("offline"); + expect(calls()).toBe(1); + }); + + it("passes on the usage of every answer, including the rejected one", async () => { + const { provider } = scripted("no es json", '{"thing":"a"}'); + const onUsage = vi.fn(); + + await generateStructured(provider, request, isThing, { ...options, onUsage }); + + expect(onUsage).toHaveBeenCalledTimes(2); + }); + + it("stops at once if the usage callback refuses, as the spending guard does", async () => { + const { provider, calls } = scripted("no es json", '{"thing":"a"}'); + + await expect( + generateStructured(provider, request, isThing, { + ...options, + onUsage: () => { + throw new Error("cost"); + }, + }), + ).rejects.toThrow("cost"); + expect(calls()).toBe(1); + }); +}); diff --git a/packages/core/test/analysis/topics.test.ts b/packages/core/test/analysis/topics.test.ts new file mode 100644 index 0000000..d6d895e --- /dev/null +++ b/packages/core/test/analysis/topics.test.ts @@ -0,0 +1,130 @@ +import { describe, expect, it } from "vitest"; +import { + appositionsIn, + namesakesIn, + nameWords, + sentenceAround, + sentencesOf, + sentencesWithoutFacts, + speaksOfDeath, + topicWords, +} from "../../src/analysis/topics"; + +describe("topicWords", () => { + const words = (text: string, excluded?: ReadonlySet) => [...topicWords(text, excluded)]; + + it("keeps what a sentence is about", () => { + expect(words("la veda empieza el martes")).toEqual(["veda", "empieza"]); + expect(words("sus ojos marrones")).toEqual(["ojo", "marrone"]); + }); + + it("finds one word in singular and plural, and with or without an accent", () => { + const both = (a: string, b: string) => + [...topicWords(a)].some((word) => topicWords(b).has(word)); + + expect(both("las maletas rojas", "una maleta")).toBe(true); + expect(both("la camión", "el camion")).toBe(true); + }); + + it("doesn't count short words, common words or times", () => { + expect(words("Hacía once años que se había ido")).toEqual(["once"]); + expect(words("esa misma noche, un martes de lluvia")).toEqual(["lluvia"]); + }); + + it("leaves out people's names, or every fact about someone would share a topic", () => { + const people = new Set(nameWords("Marta Rey")); + + expect(words("Marta miró el faro", people)).toEqual(["miro", "faro"]); + }); + + it("works in English too", () => { + expect(words("the fishing ban starts on Tuesday")).toEqual(["fishing", "start"]); + }); +}); + +describe("speaksOfDeath", () => { + it.each([ + "Enterraron a la tía Remedios", + "Remedios murió en mayo", + "su difunta tía", + "el entierro fue el lunes", + "She was buried on Tuesday", + "He died young", + ])("%s", (text) => { + expect(speaksOfDeath(text)).toBe(true); + }); + + it.each(["Marta recibió una llamada", "He went to the bakery", "el murió"])( + "not in: %s", + (text) => { + expect(speaksOfDeath(text)).toBe(text === "el murió"); + }, + ); +}); + +describe("sentences", () => { + const text = "Marta bajó. ¿Quién llama?\n\n—Soy yo —dijo Tomás. Fin"; + + it("splits by end marks and by lines", () => { + expect(sentencesOf(text)).toEqual([ + "Marta bajó.", + "¿Quién llama?", + "—Soy yo —dijo Tomás.", + "Fin", + ]); + }); + + it("finds the sentence around a quote", () => { + expect(sentenceAround(text, "dijo Tomás")).toBe("—Soy yo —dijo Tomás."); + expect(sentenceAround(text, "no está")).toBe("no está"); + }); + + it("lists sentences no quote touches, skipping the very short ones", () => { + const scene = "El jueves empezaba la veda. Inés ya había amarrado el barco. Fin."; + + expect(sentencesWithoutFacts(scene, ["Inés ya había amarrado"])).toEqual([ + "El jueves empezaba la veda.", + ]); + expect(sentencesWithoutFacts(scene, [scene])).toEqual([]); + expect(sentencesWithoutFacts("", [])).toEqual([]); + }); +}); + +describe("people the text keeps apart", () => { + const text = "—Me la dio Luis, el cartero.\n\nLuis Ferrer, el alcalde, entró sin llamar."; + + it("reads a name with what that person is", () => { + expect(appositionsIn(text)).toEqual([ + { name: "Luis", epithet: "el cartero", text: "Luis, el cartero" }, + { name: "Luis Ferrer", epithet: "el alcalde", text: "Luis Ferrer, el alcalde" }, + ]); + }); + + it("doesn't take a list or a clause for that", () => { + expect( + appositionsIn("Marta, Tomás y Julián. Luis, que era cartero. En Madrid, la capital."), + ).toEqual([{ name: "Madrid", epithet: "la capital", text: "Madrid, la capital" }]); + }); + + it("finds a shorter name with one description and a longer one with another", () => { + const [pair] = namesakesIn(text); + + expect(pair?.[0].name).toBe("Luis"); + expect(pair?.[1].name).toBe("Luis Ferrer"); + }); + + it("whichever comes first", () => { + const [pair] = namesakesIn("Luis Ferrer, el alcalde, saludó a Luis, el cartero."); + + expect(pair?.[0].name).toBe("Luis"); + }); + + it.each([ + ["the same name twice", "Luis, el cartero. Luis, el alcalde."], + ["the same description", "Luis, el cartero. Luis Ferrer, el cartero."], + ["unrelated names", "Marta, la doctora. Luis, el cartero."], + ["no appositions", "Luis entró. Luis Ferrer salió."], + ])("doesn't separate with %s", (_, sample) => { + expect(namesakesIn(sample)).toEqual([]); + }); +}); diff --git a/packages/core/test/migrate.test.ts b/packages/core/test/migrate.test.ts index a04b4dd..4bf07cf 100644 --- a/packages/core/test/migrate.test.ts +++ b/packages/core/test/migrate.test.ts @@ -38,7 +38,13 @@ describe("migrate", () => { expect(await version(driver)).toBe(MIGRATIONS.length); expect(await tableNames(driver)).toEqual([ "chapters", + "entities", + "entity_aliases", + "facts", + "issues", + "mentions", "projects", + "relations", "scene_snapshots", "scene_updates", "scenes", diff --git a/packages/core/tsconfig.json b/packages/core/tsconfig.json index 9241f01..f2f4563 100644 --- a/packages/core/tsconfig.json +++ b/packages/core/tsconfig.json @@ -3,5 +3,5 @@ "compilerOptions": { "types": ["node"] }, - "include": ["src", "test"] + "include": ["src", "test", "eval"] }