refs/Taskfile.yml at main · neoworks-dev/refs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
version: '3'

# Everything runs from the repo root, except the model service, which lives in ml/.
# Pass arguments after `--`, e.g. `task crawl -- https://linear.app --viewport both`.

tasks:
  db:
    desc: Start SurrealDB in docker on 127.0.0.1:8010
    cmds:
      - docker compose up -d surrealdb

  dev:
    desc: Start the database and the app
    deps: [db]
    cmds:
      - bun run dev {{.CLI_ARGS}}

  crawler:
    desc: Start the worker that picks up queued crawls
    deps: [db]
    cmds:
      - bun crawler/worker.ts

  ml:
    desc: Start the model servers and their API (VLM on 8020, embedder on 8021, API on 8022)
    dir: ml
    cmds:
      - uv run serve {{.CLI_ARGS}}

  ml:download:
    desc: Download the model weights
    dir: ml
    cmds:
      - uv run serve download

  ml:tunnel:
    desc: 'Reach the models on a rented GPU through SSH, e.g. task ml:tunnel -- root@1.2.3.4 -p 12345'
    cmds:
      - ssh -N -L 8022:127.0.0.1:8022 {{.CLI_ARGS}}

  up:
    desc: Run the app, the crawler worker and the model service together
    deps: [dev, crawler, ml]

  crawl:
    desc: 'Crawl a site now, e.g. task crawl -- https://linear.app --viewport both'
    deps: [db]
    cmds:
      - bun crawler/cli.ts {{.CLI_ARGS}}

  enrich:
    desc: Run the models over the current crawls, optionally of one site slug
    cmds:
      - bun crawler/enrich.ts {{.CLI_ARGS}}

  explore:
    desc: Let the model set goals for the current crawls and try them, optionally of one site slug
    cmds:
      - bun crawler/explore.ts {{.CLI_ARGS}}

  flows:
    desc: Recut the flows of the current crawls, optionally of one site slug
    cmds:
      - bun crawler/flows.ts {{.CLI_ARGS}}

  api-key:
    desc: 'Manage MCP API keys, e.g. task api-key -- create "Laptop"'
    cmds:
      - bun scripts/api-key.ts {{.CLI_ARGS}}

  check:
    desc: Type check everything, including the crawler and tests
    cmds:
      - bun run check

  test:
    desc: Run the crawler tests
    cmds:
      - bun test {{.CLI_ARGS}}

  build:
    desc: Build the app
    cmds:
      - bun run build