From 38e5c4415f5a82a3853c4d47b4e41a852fc7d3c8 Mon Sep 17 00:00:00 2001 From: mouldybread <38598858+mouldybread@users.noreply.github.com> Date: Sat, 21 Nov 2026 10:48:43 +0800 Subject: [PATCH 1/8] Add bounded disk-backed export job lifecycle --- docs/LARGE_MEDIA_EXPORTS.md | 39 ++++++ export_jobs.go | 265 ++++++++++++++++++++++++++++++++++++ export_jobs_test.go | 116 ++++++++++++++++ 3 files changed, 420 insertions(+) create mode 100644 docs/LARGE_MEDIA_EXPORTS.md create mode 100644 export_jobs.go create mode 100644 export_jobs_test.go diff --git a/docs/LARGE_MEDIA_EXPORTS.md b/docs/LARGE_MEDIA_EXPORTS.md new file mode 100644 index 0000000..9ea57de --- /dev/null +++ b/docs/LARGE_MEDIA_EXPORTS.md @@ -0,0 +1,39 @@ +# Large-media export implementation status + +This branch is work in progress. The first commit adds an isolated server-side export job store and tests only. It does NOT yet register HTTP routes, change the UI, remove the browser Blob limit, or broaden rendering compatibility. Do not release this commit as the completed export redesign. + +## Implemented foundation + +- Bounded jobs and preparation concurrency, random job IDs, and explicit owner checks. +- Private disk-backed output, immediately unlinked and retained by file descriptor; no recording cleanup or source modification. +- Independent seekable download leases for eventual HTTP Range delivery, without page-level whole-file buffering. +- Cancellation, expiry, disk-reservation accounting, bounded writes, and shutdown joining. Existing transfers retain their leases until released. +- Safe public status messages; raw renderer errors are not exposed. + +The renderer callback must validate the export and join its subprocesses before returning. The future HTTP integration must bind owners to authenticated sessions/credential versions, never caller-supplied owner strings. It must cancel and drain downloads before waiting for store shutdown. + +## Planned startup configuration + +These settings are parsed by the new helper but are NOT wired into startup yet: + +| Setting | Default | Meaning | +| --- | --- | --- | +| NVR_EXPORT_MAX_BYTES | 8589934592 | Maximum prepared output per job (8 GiB) | +| NVR_EXPORT_TOTAL_BYTES | 17179869184 | Aggregate reserved output capacity (16 GiB) | +| NVR_EXPORT_MAX_JOBS | 8 | Maximum retained job records | +| NVR_EXPORT_PARALLEL | 1 | Concurrent preparations | +| NVR_EXPORT_TTL | 30m | Queue/processing expiry; retention resets on completion | +| NVR_EXPORT_PREPARE_TIMEOUT | 30m | Deadline including queue admission wait | + +Each admitted job reserves its maximum output allowance; default aggregate capacity therefore admits at most two such reservations at once. Expired files held by active downloads remain charged. Invalid limits fail closed. Filesystem exhaustion can still fail a job; reservations do not reserve physical blocks or replace filesystem capacity monitoring. Use trusted private scratch storage with sufficient free space. + +## Remaining integration + +1. Authenticated POST creation/cancellation and GET status/download endpoints; no state-changing GET requests or bearer-token bypass. +2. Startup, shutdown-gate and HTTP transfer cancellation wiring. +3. Original-file delivery, compatible audio/video stream-copy assembly, and explicit preview/consent for keyframe-adjusted boundaries. +4. Optional accurate trimming preserving audio timing and handling full-range input correctly; no silent audio removal or conversion fallback. +5. Browser job UI without whole-file Blob buffering; update real-browser acceptance tests. +6. Media, authentication, quota, expiry, cancellation, shutdown, large-transfer and source-preservation integration tests. + +The Go tests accompanying the first commit have not been executed in the assistant environment; Go is unavailable there. diff --git a/export_jobs.go b/export_jobs.go new file mode 100644 index 0000000..1ba85b3 --- /dev/null +++ b/export_jobs.go @@ -0,0 +1,265 @@ +package main + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "io" + "os" + "strconv" + "sync" + "time" +) + +var ( + errExportJobsBusy = errors.New("export job capacity exceeded") + errExportJobMissing = errors.New("export job unavailable") + errExportJobsClosed = errors.New("export service stopping") + errExportJobSize = errors.New("export output size limit exceeded") +) + +type exportJobLimits struct { + MaxBytes, TotalBytes int64 + MaxJobs, Parallel int + Lifetime, Preparation time.Duration +} + +func defaultExportJobLimits() exportJobLimits { + return exportJobLimits{MaxBytes:8<<30, TotalBytes:16<<30, MaxJobs:8, Parallel:1, Lifetime:30*time.Minute, Preparation:30*time.Minute} +} + +func (l exportJobLimits) validate() error { + if l.MaxBytes<1 || l.TotalBytes64 || l.Parallel<1 || l.Parallel>l.MaxJobs || l.Lifetime<=0 || l.Lifetime>24*time.Hour || l.Preparation<=0 || l.Preparation>24*time.Hour { + return errors.New("invalid export job resource limits") + } + return nil +} + +// Configuration is read once by the future application startup wiring. No +// environment is read at package initialization and invalid values fail closed. +func loadExportJobLimits(getenv func(string)string) (exportJobLimits,error) { + limits:=defaultExportJobLimits() + for name,target:=range map[string]*int64{"NVR_EXPORT_MAX_BYTES":&limits.MaxBytes,"NVR_EXPORT_TOTAL_BYTES":&limits.TotalBytes} { + if raw:=getenv(name);raw!="" {value,err:=strconv.ParseInt(raw,10,64);if err!=nil {return limits,errors.New("invalid "+name)};*target=value} + } + for name,target:=range map[string]*int{"NVR_EXPORT_MAX_JOBS":&limits.MaxJobs,"NVR_EXPORT_PARALLEL":&limits.Parallel} { + if raw:=getenv(name);raw!="" {value,err:=strconv.Atoi(raw);if err!=nil {return limits,errors.New("invalid "+name)};*target=value} + } + for name,target:=range map[string]*time.Duration{"NVR_EXPORT_TTL":&limits.Lifetime,"NVR_EXPORT_PREPARE_TIMEOUT":&limits.Preparation} { + if raw:=getenv(name);raw!="" {value,err:=time.ParseDuration(raw);if err!=nil {return limits,errors.New("invalid "+name)};*target=value} + } + return limits,limits.validate() +} + +type exportJobView struct { + ID string `json:"id"` + State string `json:"state"` + Message string `json:"message"` + Bytes int64 `json:"bytes"` + ExpiresAt time.Time `json:"expires_at"` +} + +type preparedExportJob struct { + view exportJobView + owner string + ctx context.Context + cancel context.CancelFunc + file *os.File + running bool + readers int + reserved int64 +} + +// Renderers must finish/join all subprocesses before returning and validate +// their output before success. They never own or retain the output writer. +type exportJobRenderer func(context.Context,io.Writer) error + +type exportJobStore struct { + mu sync.Mutex + limits exportJobLimits + directory string + jobs map[string]*preparedExportJob + reserved int64 + slots chan struct{} + ctx context.Context + cancel context.CancelFunc + workers sync.WaitGroup + closing, workersDone, drainedClosed bool + closeOnce sync.Once + drained chan struct{} + cleanupErr error +} + +// parentDirectory must be a private, trusted export scratch filesystem, not +// camera storage. A unique directory and immediately unlinked outputs avoid +// crash leftovers and never require globbing/deleting existing recordings. +func newExportJobStore(parent context.Context,parentDirectory string,limits exportJobLimits) (*exportJobStore,error) { + if err:=limits.validate();err!=nil {return nil,err} + if err:=parent.Err();err!=nil {return nil,err} + directory,err:=os.MkdirTemp(parentDirectory,"simplenvr-exports-");if err!=nil {return nil,err} + ctx,cancel:=context.WithCancel(parent) + store:=&exportJobStore{limits:limits,directory:directory,jobs:make(map[string]*preparedExportJob),slots:make(chan struct{},limits.Parallel),ctx:ctx,cancel:cancel,drained:make(chan struct{})} + store.workers.Add(1) + go func(){ + defer store.workers.Done() + interval:=limits.Lifetime/2;if interval>time.Minute {interval=time.Minute};if interval=s.limits.MaxJobs || s.limits.MaxBytes>s.limits.TotalBytes-s.reserved {return exportJobView{},errExportJobsBusy} + ctx,cancel:=context.WithTimeout(s.ctx,s.limits.Preparation) + job:=&preparedExportJob{view:exportJobView{ID:id,State:"queued",Message:"Queued for export preparation",ExpiresAt:time.Now().Add(s.limits.Lifetime)},owner:owner,ctx:ctx,cancel:cancel,running:true,reserved:s.limits.MaxBytes} + s.jobs[id]=job;s.reserved+=job.reserved + s.workers.Add(1) + go s.run(job,render) + return job.view,nil +} + +type boundedExportJobWriter struct { + ctx context.Context + file *os.File + limit,written int64 + exceeded bool +} +func (w *boundedExportJobWriter) Write(data []byte) (int,error) { + if err:=w.ctx.Err();err!=nil {return 0,err} + if int64(len(data))>w.limit-w.written {w.exceeded=true;return 0,errExportJobSize} + n,err:=w.file.Write(data);w.written+=int64(n);return n,err +} + +func (s *exportJobStore) run(job *preparedExportJob,render exportJobRenderer) { + defer s.workers.Done();defer job.cancel() + select { + case s.slots<-struct{}{}:defer func(){<-s.slots}() + case <-job.ctx.Done():s.finish(job,nil,0,job.ctx.Err());return + } + if err:=job.ctx.Err();err!=nil {s.finish(job,nil,0,err);return} + s.mu.Lock() + if job.view.State!="queued" {s.mu.Unlock();s.finish(job,nil,0,context.Canceled);return} + job.view.State="preparing";job.view.Message="Preparing export on server" + s.mu.Unlock() + file,err:=os.CreateTemp(s.directory,"output-*.mp4") + if err!=nil {s.finish(job,nil,0,err);return} + // CreateTemp is owner-only. Unlink before handing the descriptor to a + // renderer; the kernel releases output bytes after the last reader closes. + if err:=os.Remove(file.Name());err!=nil {closeErr:=file.Close();removeErr:=os.Remove(file.Name());s.mu.Lock();s.cleanupErr=errors.Join(s.cleanupErr,closeErr,removeErr);s.mu.Unlock();s.finish(job,nil,0,err);return} + writer:=&boundedExportJobWriter{ctx:job.ctx,file:file,limit:s.limits.MaxBytes} + err=render(job.ctx,writer) + if writer.exceeded {err=errExportJobSize} + if err==nil {err=job.ctx.Err()} + if err==nil && writer.written==0 {err=errors.New("empty export")} + s.finish(job,file,writer.written,err) +} + +func (s *exportJobStore) finish(job *preparedExportJob,file *os.File,size int64,err error) { + s.mu.Lock();defer s.mu.Unlock() + job.running=false;job.file=file + if job.view.State=="canceled" || job.view.State=="expired" || s.closing || s.ctx.Err()!=nil { + if job.view.State!="expired" {job.view.State="canceled";job.view.Message="Export canceled"} + } else if err!=nil { + job.view.State="failed";job.view.Message="Export preparation failed; no download is ready" + if errors.Is(err,errExportJobSize) {job.view.Message="Export exceeds the configured server output limit"} + if errors.Is(err,context.DeadlineExceeded) {job.view.Message="Export preparation deadline exceeded"} + } else { + job.view.State="ready";job.view.Message="Ready to download";job.view.Bytes=size + } + job.view.ExpiresAt=time.Now().Add(s.limits.Lifetime) + if job.view.State!="ready" {s.discardLocked(job)} + if s.closing || job.view.State=="expired" {delete(s.jobs,job.view.ID)} +} + +func (s *exportJobStore) discardLocked(job *preparedExportJob) { + if job.running || job.readers>0 {return} + if job.file!=nil {s.cleanupErr=errors.Join(s.cleanupErr,job.file.Close());job.file=nil} + s.reserved-=job.reserved;job.reserved=0 +} + +func (s *exportJobStore) expire(now time.Time) { + s.mu.Lock();defer s.mu.Unlock() + for id,job:=range s.jobs { + if now.Before(job.view.ExpiresAt) {continue} + job.view.State="expired";job.view.Message="Prepared export expired";job.cancel() + s.discardLocked(job) + if !job.running && job.readers==0 {delete(s.jobs,id)} + } +} + +func (s *exportJobStore) status(owner,id string) (exportJobView,error) { + s.expire(time.Now()) + s.mu.Lock();defer s.mu.Unlock() + job:=s.jobs[id] + if s.closing || s.ctx.Err()!=nil {return exportJobView{},errExportJobsClosed} + if owner=="" || job==nil || job.owner!=owner {return exportJobView{},errExportJobMissing} + return job.view,nil +} + +func (s *exportJobStore) cancelJob(owner,id string) error { + s.mu.Lock();defer s.mu.Unlock() + job:=s.jobs[id] + if owner=="" || job==nil || job.owner!=owner {return errExportJobMissing} + job.cancel();job.view.State="canceled";job.view.Message="Export canceled" + s.discardLocked(job) + return nil +} + +// Each download gets an independent SectionReader; concurrent Range requests +// never share a seek offset. Expiry/cancel refuses new leases but does not close +// the descriptor out from under an existing transfer. release must be deferred. +func (s *exportJobStore) acquire(owner,id string) (*io.SectionReader,func(),error) { + s.expire(time.Now()) + s.mu.Lock();defer s.mu.Unlock() + job:=s.jobs[id] + if s.closing || s.ctx.Err()!=nil {return nil,nil,errExportJobsClosed} + if owner=="" || job==nil || job.owner!=owner || job.view.State!="ready" || job.file==nil {return nil,nil,errExportJobMissing} + job.readers++ + reader:=io.NewSectionReader(job.file,0,job.view.Bytes) + var once sync.Once + release:=func(){once.Do(func(){ + s.mu.Lock();defer s.mu.Unlock() + job.readers-- + if job.view.State!="ready" || s.closing || s.ctx.Err()!=nil {s.discardLocked(job)} + if job.readers==0 && !job.running && (s.closing || job.view.State=="expired") {delete(s.jobs,id)} + s.maybeDrainedLocked() + })} + return reader,release,nil +} + +func (s *exportJobStore) maybeDrainedLocked() { + if !s.closing || !s.workersDone || s.drainedClosed {return} + for _,job:=range s.jobs {if job.readers>0 || job.running {return}} + s.cleanupErr=errors.Join(s.cleanupErr,os.Remove(s.directory)) + s.drainedClosed=true;close(s.drained) +} + +// Application wiring must cancel/drain HTTP downloads before waiting here. A +// timed-out Close never closes active readers; callers may retry waiting later. +func (s *exportJobStore) Close(ctx context.Context) error { + s.closeOnce.Do(func(){ + s.mu.Lock();s.closing=true;s.cancel() + for _,job:=range s.jobs {job.cancel();job.view.State="canceled";job.view.Message="Export service stopping"} + s.mu.Unlock() + go func(){ + s.workers.Wait() + s.mu.Lock();defer s.mu.Unlock() + s.workersDone=true + for id,job:=range s.jobs {s.discardLocked(job);if job.readers==0 {delete(s.jobs,id)}} + s.maybeDrainedLocked() + }() + }) + select { + case <-s.drained:s.mu.Lock();defer s.mu.Unlock();return s.cleanupErr + case <-ctx.Done():return ctx.Err() + } +} diff --git a/export_jobs_test.go b/export_jobs_test.go new file mode 100644 index 0000000..a52cb99 --- /dev/null +++ b/export_jobs_test.go @@ -0,0 +1,116 @@ +package main + +import ( + "context" + "errors" + "io" + "os" + "strings" + "testing" + "time" +) + +func testExportJobs(t *testing.T,maxBytes int64,total int64) *exportJobStore { + t.Helper() + limits:=defaultExportJobLimits();limits.MaxBytes=maxBytes;limits.TotalBytes=total;limits.Lifetime=time.Hour + store,err:=newExportJobStore(context.Background(),t.TempDir(),limits);if err!=nil {t.Fatal(err)} + t.Cleanup(func(){ctx,cancel:=context.WithTimeout(context.Background(),3*time.Second);defer cancel();if err:=store.Close(ctx);err!=nil {t.Error(err)}}) + return store +} + +func awaitExportJob(t *testing.T,s *exportJobStore,id,state string) exportJobView { + t.Helper();deadline:=time.After(3*time.Second);ticker:=time.NewTicker(time.Millisecond);defer ticker.Stop() + for { + view,err:=s.status("owner",id);if err!=nil {t.Fatal(err)} + if view.State==state {return view} + select {case <-deadline:t.Fatalf("wanted %s, got %+v",state,view);case <-ticker.C:} + } +} + +func TestExportJobLimits(t *testing.T) { + defaults,err:=loadExportJobLimits(func(string)string{return ""});if err!=nil || defaults.MaxBytes!=8<<30 || defaults.TotalBytes!=16<<30 {t.Fatalf("%+v %v",defaults,err)} + for _,values:=range []map[string]string{ + {"NVR_EXPORT_MAX_BYTES":"0"}, + {"NVR_EXPORT_MAX_BYTES":"no"}, + {"NVR_EXPORT_MAX_BYTES":"9223372036854775808"}, + {"NVR_EXPORT_TOTAL_BYTES":"1"}, + {"NVR_EXPORT_MAX_JOBS":"65"}, + {"NVR_EXPORT_PARALLEL":"0"}, + {"NVR_EXPORT_TTL":"0s"}, + {"NVR_EXPORT_PREPARE_TIMEOUT":"25h"}, + } {if _,err:=loadExportJobLimits(func(k string)string{return values[k]});err==nil {t.Fatalf("accepted invalid limits: %v",values)}} + values:=map[string]string{"NVR_EXPORT_MAX_BYTES":"1073741824","NVR_EXPORT_TOTAL_BYTES":"2147483648","NVR_EXPORT_TTL":"1h"} + custom,err:=loadExportJobLimits(func(k string)string{return values[k]});if err!=nil || custom.MaxBytes!=1<<30 || custom.Lifetime!=time.Hour {t.Fatalf("%+v %v",custom,err)} +} + +func TestExportJobsReadyAuthorizationAndIndependentReaders(t *testing.T) { + s:=testExportJobs(t,1024,2048) + view,err:=s.submit("owner",func(ctx context.Context,w io.Writer)error{_,err:=io.WriteString(w,"abcdef");return err});if err!=nil {t.Fatal(err)} + ready:=awaitExportJob(t,s,view.ID,"ready");if ready.Bytes!=6 {t.Fatal(ready)} + if _,err:=s.status("other",view.ID);!errors.Is(err,errExportJobMissing) {t.Fatal("cross-owner status allowed")} + if err:=s.cancelJob("other",view.ID);!errors.Is(err,errExportJobMissing) {t.Fatal("cross-owner cancellation allowed")} + if _,_,err:=s.acquire("other",view.ID);!errors.Is(err,errExportJobMissing) {t.Fatal("cross-owner download allowed")} + a,releaseA,err:=s.acquire("owner",view.ID);if err!=nil {t.Fatal(err)};defer releaseA() + b,releaseB,err:=s.acquire("owner",view.ID);if err!=nil {t.Fatal(err)};defer releaseB() + if _,err:=a.Seek(3,io.SeekStart);err!=nil {t.Fatal(err)} + first,_:=io.ReadAll(a);second,_:=io.ReadAll(b) + if string(first)!="def" || string(second)!="abcdef" {t.Fatalf("shared download offsets: %q %q",first,second)} + files,err:=os.ReadDir(s.directory);if err!=nil || len(files)!=0 {t.Fatalf("named temporary output remains: %v %v",files,err)} +} + +func TestExportJobsBoundOutputAndSanitizeErrors(t *testing.T) { + for _,tc:=range []struct{name string;render exportJobRenderer;message string}{ + {"size",func(ctx context.Context,w io.Writer)error{_,err:=io.WriteString(w,"12345");return err},"output limit"}, + {"ignored write failure",func(ctx context.Context,w io.Writer)error{w.Write([]byte("12345"));return nil},"output limit"}, + {"empty",func(context.Context,io.Writer)error{return nil},"preparation failed"}, + {"secret",func(context.Context,io.Writer)error{return errors.New("rtsp://user:SECRET@camera/private")},"preparation failed"}, + } {t.Run(tc.name,func(t *testing.T){ + s:=testExportJobs(t,4,4);job,err:=s.submit("owner",tc.render);if err!=nil {t.Fatal(err)} + failed:=awaitExportJob(t,s,job.ID,"failed") + if !strings.Contains(failed.Message,tc.message) || strings.Contains(failed.Message,"SECRET") {t.Fatal(failed)} + s.mu.Lock();reserved:=s.reserved;s.mu.Unlock();if reserved!=0 {t.Fatal("failed output reservation leaked")} + if _,_,err:=s.acquire("owner",job.ID);!errors.Is(err,errExportJobMissing) {t.Fatal("failed output downloadable")} + })} +} + +func TestExportJobsExpiryRetainsActiveTransferAndQuota(t *testing.T) { + s:=testExportJobs(t,8,8) + render:=func(ctx context.Context,w io.Writer)error{_,err:=io.WriteString(w,"video");return err} + job,err:=s.submit("owner",render);if err!=nil {t.Fatal(err)};awaitExportJob(t,s,job.ID,"ready") + reader,release,err:=s.acquire("owner",job.ID);if err!=nil {t.Fatal(err)};defer release() + s.expire(time.Now().Add(2*time.Hour)) + if _,_,err:=s.acquire("owner",job.ID);!errors.Is(err,errExportJobMissing) {t.Fatal("expired output newly downloadable")} + if _,err:=s.submit("owner",render);!errors.Is(err,errExportJobsBusy) {t.Fatal("active transfer reservation was released early")} + data,err:=io.ReadAll(reader);if err!=nil || string(data)!="video" {t.Fatalf("expiry interrupted active download: %q %v",data,err)} + release();release() + next,err:=s.submit("owner",render);if err!=nil {t.Fatal(err)};awaitExportJob(t,s,next.ID,"ready") +} + +func TestExportJobsCancelQueueAndShutdown(t *testing.T) { + s:=testExportJobs(t,8,24) + started:=make(chan struct{});stopped:=make(chan struct{}) + first,err:=s.submit("owner",func(ctx context.Context,w io.Writer)error{close(started);<-ctx.Done();close(stopped);return ctx.Err()});if err!=nil {t.Fatal(err)} + select {case <-started:case <-time.After(3*time.Second):t.Fatal("renderer never started")} + called:=make(chan struct{},1) + second,err:=s.submit("owner",func(context.Context,io.Writer)error{called<-struct{}{};return nil});if err!=nil {t.Fatal(err)} + if err:=s.cancelJob("owner",second.ID);err!=nil {t.Fatal(err)} + if err:=s.cancelJob("owner",first.ID);err!=nil {t.Fatal(err)} + ctx,cancel:=context.WithTimeout(context.Background(),3*time.Second);defer cancel() + if err:=s.Close(ctx);err!=nil {t.Fatal(err)} + select {case <-stopped:default:t.Fatal("shutdown returned before renderer joined")} + select {case <-called:t.Fatal("canceled queued renderer ran");default:} + if _,err:=os.Stat(s.directory);!os.IsNotExist(err) {t.Fatalf("scratch directory not removed: %v",err)} + if _,err:=s.submit("owner",func(context.Context,io.Writer)error{return nil});!errors.Is(err,errExportJobsClosed) {t.Fatal("shutdown admitted new export")} +} + +func TestExportJobsShutdownWaitsForDownloadLease(t *testing.T) { + s:=testExportJobs(t,8,8) + job,err:=s.submit("owner",func(ctx context.Context,w io.Writer)error{_,err:=io.WriteString(w,"video");return err});if err!=nil {t.Fatal(err)};awaitExportJob(t,s,job.ID,"ready") + reader,release,err:=s.acquire("owner",job.ID);if err!=nil {t.Fatal(err)};defer release() + short,cancel:=context.WithTimeout(context.Background(),10*time.Millisecond);defer cancel() + if err:=s.Close(short);!errors.Is(err,context.DeadlineExceeded) {t.Fatalf("did not wait for reader: %v",err)} + data,err:=io.ReadAll(reader);if err!=nil || string(data)!="video" {t.Fatal("shutdown closed active reader")} + release() + ctx,stop:=context.WithTimeout(context.Background(),3*time.Second);defer stop() + if err:=s.Close(ctx);err!=nil {t.Fatal(err)} +} From e8648b405cbae4eab08aa242094a967d0560e687 Mon Sep 17 00:00:00 2001 From: mouldybread <38598858+mouldybread@users.noreply.github.com> Date: Sat, 21 Nov 2026 10:58:36 +0800 Subject: [PATCH 2/8] Integrate authenticated prepared-export jobs and downloads --- docs/LARGE_MEDIA_EXPORTS.md | 51 +++++---- export_http.go | 217 ++++++++++++++++++++++++++++++++++++ export_http_test.go | 102 +++++++++++++++++ main.go | 10 +- 4 files changed, 356 insertions(+), 24 deletions(-) create mode 100644 export_http.go create mode 100644 export_http_test.go diff --git a/docs/LARGE_MEDIA_EXPORTS.md b/docs/LARGE_MEDIA_EXPORTS.md index 9ea57de..9c9c0aa 100644 --- a/docs/LARGE_MEDIA_EXPORTS.md +++ b/docs/LARGE_MEDIA_EXPORTS.md @@ -1,39 +1,46 @@ # Large-media export implementation status -This branch is work in progress. The first commit adds an isolated server-side export job store and tests only. It does NOT yet register HTTP routes, change the UI, remove the browser Blob limit, or broaden rendering compatibility. Do not release this commit as the completed export redesign. +This branch remains work in progress, not a completed or release-ready export redesign. -## Implemented foundation +## Implemented -- Bounded jobs and preparation concurrency, random job IDs, and explicit owner checks. -- Private disk-backed output, immediately unlinked and retained by file descriptor; no recording cleanup or source modification. -- Independent seekable download leases for eventual HTTP Range delivery, without page-level whole-file buffering. -- Cancellation, expiry, disk-reservation accounting, bounded writes, and shutdown joining. Existing transfers retain their leases until released. -- Safe public status messages; raw renderer errors are not exposed. +- Bounded disk-backed prepared-output job store, owner checks, independent seekable readers, cancellation, expiry and shutdown joining. +- Authenticated POST /api/exports/create and /api/exports/cancel. +- Authenticated GET/HEAD /api/exports/status?id=ID and /api/exports/download?id=ID. Downloads use ServeContent and independent SectionReaders for HTTP Range requests, not browser Blobs. +- Owners are derived from session identity, session token and credential version. Another login cannot inherit jobs. Event tokens cannot access these endpoints. Existing cross-origin protection applies to mutations. +- Startup and shutdown lifecycle integration. Active requests are canceled and drained, then output leases and background preparations are joined. +- The current renderer is reused with pinned-input revalidation, safe generic job failures and prepared-output duration probing. Its input restrictions remain unchanged. -The renderer callback must validate the export and join its subprocesses before returning. The future HTTP integration must bind owners to authenticated sessions/credential versions, never caller-supplied owner strings. It must cancel and drain downloads before waiting for store shutdown. +## Opt-in configuration -## Planned startup configuration - -These settings are parsed by the new helper but are NOT wired into startup yet: +Set NVR_EXPORT_SCRATCH_DIR to an existing absolute directory on a dedicated trusted scratch mount, e.g. /exports. Mount adequate disk space there; do not use a tmpfs or the container writable layer for large exports. The directory must be outside /recordings and must not overlap camera directories or resolve through symlinks. The service makes a private temporary subdirectory. With the variable unset, only the new endpoints are disabled (503); existing downloads continue unchanged. Invalid configured limits or unavailable configured storage prevent startup. | Setting | Default | Meaning | | --- | --- | --- | | NVR_EXPORT_MAX_BYTES | 8589934592 | Maximum prepared output per job (8 GiB) | -| NVR_EXPORT_TOTAL_BYTES | 17179869184 | Aggregate reserved output capacity (16 GiB) | +| NVR_EXPORT_TOTAL_BYTES | 17179869184 | Aggregate job-output reservation (16 GiB) | | NVR_EXPORT_MAX_JOBS | 8 | Maximum retained job records | | NVR_EXPORT_PARALLEL | 1 | Concurrent preparations | | NVR_EXPORT_TTL | 30m | Queue/processing expiry; retention resets on completion | -| NVR_EXPORT_PREPARE_TIMEOUT | 30m | Deadline including queue admission wait | +| NVR_EXPORT_PREPARE_TIMEOUT | 30m | Preparation deadline including queue wait | + +Each job reserves its maximum output allowance, so defaults admit at most two reservations. Active expired downloads remain charged until release. Intermediate files used by the legacy trim renderer retain their existing separate bounds; this reservation is not a physical disk reservation or a bound on every filesystem allocation. Monitor free space. No originals are modified by the export path. Jobs are in memory and do not survive restart. + +## API + +POST create with application/json: + +```json +{"camera_id":"CAMERA_ID","start_time":"2026-11-21T00:00:00Z","end_time":"2026-11-21T00:01:00Z"} +``` -Each admitted job reserves its maximum output allowance; default aggregate capacity therefore admits at most two such reservations at once. Expired files held by active downloads remain charged. Invalid limits fail closed. Filesystem exhaustion can still fail a job; reservations do not reserve physical blocks or replace filesystem capacity monitoring. Use trusted private scratch storage with sufficient free space. +A 202 response contains id, state, message, bytes and expires_at, with a Location status URL. Poll status; only ready jobs expose download_url. Download that same-origin URL with the same authenticated session. POST cancel with {"id":"JOB_ID"}; success is 204. Cancellation and expiry prevent new downloads; already acquired download leases are released when their requests finish. Logout/credential changes block subsequent requests but do not promise immediate revocation of an existing response. -## Remaining integration +## Remaining -1. Authenticated POST creation/cancellation and GET status/download endpoints; no state-changing GET requests or bearer-token bypass. -2. Startup, shutdown-gate and HTTP transfer cancellation wiring. -3. Original-file delivery, compatible audio/video stream-copy assembly, and explicit preview/consent for keyframe-adjusted boundaries. -4. Optional accurate trimming preserving audio timing and handling full-range input correctly; no silent audio removal or conversion fallback. -5. Browser job UI without whole-file Blob buffering; update real-browser acceptance tests. -6. Media, authentication, quota, expiry, cancellation, shutdown, large-transfer and source-preservation integration tests. +- Original-file delivery, compatible audio/video stream-copy assembly and explicit consent for keyframe-adjusted ranges. +- Optional accurate audio-preserving trimming, full-range input handling and representative media validation. +- Browser job UI replacing the 64 MiB Blob workflow and corresponding real-browser acceptance updates. +- Full HTTP/media/large-transfer/slow-client/shutdown and runtime acceptance validation. -The Go tests accompanying the first commit have not been executed in the assistant environment; Go is unavailable there. +Tests added for authentication, token scope, CSRF, strict request validation, session isolation, logout/credential changes, Range/HEAD downloads, cancellation and shutdown. Go tests have not been run in the assistant environment. diff --git a/export_http.go b/export_http.go new file mode 100644 index 0000000..75735f8 --- /dev/null +++ b/export_http.go @@ -0,0 +1,217 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "mime" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +const ( + exportCreatePath="/api/exports/create" + exportCancelPath="/api/exports/cancel" + exportStatusPath="/api/exports/status" + exportDownloadPath="/api/exports/download" +) + +// Extend the existing authentication and shutdown mutation policy. The route +// handler is installed INSIDE auth.Handler, so event tokens never reach it. +func init() {mutationPaths[exportCreatePath]=true;mutationPaths[exportCancelPath]=true} + +type exportCreateRequest struct { + CameraID string `json:"camera_id"` + Start string `json:"start_time"` + End string `json:"end_time"` +} +type exportHTTPView struct { + exportJobView + DownloadURL string `json:"download_url,omitempty"` +} +type exportPlanHTTPError struct {status int;message string} +func (e *exportPlanHTTPError) Error()string{return e.message} +type exportHTTPPlanner func(context.Context,exportCreateRequest)(exportJobRenderer,error) + +type exportJobHTTP struct { + auth *authManager + store *exportJobStore + plan exportHTTPPlanner + ctx context.Context + cancel context.CancelFunc + mu sync.Mutex + closing bool + requests sync.WaitGroup + once sync.Once + done chan struct{} + closeErr error +} + +func newExportJobHTTP(auth *authManager,store *exportJobStore,plan exportHTTPPlanner)*exportJobHTTP { + ctx,cancel:=context.WithCancel(context.Background()) + return &exportJobHTTP{auth:auth,store:store,plan:plan,ctx:ctx,cancel:cancel,done:make(chan struct{})} +} + +// Opt-in scratch storage avoids silently putting GiB exports in the container +// writable layer. The operator must mount trusted private storage at this path. +func configuredExportJobHTTP(auth *authManager,n *NVR)(*exportJobHTTP,error) { + limits,err:=loadExportJobLimits(os.Getenv);if err!=nil{return nil,err} + directory:=os.Getenv("NVR_EXPORT_SCRATCH_DIR") + if directory=="" {return newExportJobHTTP(auth,nil,nil),nil} + if !filepath.IsAbs(directory){return nil,errors.New("NVR_EXPORT_SCRATCH_DIR must be absolute")} + directory=filepath.Clean(directory) + resolved,err:=filepath.EvalSymlinks(directory);if err!=nil{return nil,errors.New("export scratch directory unavailable")} + if resolved!=directory || directory=="/recordings" || strings.HasPrefix(directory,"/recordings/"){return nil,errors.New("export scratch must be outside recordings and not symlinked")} + info,err:=os.Stat(directory);if err!=nil || !info.IsDir(){return nil,errors.New("export scratch must be an existing directory")} + n.mu.RLock() + overlap:=false + for _,cam:=range n.cameras {if cam==nil{continue};path,err:=filepath.Abs(cam.RecordDir);if err!=nil{overlap=true;break};path=filepath.Clean(path);if path==directory || strings.HasPrefix(directory,path+string(os.PathSeparator)) || strings.HasPrefix(path,directory+string(os.PathSeparator)){overlap=true;break}} + n.mu.RUnlock() + if overlap{return nil,errors.New("export scratch overlaps a camera directory")} + store,err:=newExportJobStore(context.Background(),directory,limits);if err!=nil{return nil,err} + service:=newExportJobHTTP(auth,store,nil) + service.plan=func(ctx context.Context,req exportCreateRequest)(exportJobRenderer,error){return planExistingExportJob(ctx,n,store.directory,req)} + return service,nil +} + +func exportJobRoute(path string)bool {return path==exportCreatePath || path==exportCancelPath || path==exportStatusPath || path==exportDownloadPath} + +// The identity comes only from a validated session, not a header, query or +// JSON owner. A new session or changed credentials cannot inherit old jobs. +func (s *exportJobHTTP) owner(r *http.Request)(string,error) { + if s.auth==nil{return "",errExportJobMissing} + cfg,stamp,err:=readAuthConfig(s.auth.path);if err!=nil{return "",err} + user:=s.auth.sessions.GetString(r.Context(),"user") + version:=s.auth.sessions.GetString(r.Context(),"credential_version") + token:=s.auth.sessions.Token(r.Context()) + if user!=cfg.Username || version!=stamp || token==""{return "",errExportJobMissing} + digest:=sha256.Sum256([]byte(user+"\x00"+version+"\x00"+token)) + return hex.EncodeToString(digest[:]),nil +} + +func (s *exportJobHTTP) Handler(next http.Handler)http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter,r *http.Request){ + if !exportJobRoute(r.URL.Path){next.ServeHTTP(w,r);return} + w.Header().Set("Cache-Control","private, no-store");w.Header().Set("X-Content-Type-Options","nosniff");w.Header().Set("Referrer-Policy","no-referrer") + owner,err:=s.owner(r);if err!=nil{http.Error(w,"Unauthorized",401);return} + allowed:="GET, HEAD" + if r.URL.Path==exportCreatePath || r.URL.Path==exportCancelPath {allowed="POST"} + if (allowed=="POST" && r.Method!=http.MethodPost) || (allowed!="POST" && r.Method!=http.MethodGet && r.Method!=http.MethodHead){w.Header().Set("Allow",allowed);http.Error(w,"Method not allowed",405);return} + s.mu.Lock() + if s.closing || s.store==nil {s.mu.Unlock();http.Error(w,"Prepared exports unavailable; configure export scratch storage or retry after restart",503);return} + s.requests.Add(1);s.mu.Unlock();defer s.requests.Done() + ctx,cancel:=context.WithCancel(r.Context());defer cancel() + stopService:=context.AfterFunc(s.ctx,cancel);defer stopService() + r=r.WithContext(ctx) + // This also releases readers blocked behind a slow/disconnected client. + writeDone:=make(chan struct{}) + stopWrite:=context.AfterFunc(ctx,func(){_ =http.NewResponseController(w).SetWriteDeadline(time.Now());close(writeDone)}) + defer func(){if !stopWrite(){<-writeDone}}() + switch r.URL.Path { + case exportCreatePath:s.create(w,r,owner) + case exportCancelPath:s.cancelRequest(w,r,owner) + case exportStatusPath: + id,ok:=exportRequestID(w,r);if !ok{return} + view,err:=s.store.status(owner,id);if err!=nil{writeExportJobHTTPError(w,err);return} + writeExportJobView(w,r,200,view) + case exportDownloadPath: + id,ok:=exportRequestID(w,r);if !ok{return} + reader,release,err:=s.store.acquire(owner,id);if err!=nil{writeExportJobHTTPError(w,err);return};defer release() + w.Header().Set("Content-Type","video/mp4");w.Header().Set("Content-Disposition",`attachment; filename="export-`+id+`.mp4"`) + http.ServeContent(w,r,"export-"+id+".mp4",time.Time{},reader) + } + }) +} + +func exportRequestID(w http.ResponseWriter,r *http.Request)(string,bool) { + query:=r.URL.Query();values:=query["id"] + if len(query)!=1 || len(values)!=1 || !validExportJobID(values[0]){http.Error(w,"One valid export id is required",400);return "",false} + return values[0],true +} +func validExportJobID(id string)bool {if len(id)!=64{return false};raw,err:=hex.DecodeString(id);return err==nil && len(raw)==32} +func decodeExportJobJSON(w http.ResponseWriter,r *http.Request,value interface{})bool { + if r.URL.RawQuery!=""{http.Error(w,"Query parameters not accepted",400);return false} + kind,_,err:=mime.ParseMediaType(r.Header.Get("Content-Type"));if err!=nil || kind!="application/json"{http.Error(w,"Content-Type must be application/json",415);return false} + decoder:=json.NewDecoder(http.MaxBytesReader(w,r.Body,4096));decoder.DisallowUnknownFields() + if err:=decoder.Decode(value);err!=nil{http.Error(w,"Invalid export request",400);return false} + if err:=decoder.Decode(&struct{}{});err!=io.EOF{http.Error(w,"One export JSON object is required",400);return false} + return true +} +func writeExportJobView(w http.ResponseWriter,r *http.Request,status int,view exportJobView) { + body:=exportHTTPView{exportJobView:view} + if view.State=="ready"{body.DownloadURL=exportDownloadPath+"?id="+view.ID} + w.Header().Set("Content-Type","application/json");w.WriteHeader(status) + if r.Method!=http.MethodHead{json.NewEncoder(w).Encode(body)} +} +func writeExportJobHTTPError(w http.ResponseWriter,err error) { + status,message:=404,"Export unavailable, expired, or not ready" + if errors.Is(err,errExportJobsBusy){status,message=429,"Export capacity exhausted; release prepared exports or wait for expiry";w.Header().Set("Retry-After","5")} + if errors.Is(err,errExportJobsClosed){status,message=503,"Export service stopping"} + http.Error(w,message,status) +} +func (s *exportJobHTTP) create(w http.ResponseWriter,r *http.Request,owner string) { + var req exportCreateRequest;if !decodeExportJobJSON(w,r,&req){return} + start,e1:=time.Parse(time.RFC3339Nano,req.Start);end,e2:=time.Parse(time.RFC3339Nano,req.End) + if req.CameraID=="" || len(req.CameraID)>256 || e1!=nil || e2!=nil || !end.After(start){http.Error(w,"Valid camera_id and ordered RFC3339 start_time/end_time required",400);return} + if s.plan==nil{http.Error(w,"Export renderer unavailable",503);return} + render,err:=s.plan(r.Context(),req) + if err!=nil{ + var requestErr *exportPlanHTTPError + if errors.As(err,&requestErr){http.Error(w,requestErr.message,requestErr.status)}else if errors.Is(err,errExportPlanGap) || errors.Is(err,errExportPlanOverlap) || errors.Is(err,errExportInputLimit) || errors.Is(err,errExportPlanInvalid){writeExportPlanError(w,err)}else{writeRecordingIntervalError(w,r,err)} + return + } + if r.Context().Err()!=nil{return} + view,err:=s.store.submit(owner,render);if err!=nil{writeExportJobHTTPError(w,err);return} + w.Header().Set("Location",exportStatusPath+"?id="+view.ID);writeExportJobView(w,r,202,view) +} +func (s *exportJobHTTP) cancelRequest(w http.ResponseWriter,r *http.Request,owner string) { + var req struct{ID string `json:"id"`};if !decodeExportJobJSON(w,r,&req){return} + if !validExportJobID(req.ID){http.Error(w,"Valid export id required",400);return} + if err:=s.store.cancelJob(owner,req.ID);err!=nil{writeExportJobHTTPError(w,err);return} + w.WriteHeader(http.StatusNoContent) +} + +// This integration deliberately retains the existing render restrictions. +// Stream-copy compatibility and audio-preserving accurate trim are next steps. +func planExistingExportJob(parent context.Context,n *NVR,scratch string,req exportCreateRequest)(exportJobRenderer,error) { + start,_:=time.Parse(time.RFC3339Nano,req.Start);end,_:=time.Parse(time.RFC3339Nano,req.End) + n.mu.RLock();cam:=n.cameras[req.CameraID];var camera timelineCamera;if cam!=nil{camera=timelineCamera{cam.ID,cam.Name,cam.RecordDir}};n.mu.RUnlock() + if cam==nil{return nil,&exportPlanHTTPError{404,"Camera not found"}} + ctx,cancel:=context.WithTimeout(parent,recordingMetadataBudget);defer cancel() + window:=timelineWindow{Start:start.UTC(),End:end.UTC()} + clips,err:=collectRecordingIntervals(ctx,[]timelineCamera{camera},window,recordingDurations);if err!=nil{return nil,err} + plan,err:=buildExportPlan(clips,window);if err!=nil{return nil,err} + return func(ctx context.Context,w io.Writer)error { + pins,err:=exportInputPins.prepare(ctx,plan.clips());if err!=nil{return err};defer pins.Close() + if plan.NeedsTrim { + output,err:=renderTrimmedExport(ctx,pins,plan,scratch,maxTrimBytes);if err!=nil{return err};defer output.Close() + if _,err=io.Copy(w,output.file);err!=nil{return err} + }else{ + cmd,err:=pins.command(ctx);if err!=nil{return err};cmd.Stdout=w;cmd.Stderr=io.Discard + if err=cmd.Run();err!=nil{return err} + } + if err:=pins.validate(ctx);err!=nil{return err} + output,ok:=w.(*boundedExportJobWriter);if !ok{return errors.New("export output descriptor unavailable")} + verify,stop:=context.WithTimeout(ctx,5*time.Second);defer stop() + duration,err:=probeVideoDuration(verify,output.file);if err!=nil{return err} + difference:=duration-plan.End.Sub(plan.Start);if difference<0{difference=-difference} + if duration<=0 || difference>250*time.Millisecond{return errors.New("prepared export duration validation failed")} + return ctx.Err() + },nil +} + +func (s *exportJobHTTP) Close(ctx context.Context)error { + s.once.Do(func(){ + s.mu.Lock();s.closing=true;s.cancel();if s.store!=nil{s.store.cancel()};s.mu.Unlock() + go func(){s.requests.Wait();if s.store!=nil{s.closeErr=s.store.Close(context.Background())};close(s.done)}() + }) + select{case <-s.done:return s.closeErr;case <-ctx.Done():return ctx.Err()} +} +func (s *exportJobHTTP) runLifecycle(ctx context.Context){<-ctx.Done();_ =s.Close(context.Background())} diff --git a/export_http_test.go b/export_http_test.go new file mode 100644 index 0000000..5a52480 --- /dev/null +++ b/export_http_test.go @@ -0,0 +1,102 @@ +package main + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "strings" + "testing" + "time" +) + +func exportHTTPLogin(t *testing.T,a *authManager,h http.Handler)*http.Cookie { + t.Helper();a.mu.Lock();a.nextLogin=time.Time{};a.mu.Unlock() + form:=url.Values{"username":{"admin"},"password":{"correct horse battery staple"}} + r:=httptest.NewRequest("POST","https://nvr.test/auth/login",strings.NewReader(form.Encode()));r.Header.Set("Content-Type","application/x-www-form-urlencoded") + w:=httptest.NewRecorder();h.ServeHTTP(w,r) + if w.Code!=303 || len(w.Result().Cookies())==0 {t.Fatalf("login %d %s",w.Code,w.Body.String())} + return w.Result().Cookies()[0] +} +func exportHTTPRequest(h http.Handler,cookie *http.Cookie,method,path,body string)*httptest.ResponseRecorder { + r:=httptest.NewRequest(method,"https://nvr.test"+path,strings.NewReader(body));if cookie!=nil{r.AddCookie(cookie)} + if method=="POST"{r.Header.Set("Content-Type","application/json")} + w:=httptest.NewRecorder();h.ServeHTTP(w,r);return w +} +func exportHTTPFixture(t *testing.T)(*authManager,*exportJobHTTP,http.Handler,*http.Cookie) { + t.Helper();a,_:=authFixture(t);store:=testExportJobs(t,1024,4096) + service:=newExportJobHTTP(a,store,func(context.Context,exportCreateRequest)(exportJobRenderer,error){return func(ctx context.Context,w io.Writer)error{_,err:=io.WriteString(w,"0123456789");return err},nil}) + t.Cleanup(func(){ctx,cancel:=context.WithTimeout(context.Background(),3*time.Second);defer cancel();if err:=service.Close(ctx);err!=nil{t.Error(err)}}) + h:=a.Handler(service.Handler(http.NotFoundHandler()));return a,service,h,exportHTTPLogin(t,a,h) +} +const exportHTTPCreateBody=`{"camera_id":"one","start_time":"2026-11-21T00:00:00Z","end_time":"2026-11-21T00:00:10Z"}` +func createHTTPExport(t *testing.T,h http.Handler,cookie *http.Cookie)exportHTTPView { + t.Helper();w:=exportHTTPRequest(h,cookie,"POST",exportCreatePath,exportHTTPCreateBody) + if w.Code!=202{t.Fatalf("create %d %s",w.Code,w.Body.String())} + var view exportHTTPView;if err:=json.Unmarshal(w.Body.Bytes(),&view);err!=nil{t.Fatal(err)} + deadline:=time.Now().Add(3*time.Second) + for time.Now().Before(deadline){ + w=exportHTTPRequest(h,cookie,"GET",exportStatusPath+"?id="+view.ID,"");if w.Code!=200{t.Fatal(w.Code,w.Body.String())} + if err:=json.Unmarshal(w.Body.Bytes(),&view);err!=nil{t.Fatal(err)} + if view.State=="ready"{return view};time.Sleep(time.Millisecond) + } + t.Fatal("job did not become ready");return view +} + +func TestExportHTTPAuthenticationMethodsAndCSRF(t *testing.T) { + _,_,h,cookie:=exportHTTPFixture(t) + for _,tc:=range []struct{method,path string}{ + {"POST",exportCreatePath},{"POST",exportCancelPath},{"GET",exportStatusPath},{"GET",exportDownloadPath},{"HEAD",exportDownloadPath}, + }{ + if got:=exportHTTPRequest(h,nil,tc.method,tc.path,"{}").Code;got!=401{t.Fatalf("anonymous %s: %d",tc.path,got)} + r:=httptest.NewRequest(tc.method,"https://nvr.test"+tc.path,strings.NewReader("{}"));r.Header.Set("Authorization","Bearer 0123456789012345678901234567890123456789012") + w:=httptest.NewRecorder();h.ServeHTTP(w,r);if w.Code!=401{t.Fatalf("event token accepted: %s %d",tc.path,w.Code)} + } + for _,tc:=range []struct{method,path string}{{"GET",exportCreatePath},{"GET",exportCancelPath},{"POST",exportStatusPath},{"POST",exportDownloadPath}}{if got:=exportHTTPRequest(h,cookie,tc.method,tc.path,"{}").Code;got!=405{t.Fatalf("wrong method %s: %d",tc.path,got)}} + r:=httptest.NewRequest("POST","https://nvr.test"+exportCreatePath,strings.NewReader(exportHTTPCreateBody));r.AddCookie(cookie);r.Header.Set("Content-Type","application/json");r.Header.Set("Origin","https://evil.invalid") + w:=httptest.NewRecorder();h.ServeHTTP(w,r);if w.Code!=403{t.Fatal("cross-origin creation accepted",w.Code)} + for _,body:=range []string{"{}",exportHTTPCreateBody+" {}",strings.Replace(exportHTTPCreateBody,`"camera_id":"one"`,`"camera_id":"one","owner":"forged"`,1),strings.Replace(exportHTTPCreateBody,"00:00:10Z","00:00:00Z",1)}{if got:=exportHTTPRequest(h,cookie,"POST",exportCreatePath,body).Code;got!=400{t.Fatalf("accepted invalid JSON/range: %d",got)}} +} + +func TestExportHTTPDownloadRangeAndSessionIsolation(t *testing.T) { + a,_,h,cookie:=exportHTTPFixture(t);view:=createHTTPExport(t,h,cookie) + if view.DownloadURL!=exportDownloadPath+"?id="+view.ID{t.Fatal(view)} + r:=httptest.NewRequest("GET","https://nvr.test"+view.DownloadURL,nil);r.AddCookie(cookie);r.Header.Set("Range","bytes=2-5") + w:=httptest.NewRecorder();h.ServeHTTP(w,r) + if w.Code!=206 || w.Body.String()!="2345" || w.Header().Get("Content-Range")!="bytes 2-5/10" || w.Header().Get("Content-Length")!="4"{t.Fatalf("range %d %v %q",w.Code,w.Header(),w.Body.String())} + if w.Header().Get("Cache-Control")!="private, no-store" || !strings.HasPrefix(w.Header().Get("Content-Disposition"),"attachment;"){t.Fatal("unsafe download headers")} + head:=exportHTTPRequest(h,cookie,"HEAD",view.DownloadURL,"");if head.Code!=200 || head.Body.Len()!=0 || head.Header().Get("Content-Length")!="10"{t.Fatal("HEAD body or length wrong")} + second:=exportHTTPLogin(t,a,h) + for _,path:=range []string{exportStatusPath+"?id="+view.ID,view.DownloadURL}{if got:=exportHTTPRequest(h,second,"GET",path,"").Code;got!=404{t.Fatalf("second session inherited export: %d",got)}} + if got:=exportHTTPRequest(h,second,"POST",exportCancelPath,`{"id":"`+view.ID+`"}`).Code;got!=404{t.Fatal("cross-session cancel accepted")} + if got:=exportHTTPRequest(h,cookie,"POST",exportCancelPath,`{"id":"`+view.ID+`"}`).Code;got!=204{t.Fatal(got)} + if got:=exportHTTPRequest(h,cookie,"GET",view.DownloadURL,"").Code;got!=404{t.Fatal("canceled download available")} +} + +func TestExportHTTPCredentialChangeAndLogout(t *testing.T) { + a,_,h,cookie:=exportHTTPFixture(t);view:=createHTTPExport(t,h,cookie) + if got:=exportHTTPRequest(h,cookie,"POST","/auth/logout","").Code;got!=303{t.Fatal(got)} + if got:=exportHTTPRequest(h,cookie,"GET",view.DownloadURL,"").Code;got!=401{t.Fatal("logged-out download allowed")} + cookie=exportHTTPLogin(t,a,h);view=createHTTPExport(t,h,cookie) + cfg,_,err:=readAuthConfig(a.path);if err!=nil{t.Fatal(err)};cfg.EventTokenHash="";if err:=writeAuthConfig(a.path,cfg);err!=nil{t.Fatal(err)} + if got:=exportHTTPRequest(h,cookie,"GET",view.DownloadURL,"").Code;got!=401{t.Fatal("stale credential session accepted")} + cookie=exportHTTPLogin(t,a,h) + if got:=exportHTTPRequest(h,cookie,"GET",view.DownloadURL,"").Code;got!=404{t.Fatal("new credentials inherited job")} +} + +func TestExportHTTPShutdownAndConfiguration(t *testing.T) { + _,service,h,cookie:=exportHTTPFixture(t) + started:=make(chan struct{});joined:=make(chan struct{}) + service.plan=func(context.Context,exportCreateRequest)(exportJobRenderer,error){return func(ctx context.Context,w io.Writer)error{close(started);<-ctx.Done();close(joined);return ctx.Err()},nil} + if got:=exportHTTPRequest(h,cookie,"POST",exportCreatePath,exportHTTPCreateBody).Code;got!=202{t.Fatal(got)} + select{case <-started:case <-time.After(3*time.Second):t.Fatal("preparation not started")} + ctx,cancel:=context.WithTimeout(context.Background(),3*time.Second);defer cancel();if err:=service.Close(ctx);err!=nil{t.Fatal(err)} + select{case <-joined:default:t.Fatal("shutdown did not join preparation")} + if got:=exportHTTPRequest(h,cookie,"POST",exportCreatePath,exportHTTPCreateBody).Code;got!=503{t.Fatal("shutdown admitted creation")} + if _,err:=os.Stat(service.store.directory);!os.IsNotExist(err){t.Fatal("scratch storage not released")} + t.Setenv("NVR_EXPORT_SCRATCH_DIR","/recordings/.exports") + if _,err:=configuredExportJobHTTP(service.auth,&NVR{});err==nil{t.Fatal("recording directory accepted as scratch")} +} diff --git a/main.go b/main.go index 237d72c..c2353b0 100644 --- a/main.go +++ b/main.go @@ -8,6 +8,7 @@ import ( "crypto/x509/pkix" "encoding/hex" "encoding/pem" + "errors" "html/template" "log" "math/big" @@ -111,6 +112,8 @@ func main() { certPath,keyPath:=filepath.Join("/certs","cert.pem"),filepath.Join("/certs","key.pem") if err=ensureTLSCertificates(certPath,keyPath);err!=nil {log.Fatal("Failed to initialize TLS certificates: ",err)} if appCtx.Err()!=nil {return} + exports,err:=configuredExportJobHTTP(auth,nvr);if err!=nil {log.Fatal("Failed to initialize prepared exports: ",err)} + if exports.store==nil {log.Println("Prepared export jobs disabled: NVR_EXPORT_SCRATCH_DIR not configured")} http.HandleFunc("/api/cameras",camerasHandler) http.HandleFunc("/api/cameras/add",addCameraHandler) http.HandleFunc("/api/cameras/delete",deleteCameraHandler) @@ -137,11 +140,14 @@ func main() { http.HandleFunc("/api/recordings/cleanup-orphaned",cleanupOrphanedRecordingsHandler) http.Handle("/static/",http.StripPrefix("/static/",http.FileServer(http.Dir("static")))) http.HandleFunc("/",indexHandler) - gate:=newShutdownGate(themedLoginPage(auth.Handler(http.DefaultServeMux))) + gate:=newShutdownGate(themedLoginPage(auth.Handler(exports.Handler(http.DefaultServeMux)))) + gate.startBackgroundJob(exports.runLifecycle) server:=&http.Server{Addr:":443",Handler:gate,ReadHeaderTimeout:10*time.Second,ReadTimeout:30*time.Second,IdleTimeout:60*time.Second,MaxHeaderBytes:32768} autoStartRecordings() gate.startBackgroundJob(func(ctx context.Context){cleanupRoutine(ctx,nvr)}) log.Println("SimpleNVR starting authenticated HTTPS server on :443...") - if err=runAuthenticatedServer(appCtx,nvr,gate,server,certPath,keyPath);err!=nil {log.Fatal("Shutdown/server failure: ",err)} + serveErr:=runAuthenticatedServer(appCtx,nvr,gate,server,certPath,keyPath) + cleanupCtx,cancel:=context.WithTimeout(context.Background(),5*time.Second);defer cancel() + if err=errors.Join(serveErr,exports.Close(cleanupCtx));err!=nil {log.Fatal("Shutdown/server failure: ",err)} log.Println("Shutdown complete; recording intent persisted") } From 9fa561994bb7a372c4b2f3b9de3a59852b066c2e Mon Sep 17 00:00:00 2001 From: mouldybread <38598858+mouldybread@users.noreply.github.com> Date: Sat, 21 Nov 2026 11:30:19 +0800 Subject: [PATCH 3/8] Replace browser video buffering with prepared export jobs --- docs/LARGE_MEDIA_EXPORTS.md | 50 +++---- static/js/export-ui.js | 258 ++++++++++++++++---------------- tests/export-range-controls.cjs | 85 +++-------- tests/export-ui-fixture.cjs | 30 ++++ tests/export-ui.cjs | 93 +++++------- 5 files changed, 234 insertions(+), 282 deletions(-) create mode 100644 tests/export-ui-fixture.cjs diff --git a/docs/LARGE_MEDIA_EXPORTS.md b/docs/LARGE_MEDIA_EXPORTS.md index 9c9c0aa..839373d 100644 --- a/docs/LARGE_MEDIA_EXPORTS.md +++ b/docs/LARGE_MEDIA_EXPORTS.md @@ -1,46 +1,38 @@ -# Large-media export implementation status +# Large-media exports: work in progress -This branch remains work in progress, not a completed or release-ready export redesign. +Do not release this branch as the completed export redesign. ## Implemented -- Bounded disk-backed prepared-output job store, owner checks, independent seekable readers, cancellation, expiry and shutdown joining. -- Authenticated POST /api/exports/create and /api/exports/cancel. -- Authenticated GET/HEAD /api/exports/status?id=ID and /api/exports/download?id=ID. Downloads use ServeContent and independent SectionReaders for HTTP Range requests, not browser Blobs. -- Owners are derived from session identity, session token and credential version. Another login cannot inherit jobs. Event tokens cannot access these endpoints. Existing cross-origin protection applies to mutations. -- Startup and shutdown lifecycle integration. Active requests are canceled and drained, then output leases and background preparations are joined. -- The current renderer is reused with pinned-input revalidation, safe generic job failures and prepared-output duration probing. Its input restrictions remain unchanged. +- Bounded disk-backed job store, random IDs, owner checks, independent seekable readers, cancellation, expiry and shutdown joining. +- Authenticated POST /api/exports/create and /api/exports/cancel, plus GET/HEAD /api/exports/status?id=ID and /api/exports/download?id=ID. Event tokens cannot access these routes; jobs are bound to session and credential version. +- Startup and shutdown integration. Download responses use ServeContent with Range support. +- Browser UI creates jobs, polls state, and offers a normal same-origin download. Page JavaScript does not fetch/buffer video or impose the old 64 MiB cap. Control responses are bounded to 8 KiB. +- Range controls are retained. Cameras are processed sequentially; the next job starts after release or expiry of the prepared file. +- Cancel/release requires server acknowledgement; failures remain visible with a retry control. Lost creation responses are not retried automatically. Navigation leaves jobs to server expiry rather than canceling a download during handoff. -## Opt-in configuration +## Configuration -Set NVR_EXPORT_SCRATCH_DIR to an existing absolute directory on a dedicated trusted scratch mount, e.g. /exports. Mount adequate disk space there; do not use a tmpfs or the container writable layer for large exports. The directory must be outside /recordings and must not overlap camera directories or resolve through symlinks. The service makes a private temporary subdirectory. With the variable unset, only the new endpoints are disabled (503); existing downloads continue unchanged. Invalid configured limits or unavailable configured storage prevent startup. +Mount trusted disk-backed scratch storage outside /recordings and camera directories (for example /exports) and set NVR_EXPORT_SCRATCH_DIR=/exports. It must exist and must not resolve through symlinks. Without it the new endpoints return 503; the job UI reports that configuration is required and does not fall back to the old Blob path. Invalid configured limits/storage fail startup. Do not put GiB exports in a tmpfs or container writable layer. | Setting | Default | Meaning | | --- | --- | --- | -| NVR_EXPORT_MAX_BYTES | 8589934592 | Maximum prepared output per job (8 GiB) | +| NVR_EXPORT_MAX_BYTES | 8589934592 | Per-job prepared output (8 GiB) | | NVR_EXPORT_TOTAL_BYTES | 17179869184 | Aggregate job-output reservation (16 GiB) | -| NVR_EXPORT_MAX_JOBS | 8 | Maximum retained job records | +| NVR_EXPORT_MAX_JOBS | 8 | Retained records | | NVR_EXPORT_PARALLEL | 1 | Concurrent preparations | -| NVR_EXPORT_TTL | 30m | Queue/processing expiry; retention resets on completion | +| NVR_EXPORT_TTL | 30m | Processing expiry; retention resets on completion | | NVR_EXPORT_PREPARE_TIMEOUT | 30m | Preparation deadline including queue wait | -Each job reserves its maximum output allowance, so defaults admit at most two reservations. Active expired downloads remain charged until release. Intermediate files used by the legacy trim renderer retain their existing separate bounds; this reservation is not a physical disk reservation or a bound on every filesystem allocation. Monitor free space. No originals are modified by the export path. Jobs are in memory and do not survive restart. +Each admitted job reserves its maximum output; defaults allow at most two reservations. Active expired downloads remain charged until their leases end. Legacy trim intermediates retain separate existing bounds. Reservations are not physical disk reservations or a bound on all filesystem allocations. Monitor free space. Jobs do not survive restart. -## API +POST create JSON contains camera_id, start_time and end_time (ordered RFC3339 timestamps). A 202 response has id, state, message, bytes, expires_at and a Location status URL. Ready status adds download_url. POST cancel uses {"id":"JOB_ID"} and confirms with 204. Subsequent requests require the creating session. Cancellation/expiry prevent new leases; already-started responses may continue. Logout and credential changes block subsequent requests, not necessarily existing responses. -POST create with application/json: +## Still required before release -```json -{"camera_id":"CAMERA_ID","start_time":"2026-11-21T00:00:00Z","end_time":"2026-11-21T00:01:00Z"} -``` +- Original-file delivery, compatible audio/video stream-copy assembly, and consent for adjusted keyframe ranges. The current renderer and its audio/pixel-format/frame-rate restrictions are unchanged. +- Optional accurate audio-preserving trimming and representative media validation. +- Migrate the real Chrome and routed browser acceptance harnesses from Blob URLs to job create/status/download/release semantics; enable scratch mounts in runtime fixtures. Existing browser harnesses still contain old assumptions and are not expected to pass this intermediate branch. Do not delete their media, hash, duration, playback or cancellation checks. +- Real large-file, slow-client, media, Range, authenticated-session and shutdown runtime checks. -A 202 response contains id, state, message, bytes and expires_at, with a Location status URL. Poll status; only ready jobs expose download_url. Download that same-origin URL with the same authenticated session. POST cancel with {"id":"JOB_ID"}; success is 204. Cancellation and expiry prevent new downloads; already acquired download leases are released when their requests finish. Logout/credential changes block subsequent requests but do not promise immediate revocation of an existing response. - -## Remaining - -- Original-file delivery, compatible audio/video stream-copy assembly and explicit consent for keyframe-adjusted ranges. -- Optional accurate audio-preserving trimming, full-range input handling and representative media validation. -- Browser job UI replacing the 64 MiB Blob workflow and corresponding real-browser acceptance updates. -- Full HTTP/media/large-transfer/slow-client/shutdown and runtime acceptance validation. - -Tests added for authentication, token scope, CSRF, strict request validation, session isolation, logout/credential changes, Range/HEAD downloads, cancellation and shutdown. Go tests have not been run in the assistant environment. +JavaScript syntax and the updated Node export UI/range regressions passed locally in UTC and America/Chicago. The 5 GiB test uses reported size metadata, not an actual 5 GiB transfer. Go and real-browser tests have not been run here. diff --git a/static/js/export-ui.js b/static/js/export-ui.js index 567e0ce..9de7348 100644 --- a/static/js/export-ui.js +++ b/static/js/export-ui.js @@ -1,6 +1,4 @@ -// Export dialog controls and bounded, monitored browser downloads. -const EXPORT_UI_LIMIT = 64 * 1024 * 1024; -const EXPORT_UI_TIMEOUT = 4 * 60 * 1000; +// Server-prepared export UI; only bounded control JSON is fetched. let exportUiBatch = null; let exportInitialRange = null; const EXPORT_RANGE_STEPS = [1000,10000,60000,600000]; @@ -20,8 +18,8 @@ function adjustDownloadBoundary(boundary,direction) { if(boundary==='start')downloadStartTime=new Date(value);else downloadEndTime=new Date(value); updateDownloadTimeDisplay();return true; } -// Replace the legacy expansion-only toolbar without changing playback controls. function installExportRangeControls() { + const button=document.getElementById('export-start-btn');if(button)button.textContent='Prepare exports on server'; if(document.getElementById('download-range-step'))return; const old=document.getElementById('download-time-display'); const toolbar=old?.parentElement?.parentElement;if(!toolbar)return; @@ -46,13 +44,13 @@ function installExportRangeControls() { const reset=exportUiNode('button','Reset to initial selection');reset.type='button';reset.className='btn';reset.id='download-range-reset'; reset.addEventListener('click',resetDownloadTimeRange);group.appendChild(reset); for(const id of ['download-time-display','download-duration-display']){const node=exportUiNode('div');node.id=id;group.appendChild(node);} - group.appendChild(exportUiNode('p','Times are local with time-zone labels. Exports require complete, unambiguous coverage. Trimming currently requires supported video-only inputs. After receiving, click Download prepared video.')); + group.appendChild(exportUiNode('p','Times are local with time-zone labels. Exports require complete coverage. Existing media compatibility restrictions still apply. Prepare on the server, then click Download prepared video.')); toolbar.replaceChildren(group); } function exportBoundaryLabel(time) { return time.toLocaleString(undefined,{year:'numeric',month:'2-digit',day:'2-digit',hour:'2-digit',minute:'2-digit',second:'2-digit',timeZoneName:'short',hour12:false}); } -// These dialog entry points supersede the legacy implementations in playback.js. +// Loaded after playback.js; these entry points own the export dialog. function updateDownloadTimeDisplay() { const start=downloadStartTime?.getTime(),end=downloadEndTime?.getTime(),valid=Number.isFinite(start)&&Number.isFinite(end)&&end>start; const busy=!!exportUiBatch?.running; @@ -86,150 +84,156 @@ function resetDownloadTimeRange() { if(exportUiBatch?.running || !exportInitialRange)return; downloadStartTime=new Date(exportInitialRange.start);downloadEndTime=new Date(exportInitialRange.end);updateDownloadTimeDisplay(); } -function exportUiError(kind,message) {const error=new Error(message);error.kind=kind;return error;} -function exportUiAbortError() {const error=new Error('Export canceled');error.name='AbortError';return error;} -function exportUiAwait(promise,signal) { - return new Promise((resolve,reject)=>{ - let settled=false; - const finish=(fn,value)=>{if(settled)return;settled=true;signal.removeEventListener('abort',abort);fn(value);}; - const abort=()=>finish(reject,exportUiAbortError()); - Promise.resolve(promise).then(value=>finish(resolve,value),error=>finish(reject,error)); - if(signal.aborted){abort();return;}signal.addEventListener('abort',abort,{once:true}); - }); -} -function exportUiState(job,state,text) {job.state=state;job.row.dataset.state=state;job.status.textContent=text;} -function exportUiCancel(job) { - if(job.finished)return; - if(job.finishReady){job.finishReady(job.handed?'handed':'canceled',job.handed?'Browser download requested; prepared file released. Disk saving is not verified.':'Canceled; prepared file discarded.');return;} - job.reason='canceled';job.controller.abort();exportUiState(job,'canceled','Canceled; no browser download requested.'); -} -function cancelExportDownloads() {if(exportUiBatch)exportUiBatch.jobs.forEach(exportUiCancel);} -function exportUiCancelReader(reader) {try{Promise.resolve(reader.cancel()).catch(()=>{});}catch(_){} } -async function exportUiErrorText(response,signal) { - if(!response.body || typeof response.body.getReader!=='function')return ''; + +const EXPORT_JOB_POLL_MS=1500; +const EXPORT_JOB_REQUEST_MS=45000; +const EXPORT_JOB_WAIT_MS=2*60*60*1000; +function exportUiState(job,state,message){job.state=state;job.row.dataset.state=state;job.status.textContent=String(message).slice(0,1200);} +function exportUiText(text){return String(text ?? '').replace(/[\u0000-\u001f\u007f]/g,' ').slice(0,600);} +function exportUiError(message){return new Error(message);} +async function exportUiRead(response,signal,limit=8192){ + if(!response.body || typeof response.body.getReader!=='function')throw exportUiError('Streaming JSON responses are unavailable.'); const reader=response.body.getReader(),chunks=[];let size=0,done=false; + const canceled=()=>{try{Promise.resolve(reader.cancel()).catch(()=>{});}catch(_){}}; + signal.addEventListener('abort',canceled,{once:true}); try{ - while(size<2048){const part=await exportUiAwait(reader.read(),signal);if(part.done){done=true;break;}const chunk=part.value.slice(0,2048-size);chunks.push(chunk);size+=chunk.byteLength;} - const raw=new Uint8Array(size);let offset=0;for(const chunk of chunks){raw.set(chunk,offset);offset+=chunk.byteLength;} - return new TextDecoder().decode(raw).replace(/[\u0000-\u001f\u007f]/g,' ').slice(0,600); - }finally{if(!done)exportUiCancelReader(reader);try{reader.releaseLock();}catch(_){} } + while(true){ + if(signal.aborted)throw exportUiError('Export control request timed out.'); + const part=await reader.read();if(part.done){done=true;break;} + if(part.value.byteLength>limit-size)throw exportUiError('Export control response exceeded its size limit.'); + size+=part.value.byteLength;chunks.push(part.value); + } + if(signal.aborted)throw exportUiError('Export control request timed out.'); + const bytes=new Uint8Array(size);let offset=0;for(const chunk of chunks){bytes.set(chunk,offset);offset+=chunk.byteLength;} + return new TextDecoder().decode(bytes); + }finally{signal.removeEventListener('abort',canceled);if(!done)canceled();try{reader.releaseLock();}catch(_){}} } -function exportUiFailureMessage(response) { - const plan=response.headers.get('X-Export-Plan-Status'),trim=response.headers.get('X-Export-Trim-Status'); +function exportUiHTTPMessage(response){ if(response.status===401 || response.status===403)return 'Authentication expired or access denied; sign in again.'; - if(plan==='gap')return 'The selected interval includes missing recording coverage. Some footage may exist, but the full interval is not covered. Move Start later or End earlier as appropriate; no gaps were silently skipped.'; - if(plan==='overlap')return 'The selected interval contains overlapping recordings. No automatic clip choice was made.'; - if(trim==='unsupported' || response.status===422)return 'Footage may be playable but this subsection cannot be trimmed with the current input profile. Audio-containing or otherwise incompatible inputs are unsupported; audio was not silently removed and the interval was not widened.'; - if(response.status===404)return 'Camera or recording coverage was not found for the selected interval.'; - if(response.status===413)return 'Server export limit exceeded; shorten the interval or select fewer clips.'; - if(response.status===504)return 'Server preparation or rendering timed out; no prepared download was received.'; - if(response.status===503)return 'Recording metadata unavailable, changed, or server shutting down; retry after reviewing recording availability.'; - return `Export request failed (HTTP ${response.status}).`; + if(response.status===503)return 'Prepared exports unavailable: check NVR_EXPORT_SCRATCH_DIR, recording metadata, and whether the server is restarting.'; + if(response.status===429)return 'Server export capacity is full. Release earlier prepared files or wait for expiry, then retry.'; + if(response.status===404)return 'Export or camera unavailable; the job may have expired or the server restarted.'; + const plan=response.headers.get('X-Export-Plan-Status'); + if(plan==='gap')return 'The full interval is not covered by recordings; adjust Start or End. No gaps were silently skipped.'; + if(plan==='overlap')return 'Overlapping recordings require an explicit clip choice.'; + if(response.status===413)return 'Server export limit exceeded; reduce the selected interval.'; + if(response.status===422)return 'The recording profile is not supported by the current trim renderer.'; + return `Export control request failed (HTTP ${response.status}).`; +} +// Only bounded JSON/control bodies use fetch. Video uses a normal download URL. +async function exportUiRequest(path,body){ + const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),EXPORT_JOB_REQUEST_MS); + try{ + const options={credentials:'same-origin',cache:'no-store',redirect:'error',signal:controller.signal}; + if(body!==undefined){options.method='POST';options.headers={'Content-Type':'application/json'};options.body=JSON.stringify(body);} + const response=await fetch(path,options); + if(response.redirected)throw exportUiError('Unexpected redirect; sign in again.'); + if(response.status===204)return {status:204}; + const text=await exportUiRead(response,controller.signal); + if(response.status!==200 && response.status!==202)throw exportUiError(exportUiHTTPMessage(response)+(text?' Server: '+exportUiText(text):'')); + if((response.headers.get('Content-Type') || '').split(';')[0].trim().toLowerCase()!=='application/json')throw exportUiError('Expected export JSON, not a video or login page.'); + let value;try{value=JSON.parse(text);}catch(_){throw exportUiError('Invalid export JSON response.');} + return {status:response.status,value}; + }finally{clearTimeout(timer);} +} +function exportUiView(value,expected){ + if(!value || typeof value!=='object' || Array.isArray(value) || !/^[0-9a-f]{64}$/.test(value.id || '') || (expected && value.id!==expected) || !['queued','preparing','ready','failed','canceled','expired'].includes(value.state) || typeof value.message!=='string' || !Number.isSafeInteger(value.bytes) || value.bytes<0 || !Number.isFinite(Date.parse(value.expires_at)))throw exportUiError('Invalid export status response.'); + if(value.state==='ready' && (value.bytes===0 || value.download_url!==`/api/exports/download?id=${value.id}`))throw exportUiError('Invalid prepared download response.'); + return value; } -function exportUiPolicy(response) { - const policy=response.headers.get('X-Export-Range-Policy'); - if(policy==='whole-clips-untrimmed')return 'Whole-clip copy; timing is metadata-derived, not a calibrated camera clock.'; - if(policy==='decoded-video-frame-window'){ - const raw=response.headers.get('X-Export-Output-Duration-Seconds') || ''; - const duration=/^\d+(?:\.\d{1,9})?$/.test(raw) && Number(raw)>0 && Number(raw)<86400?` Reported output: ${Number(raw)} seconds.`:''; - return 'Frame-granular trim; video-only inputs; not exact camera wall-clock boundaries.'+duration; +function exportUiRemoveDownload(job){if(job.save){job.save.disabled=true;job.save.remove();job.save=null;}job.url=null;clearTimeout(job.expiryTimer);} +function exportUiFinish(job,state,text){exportUiRemoveDownload(job);job.finished=true;exportUiState(job,state,text);if(job.finishReady){const finish=job.finishReady;job.finishReady=null;finish();}} +function exportUiWait(job){return new Promise(resolve=>{job.wake=()=>{clearTimeout(job.pollTimer);job.wake=null;resolve();};job.pollTimer=setTimeout(job.wake,EXPORT_JOB_POLL_MS);});} +async function exportUiCancel(job){ + if(job.canceling || ['canceled','expired','released'].includes(job.state))return; + job.reason='canceled';job.wake?.(); + if(!job.serverId){ + if(job.creating){exportUiState(job,'canceling','Cancellation requested; waiting for the creation response to identify the server job.');return;} + exportUiFinish(job,'canceled','Canceled before a server job was created.');job.cancel.disabled=true;return; } - throw exportUiError('policy','Export policy missing or unrecognized; reload after updating the server.'); -} -async function exportUiReceive(response,job) { - const length=response.headers.get('Content-Length');let expected=null; - if(length!==null){if(!/^\d+$/.test(length))throw exportUiError('length','Invalid response length.');expected=Number(length);if(!Number.isSafeInteger(expected) || expected>EXPORT_UI_LIMIT)throw exportUiError('limit','Response exceeds the 64 MiB monitored-browser limit; reduce the range.');} - if(!response.body || typeof response.body.getReader!=='function')throw exportUiError('reader','Streaming response reading is unavailable; no unbounded buffering fallback was used.'); - const reader=response.body.getReader();let chunks=[],size=0,done=false; + job.canceling=true;job.cancel.disabled=true;exportUiRemoveDownload(job); + exportUiState(job,'canceling','Requesting cancellation/release on the server. Existing browser transfers may continue.'); try{ - while(true){ - const part=await exportUiAwait(reader.read(),job.controller.signal);if(part.done){done=true;break;} - if(part.value.byteLength>EXPORT_UI_LIMIT-size)throw exportUiError('limit','Response exceeds the 64 MiB monitored-browser limit; reduce the range.'); - chunks.push(part.value);size+=part.value.byteLength; - exportUiState(job,'receiving',`Receiving ${(size/(1024*1024)).toFixed(1)} MiB${expected===null?'':` of ${(expected/(1024*1024)).toFixed(1)} MiB`}; not saved yet.`); - } - if(size===0 || (expected!==null && size!==expected))throw exportUiError('incomplete','Transfer ended with an empty or mismatched-length response; no file was offered.'); - const prefix=new Uint8Array(Math.min(12,size));let copied=0; - for(const chunk of chunks){const count=Math.min(chunk.byteLength,prefix.length-copied);prefix.set(chunk.subarray(0,count),copied);copied+=count;if(copied===prefix.length)break;} - if(size<12 || prefix[4]!==102 || prefix[5]!==116 || prefix[6]!==121 || prefix[7]!==112)throw exportUiError('format','Response did not contain the expected MP4 header; no file was offered.'); - const blob=new Blob(chunks,{type:'video/mp4'});chunks=[];return {blob,size}; - }finally{if(!done)exportUiCancelReader(reader);try{reader.releaseLock();}catch(_){} } -} -function exportUiFilename(job) { - const name=job.name.replace(/[^A-Za-z0-9._-]/g,'_').slice(0,80) || 'camera'; - const stamp=value=>value.replace(/[^A-Za-z0-9]/g,''); - return `${name}_${stamp(job.start)}_${stamp(job.end)}.mp4`; -} -function exportUiOffer(job,received) { - return new Promise((resolve,reject)=>{ - try{job.url=URL.createObjectURL(received.blob);}catch(error){reject(error);return;} - let finished=false; - job.finishReady=(state,message)=>{ - if(finished)return;finished=true;clearTimeout(job.readyTimer); - if(job.url){URL.revokeObjectURL(job.url);job.url=null;} - job.save.disabled=true;job.cancel.disabled=true;job.finishReady=null;exportUiState(job,state,message);resolve(); - }; + const response=await exportUiRequest('/api/exports/cancel',{id:job.serverId}); + if(response.status!==204)throw exportUiError('Unexpected cancellation response.'); + exportUiFinish(job,job.handed?'released':'canceled',job.handed?'Server confirmed release. Any already-started download may continue; disk saving is not verified.':'Server confirmed export cancellation/release.');job.cancel.disabled=true; + }catch(error){ + exportUiFinish(job,'release-failed',`Cancellation could not be confirmed. ${error.message} The job may remain until server expiry.`);job.cancel.textContent='Retry cancellation/release';job.cancel.disabled=false; + }finally{job.canceling=false;} +} +function cancelExportDownloads(){return exportUiBatch?Promise.all(exportUiBatch.jobs.map(job=>exportUiCancel(job))):Promise.resolve();} +function exportUiOffer(job,view){ + return new Promise(resolve=>{ + job.finishReady=resolve;job.url=`/api/exports/download?id=${job.serverId}`; + const expires=Date.parse(view.expires_at),remaining=expires-Date.now(); + if(remaining<=0){exportUiFinish(job,'expired','Prepared export expired; prepare it again.');return;} job.save=exportUiNode('button','Download prepared video');job.save.type='button';job.save.className='btn btn-primary'; job.save.addEventListener('click',()=>{ - if(finished || job.handed)return; - const anchor=document.createElement('a');anchor.href=job.url;anchor.download=exportUiFilename(job); - try{document.body.appendChild(anchor);anchor.click();anchor.remove();} - catch(_){anchor.remove();job.finishReady('failed','Browser download request failed; no disk save was verified.');return;} - job.handed=true;job.save.disabled=true;job.cancel.textContent='Release prepared file / continue'; - exportUiState(job,'handed','Browser download requested. Check your browser downloads; disk saving is not verified. Release after the browser accepts it, or wait 60 seconds.'); - clearTimeout(job.readyTimer);job.readyTimer=setTimeout(()=>job.finishReady?.('handed','Browser download requested; prepared file released. Disk saving is not verified.'),60000); + if(job.finished || job.canceling || job.handed)return; + if(Date.now()>=expires){exportUiFinish(job,'expired','Prepared export expired; prepare it again.');job.cancel.disabled=true;return;} + const anchor=exportUiNode('a');anchor.href=job.url;anchor.download=`export-${job.serverId}.mp4`; + try{document.body.appendChild(anchor);anchor.click();}catch(_){exportUiState(job,'ready','Browser download request failed. Try again before expiry.');return;}finally{anchor.remove();} + job.handed=true;job.save.disabled=true; + exportUiState(job,'handed','Browser download requested; check your browser downloads. Disk saving is not verified. Release after the browser accepts it; existing transfers retain their server file lease.'); }); - job.row.appendChild(job.save); - job.readyTimer=setTimeout(()=>job.finishReady?.('expired','Prepared file expired after 2 minutes without a download request. Prepare the export again.'),120000); - exportUiState(job,'ready',`Ready to download: ${(received.size/(1024*1024)).toFixed(1)} MiB received. Click Download prepared video below within 2 minutes. Nothing has been saved yet; media completeness and disk saving are not verified.`); + job.row.appendChild(job.save);job.cancel.textContent='Release prepared file / continue';job.cancel.disabled=false; + exportUiState(job,'ready',`Ready on server: ${(view.bytes/(1024*1024)).toFixed(1)} MiB. Click Download prepared video before ${new Date(expires).toLocaleString()}. Nothing has been saved by this page; disk saving is not verified.`); + job.expiryTimer=setTimeout(()=>{exportUiFinish(job,'expired','Prepared export reached its expiry time; request a new export if needed. An already-started browser transfer may continue.');job.cancel.disabled=true;},Math.min(remaining,2147483647)); job.save.focus?.(); }); } -async function exportUiRunJob(job) { - if(job.reason==='canceled'){job.finished=true;job.cancel.disabled=true;return;} - let response;job.transferring=true; - const timer=setTimeout(()=>{if(job.transferring && !job.finished){job.reason='timeout';job.controller.abort();}},EXPORT_UI_TIMEOUT); +async function exportUiRunJob(job){ + if(job.reason==='canceled'){job.cancel.disabled=true;return;} try{ - exportUiState(job,'preparing','Preparing export on server. After receiving it, click Download prepared video; preparing alone does not save a file.'); - const query=`camera_id=${encodeURIComponent(job.id)}&start_time=${encodeURIComponent(job.start)}&end_time=${encodeURIComponent(job.end)}`; - response=await exportUiAwait(fetch(`/api/download?${query}`,{credentials:'same-origin',cache:'no-store',redirect:'error',signal:job.controller.signal}),job.controller.signal); - if(!response.ok || response.status!==200){const excerpt=await exportUiErrorText(response,job.controller.signal);throw exportUiError('http',exportUiFailureMessage(response)+(excerpt?' Server: '+excerpt:''));} - if(response.redirected || (response.headers.get('Content-Type') || '').split(';')[0].trim().toLowerCase()!=='video/mp4')throw exportUiError('type','Unexpected response, possibly a sign-in page; no file was offered.'); - job.policy.textContent=exportUiPolicy(response); - const received=await exportUiReceive(response,job); - job.transferring=false;clearTimeout(timer);if(job.controller.signal.aborted)throw exportUiAbortError(); - await exportUiOffer(job,received); + job.creating=true;exportUiState(job,'preparing','Requesting server preparation. Video will not be buffered in this page.'); + let result; + try{result=await exportUiRequest('/api/exports/create',{camera_id:job.id,start_time:job.start,end_time:job.end});}finally{job.creating=false;} + if(result.status!==202)throw exportUiError('Expected server job acceptance.'); + let view=exportUiView(result.value);job.serverId=view.id; + if(job.reason==='canceled'){await exportUiCancel(job);return;} + const deadline=Date.now()+EXPORT_JOB_WAIT_MS; + while(true){ + if(job.reason==='canceled')return; + if(view.state==='ready'){await exportUiOffer(job,view);return;} + if(['failed','canceled','expired'].includes(view.state)){exportUiFinish(job,view.state,exportUiText(view.message));job.cancel.disabled=true;return;} + exportUiState(job,view.state,view.state==='queued'?'Queued on server.':'Preparing on server; large exports can take time.'); + if(Date.now()>=deadline)throw exportUiError('Stopped waiting after two hours. The job may continue until its server deadline or expiry.'); + await exportUiWait(job);if(job.reason==='canceled')return; + result=await exportUiRequest(`/api/exports/status?id=${job.serverId}`); + if(job.reason==='canceled')return; + if(result.status!==200)throw exportUiError('Unexpected export status response.'); + view=exportUiView(result.value,job.serverId); + } }catch(error){ - job.controller.abort(); - if(job.reason==='timeout')exportUiState(job,'failed','Browser export deadline exceeded; request canceled.'); - else if(job.reason==='canceled')exportUiState(job,'canceled','Canceled; no browser download requested.'); - else exportUiState(job,'failed',String(error.message || 'Export failed').slice(0,1000)); - if(response?.body){try{Promise.resolve(response.body.cancel()).catch(()=>{});}catch(_){} } - }finally{clearTimeout(timer);job.transferring=false;job.finished=true;job.cancel.disabled=true;} + if(job.reason==='canceled' && job.serverId){await exportUiCancel(job);return;} + exportUiFinish(job,'failed',`${error.message} ${job.serverId?'The server job may still exist; use cancellation below or wait for expiry.':'Creation outcome may be unknown if the response was lost; no automatic retry was made.'}`); + job.cancel.disabled=!job.serverId;if(job.serverId)job.cancel.textContent='Cancel server job'; + } } -function startExportDownloads() { +function startExportDownloads(){ if(exportUiBatch?.running)return exportUiBatch.promise; const host=document.getElementById('download-status'),list=document.getElementById('download-progress-list'); - if(!host || !list){console.error('Export status region missing; no request started');return Promise.resolve();} + if(!host || !list)return Promise.resolve(); + // Keep unresolved release failures visible rather than orphaning their controls. + if(exportUiBatch?.jobs.some(job=>job.state==='release-failed' || (job.serverId && job.state==='failed' && !job.cancel.disabled)))return exportUiBatch.promise; host.style.display='block';list.replaceChildren(); const start=downloadStartTime?.getTime(),end=downloadEndTime?.getTime(),ids=[...new Set(downloadCameras)]; if(!Number.isFinite(start) || !Number.isFinite(end) || end<=start || ids.length<1 || ids.length>8){list.appendChild(exportUiNode('p','Choose a valid range and one to eight cameras. No request started.'));return Promise.resolve();} - const cameras=ids.map(id=>allCameras.find(camera=>camera.id===id)); - if(cameras.some(camera=>!camera || typeof camera.id!=='string' || camera.id.length>256)){list.appendChild(exportUiNode('p','A selected camera is unavailable. Reload the camera list.'));return Promise.resolve();} - const stop=exportUiNode('button','Cancel all exports');stop.type='button';stop.addEventListener('click',cancelExportDownloads);list.appendChild(stop); - list.appendChild(exportUiNode('p','Two steps: prepare and receive, then click Download prepared video for each camera. Maximum 64 MiB per export, one at a time. The queue pauses for your download action and file release. Disk saving cannot be verified.')); - const jobs=cameras.map(camera=>{ - const row=exportUiNode('div');row.className='export-job';row.dataset.cameraId=camera.id; - const name=String(camera.name ?? camera.id).slice(0,256),status=exportUiNode('p','Queued'),policy=exportUiNode('p'); - status.setAttribute('aria-live','polite');row.appendChild(exportUiNode('h4',name));row.appendChild(status);row.appendChild(policy); - const cancel=exportUiNode('button','Cancel');cancel.type='button';row.appendChild(cancel);list.appendChild(row); - const job={id:camera.id,name,start:new Date(start).toISOString(),end:new Date(end).toISOString(),row,status,policy,cancel,controller:new AbortController(),reason:null,finished:false,handed:false,url:null}; - cancel.addEventListener('click',()=>exportUiCancel(job));exportUiState(job,'queued','Queued');return job; + const cameras=ids.map(id=>allCameras.find(cam=>cam.id===id)); + if(cameras.some(cam=>!cam || typeof cam.id!=='string' || cam.id.length>256)){list.appendChild(exportUiNode('p','Selected camera unavailable; reload the camera list.'));return Promise.resolve();} + const stop=exportUiNode('button','Cancel all exports');stop.type='button';stop.addEventListener('click',()=>{void cancelExportDownloads();});list.appendChild(stop); + list.appendChild(exportUiNode('p','Prepare on the server, then download with your browser. Server storage limits apply; this page does not buffer video or impose a 64 MiB file limit. The next camera starts after release or expiry of the current prepared export. Existing rendering compatibility restrictions still apply.')); + const jobs=cameras.map(cam=>{ + const row=exportUiNode('div');row.className='export-job';row.dataset.cameraId=cam.id; + const status=exportUiNode('p','Queued'),cancel=exportUiNode('button','Cancel');cancel.type='button';status.setAttribute('aria-live','polite'); + row.appendChild(exportUiNode('h4',String(cam.name ?? cam.id).slice(0,256)));row.appendChild(status);row.appendChild(cancel);list.appendChild(row); + const job={id:cam.id,start:new Date(start).toISOString(),end:new Date(end).toISOString(),row,status,cancel,state:'queued',reason:null,serverId:null,creating:false,canceling:false,finished:false,handed:false,url:null,save:null,finishReady:null,wake:null}; + cancel.addEventListener('click',()=>{void exportUiCancel(job);});return job; }); const batch={jobs,running:true,promise:null};exportUiBatch=batch;updateDownloadTimeDisplay(); const button=document.getElementById('export-start-btn');if(button)button.disabled=true; - batch.promise=(async()=>{try{for(const job of jobs)await exportUiRunJob(job);}finally{batch.running=false;stop.disabled=true;if(button)button.disabled=false;updateDownloadTimeDisplay();}})(); - return batch.promise; + batch.promise=(async()=>{try{for(const job of jobs)await exportUiRunJob(job);}finally{batch.running=false;if(button)button.disabled=false;updateDownloadTimeDisplay();}})();return batch.promise; } -window.addEventListener('pagehide',cancelExportDownloads); +// Do not auto-release on navigation: downloading can itself trigger pagehide, +// and aborting creation can lose the server ID. Server TTL bounds abandoned jobs. if(document.readyState==='loading')document.addEventListener('DOMContentLoaded',installExportRangeControls);else installExportRangeControls(); diff --git a/tests/export-range-controls.cjs b/tests/export-range-controls.cjs index ca8ee0c..2f553ba 100644 --- a/tests/export-range-controls.cjs +++ b/tests/export-range-controls.cjs @@ -1,70 +1,19 @@ const assert=require('node:assert/strict'); -const fs=require('node:fs'); -const vm=require('node:vm'); -const path=require('node:path'); -const root=process.argv[2] || '.'; -const nodes=new Map();let downloads=0,revoked=0,nextTimer=0; -const timers=new Map(); -class Node { - constructor(tag){this.tagName=tag;this.children=[];this.style={};this.dataset={};this.handlers={};this.disabled=false;this.classList={add(){}};} - set id(value){this._id=value;nodes.set(value,this);}get id(){return this._id;} - appendChild(child){child.parentElement=this;this.children.push(child);return child;} - replaceChildren(...children){const remove=node=>{if(node.id)nodes.delete(node.id);node.children.forEach(remove);};this.children.forEach(remove);this.children=[];children.forEach(child=>this.appendChild(child));} - setAttribute(key,value){this[key]=value;}addEventListener(name,fn){this.handlers[name]=fn;} - click(){if(!this.disabled){if(this.tagName==='a')downloads++;this.handlers.click?.();}} - remove(){}focus(){this.focused=true;} +const {environment}=require('./export-ui-fixture.cjs'); +const env=environment(()=>{throw Error('Range controls must not start requests');}),run=env.run; +run('openDownloadModal()');assert.equal(env.find('download-range-step').value,'60000');assert.equal(run('downloadEndTime-downloadStartTime'),1200000); +const start=run('downloadStartTime.getTime()'),end=run('downloadEndTime.getTime()'); +for(const boundary of ['start','end'])for(const direction of [-1,1]){ + run('resetDownloadTimeRange()');env.find(`download-${boundary}-${direction<0?'earlier':'later'}`).click(); + assert.equal(run('downloadStartTime.getTime()'),boundary==='start'?start+direction*60000:start); + assert.equal(run('downloadEndTime.getTime()'),boundary==='end'?end+direction*60000:end); } -const document={readyState:'loading',addEventListener(){},createElement:tag=>new Node(tag),getElementById:id=>nodes.get(id)||null,body:new Node('body')}; -const toolbar=new Node('div'),wrap=toolbar.appendChild(new Node('div')),old=wrap.appendChild(new Node('div'));old.id='download-time-display'; -for(const id of ['download-modal','download-status','download-progress-list','export-start-btn']){const node=new Node('div');node.id=id;} -const context=vm.createContext({console,document,window:{addEventListener(){}},AbortController,Blob,TextDecoder,Uint8Array,URL:{createObjectURL:()=> 'blob:prepared',revokeObjectURL(){revoked++;}},setTimeout(fn,ms){const id=++nextTimer;timers.set(id,{fn,ms});return id;},clearTimeout(id){timers.delete(id);}}); -vm.runInContext('let downloadStartTime=null,downloadEndTime=null,downloadCameras=[],currentTimePosition=Date.parse("2026-11-21T23:59:59Z"),selectedCameras=["one"],allCameras=[{id:"one",name:"Hall"}];function renderDownloadCameraSelection(){}',context); -vm.runInContext(fs.readFileSync(path.join(root,'static/js/export-ui.js'),'utf8'),context); -const run=code=>vm.runInContext(code,context); -const headers=values=>({get:key=>values[key]??null}); -const mp4=new Uint8Array([0,0,0,12,102,116,121,112,105,115,111,109]); -function response(status,values,data=mp4){let sent=false;return {status,ok:status===200,redirected:false,headers:headers(values),body:{getReader(){return {async read(){if(sent)return {done:true};sent=true;return {done:false,value:data};},cancel(){},releaseLock(){}};},cancel(){}}};} -const flush=()=>new Promise(resolve=>setImmediate(resolve)); -async function main(){ - run('openDownloadModal()'); - assert.equal(nodes.get('download-range-step').value,'60000'); - assert.equal(run('downloadEndTime-downloadStartTime'),1200000); - const start=run('downloadStartTime.getTime()'),end=run('downloadEndTime.getTime()'); - for(const boundary of ['start','end'])for(const direction of [-1,1]){ - run('resetDownloadTimeRange()');nodes.get(`download-${boundary}-${direction<0?'earlier':'later'}`).click(); - assert.equal(run('downloadStartTime.getTime()'),boundary==='start'?start+direction*60000:start); - assert.equal(run('downloadEndTime.getTime()'),boundary==='end'?end+direction*60000:end); - } - for(const step of [1000,10000,60000,600000]){run('resetDownloadTimeRange()');nodes.get('download-range-step').value=String(step);run('adjustDownloadBoundary("start",1)');assert.equal(run('downloadStartTime.getTime()'),start+step);} - run('downloadStartTime=new Date(1000);downloadEndTime=new Date(2000)');nodes.get('download-range-step').value='1000';run('updateDownloadTimeDisplay()'); - assert.equal(nodes.get('download-start-later').disabled,true);assert.equal(nodes.get('download-end-earlier').disabled,true); - assert.equal(run('adjustDownloadBoundary("start",1)'),false);assert.equal(run('downloadEndTime.getTime()'),2000); - assert.equal(run('exportRangeCandidate(NaN,2000,"start",1000)'),null); - assert.equal(run('exportRangeCandidate(0,1000,"end",1e20)'),null); - run('currentTimePosition=0;resetDownloadTimeRange()');assert.equal(run('downloadStartTime.getTime()'),start); - run('currentTimePosition=Date.parse("2026-11-21T23:59:59Z");openDownloadModal()'); - assert.match(nodes.get('download-start-display').textContent,/2026/);assert.match(nodes.get('download-end-display').textContent,/2026/); - // Elapsed-time arithmetic crosses DST transitions without a calendar-hour jump. - run('downloadStartTime=new Date("2026-11-01T06:59:59Z");downloadEndTime=new Date("2026-11-01T08:00:00Z")'); - nodes.get('download-range-step').value='1000';run('adjustDownloadBoundary("start",1)');assert.equal(run('downloadStartTime.toISOString()'),'2026-11-01T07:00:00.000Z'); - run('resetDownloadTimeRange()'); - context.fetch=async()=>response(200,{'Content-Type':'video/mp4','Content-Length':String(mp4.length),'X-Export-Range-Policy':'decoded-video-frame-window','X-Export-Output-Duration-Seconds':'120'}); - const promise=run('startExportDownloads()');await flush(); - assert.equal(run('exportUiBatch.jobs[0].state'),'ready');assert.equal(downloads,0); - assert.equal(run('exportUiBatch.jobs[0].save.textContent'),'Download prepared video'); - assert.equal(nodes.get('download-start-earlier').disabled,true); - run('exportUiBatch.jobs[0].save.click()');assert.equal(downloads,1);assert.equal(run('exportUiBatch.jobs[0].state'),'handed'); - run('exportUiBatch.jobs[0].save.click()');assert.equal(downloads,1); - run('exportUiBatch.jobs[0].cancel.click()');await promise;assert.equal(revoked,1);assert.equal(nodes.get('download-start-earlier').disabled,false); - context.fetch=async()=>response(409,{'X-Export-Plan-Status':'gap'},new TextEncoder().encode('Missing coverage')); - await run('startExportDownloads()');assert.match(run('exportUiBatch.jobs[0].status.textContent'),/full interval is not covered/);assert.equal(downloads,1); - context.fetch=async()=>response(422,{'X-Export-Trim-Status':'unsupported'},new TextEncoder().encode('Audio-containing inputs unsupported')); - await run('startExportDownloads()');assert.match(run('exportUiBatch.jobs[0].status.textContent'),/audio was not silently removed/);assert.equal(downloads,1); - context.fetch=async()=>response(200,{'Content-Type':'video/mp4','X-Export-Range-Policy':'whole-clips-untrimmed'}); - const expires=run('startExportDownloads()');await flush(); - const expiry=[...timers.values()].find(timer=>timer.ms===120000);assert.ok(expiry);expiry.fn();await expires; - assert.equal(run('exportUiBatch.jobs[0].state'),'expired');assert.equal(revoked,2);assert.equal(downloads,1); - assert.equal(timers.size,0); - console.log('PASS range directions, steps, crossing, reset, DST, covered-subsection handoff, gap, unsupported audio, expiry'); -} -main().catch(error=>{console.error(error);process.exitCode=1;}); +for(const step of [1000,10000,60000,600000]){run('resetDownloadTimeRange()');env.find('download-range-step').value=String(step);run('adjustDownloadBoundary("start",1)');assert.equal(run('downloadStartTime.getTime()'),start+step);} +run('downloadStartTime=new Date(1000);downloadEndTime=new Date(2000)');env.find('download-range-step').value='1000';run('updateDownloadTimeDisplay()'); +assert.equal(env.find('download-start-later').disabled,true);assert.equal(env.find('download-end-earlier').disabled,true);assert.equal(run('adjustDownloadBoundary("start",1)'),false); +assert.equal(run('exportRangeCandidate(NaN,2000,"start",1000)'),null);assert.equal(run('exportRangeCandidate(0,1000,"end",1e20)'),null); +run('currentTimePosition=0;resetDownloadTimeRange()');assert.equal(run('downloadStartTime.getTime()'),start); +assert.match(env.find('download-start-display').textContent,/2026/);assert.match(env.find('download-end-display').textContent,/2026/); +run('downloadStartTime=new Date("2026-11-01T06:59:59Z");downloadEndTime=new Date("2026-11-01T08:00:00Z")');run('adjustDownloadBoundary("start",1)');assert.equal(run('downloadStartTime.toISOString()'),'2026-11-01T07:00:00.000Z'); +run('exportUiBatch={running:true};updateDownloadTimeDisplay()');assert.equal(env.find('download-start-earlier').disabled,true);assert.equal(run('adjustDownloadBoundary("start",-1)'),false); +assert.equal(env.requests.length,0);console.log('PASS range directions, steps, boundary guards, initial reset, full dates, DST elapsed arithmetic and busy controls'); diff --git a/tests/export-ui-fixture.cjs b/tests/export-ui-fixture.cjs new file mode 100644 index 0000000..b56a24e --- /dev/null +++ b/tests/export-ui-fixture.cjs @@ -0,0 +1,30 @@ +const fs=require('node:fs'),vm=require('node:vm'),path=require('node:path'); +const root=process.argv[2] || '.'; +const source=fs.readFileSync(path.join(root,'static/js/export-ui.js'),'utf8'); +function environment(fetcher,ids=['front']){ + const timers=new Map(),requests=[],downloads=[];let serial=0; + class Element{ + constructor(tag){this.tagName=tag;this.children=[];this.style={};this.dataset={};this.listeners={};this.disabled=false;this.text='';this.classList={add(){}};} + set textContent(v){this.text=String(v);this.children=[];}get textContent(){return this.text+this.children.map(c=>c.textContent).join('');} + set innerHTML(_){throw Error('HTML sink');}set outerHTML(_){throw Error('HTML sink');}insertAdjacentHTML(){throw Error('HTML sink');} + appendChild(c){c.parentElement=this;this.children.push(c);return c;}replaceChildren(...children){this.children=[];this.text='';children.forEach(c=>this.appendChild(c));} + setAttribute(k,v){if(/^on/i.test(k))throw Error('inline handler');this[k]=v;}addEventListener(k,fn){this.listeners[k]=fn;} + remove(){if(this.parentElement)this.parentElement.children=this.parentElement.children.filter(c=>c!==this);} + click(){if(this.disabled)return;if(this.tagName==='a')downloads.push(this.href);this.listeners.click?.();}focus(){} + } + const body=new Element('body');const make=(id,parent=body)=>{const e=new Element('div');e.id=id;parent.appendChild(e);return e;}; + const toolbar=make('toolbar'),wrap=make('wrap',toolbar);make('download-time-display',wrap);make('download-duration-display',wrap); + for(const id of ['download-modal','download-status','download-progress-list','export-start-btn'])make(id); + function find(id,node=body){if(node.id===id)return node;for(const child of node.children){const result=find(id,child);if(result)return result;}return null;} + const context=vm.createContext({console,AbortController,TextDecoder,Uint8Array,document:{readyState:'loading',addEventListener(){},body,createElement:t=>new Element(t),getElementById:find},setTimeout(fn,ms){const id=++serial;timers.set(id,{fn,ms});return id;},clearTimeout(id){timers.delete(id);},fetch(url,options){requests.push({url,options});return fetcher(url,options);}}); + vm.runInContext(`let downloadStartTime=new Date('2026-11-21T00:00:00Z'),downloadEndTime=new Date('2026-11-21T00:01:00Z'),downloadCameras=${JSON.stringify(ids)},currentTimePosition=Date.parse('2026-11-21T23:59:59Z'),selectedCameras=${JSON.stringify(ids)},allCameras=[{id:'front',name:''},{id:'back',name:'Back'}];function renderDownloadCameraSelection(){}`,context); + vm.runInContext(source,context); + const run=code=>vm.runInContext(code,context); + const flush=async()=>{for(let i=0;i<8;i++)await new Promise(resolve=>setImmediate(resolve));}; + const fire=async ms=>{const pair=[...timers].find(([,t])=>t.ms===ms);if(!pair)throw Error('Missing timer '+ms);timers.delete(pair[0]);pair[1].fn();await flush();}; + return {context,run,flush,fire,find,timers,requests,downloads,jobs:()=>run('exportUiBatch.jobs')}; +} +const id=letter=>letter.repeat(64); +const view=(letter,state='ready',extra={})=>({id:id(letter),state,message:state,bytes:state==='ready'?5*1024*1024*1024:0,expires_at:new Date(Date.now()+3600000).toISOString(),...(state==='ready'?{download_url:'/api/exports/download?id='+id(letter)}:{}),...extra}); +function response(status,data,headers={}){return new Response(status===204?null:(typeof data==='string'?data:JSON.stringify(data)),{status,headers:{'Content-Type':'application/json',...headers}});} +module.exports={environment,source,id,view,response}; diff --git a/tests/export-ui.cjs b/tests/export-ui.cjs index bfda926..8db046d 100644 --- a/tests/export-ui.cjs +++ b/tests/export-ui.cjs @@ -1,62 +1,39 @@ const assert=require('node:assert/strict'); -const fs=require('node:fs'); -const vm=require('node:vm'); -const source=fs.readFileSync('static/js/export-ui.js','utf8'); -const template=fs.readFileSync('templates/index.html','utf8'); -assert.ok(template.includes('onclick="startExportDownloads()"')); -assert.ok(!template.includes('onclick="startDownload()"')); -assert.ok(template.indexOf('/static/js/playback.js')child.textContent).join('');} - set innerHTML(_){throw Error('HTML sink used');} - set outerHTML(_){throw Error('HTML sink used');} - insertAdjacentHTML(){throw Error('HTML sink used');} - setAttribute(name){assert.ok(!/^on/i.test(name));} - appendChild(child){child.parent=this;this.children.push(child);return child;} - replaceChildren(){this.children=[];this.text='';} - addEventListener(name,fn){this.listeners[name]=fn;} - remove(){if(this.parent)this.parent.children=this.parent.children.filter(child=>child!==this);} - click(){if(this.disabled)return;if(this.tagName==='A')handed.push({url:this.href,name:this.download});else this.listeners.click?.();} - } - const body=new Element('body'),list=new Element('div'),host=new Element('div'),button=new Element('button'); - const ctx=vm.createContext({console,AbortController,Blob,TextDecoder,Uint8Array,URL:{createObjectURL(blob){const url='blob:test-'+created.length;created.push({url,size:blob.size});return url;},revokeObjectURL(url){revoked.push(url);}},document:{body,createElement:tag=>new Element(tag),getElementById:id=>({'download-status':host,'download-progress-list':list,'export-start-btn':button}[id] || null)},window:{addEventListener(){}},setTimeout(fn,delay){const id=++next;timers.set(id,{fn,delay});return id;},clearTimeout(id){timers.delete(id);},fetch(url,options){requests.push({url,options});return fetcher(url,options);}}); - vm.runInContext(`let downloadCameras=${JSON.stringify(ids)};let allCameras=[{id:'front',name:''},{id:'back',name:'Back'}];let downloadStartTime=new Date('2026-11-10T00:00:00Z');let downloadEndTime=new Date('2026-11-10T00:00:02Z');`,ctx); - vm.runInContext(source,ctx); - return {ctx,timers,created,revoked,handed,requests,list,jobs:()=>vm.runInContext('exportUiBatch.jobs',ctx)}; -} -function response(status=200,data=media,extra={}){ - return {status,ok:status===200,redirected:false,headers:new Headers({'Content-Type':status===200?'video/mp4':'text/plain','X-Export-Range-Policy':'decoded-video-frame-window','X-Export-Output-Duration-Seconds':'2',...extra}),body:new ReadableStream({start(controller){if(data)controller.enqueue(data);controller.close();}})}; -} -async function ticks(){for(let i=0;i<5;i++)await new Promise(resolve=>setImmediate(resolve));} +const {environment,source,id,view,response}=require('./export-ui-fixture.cjs'); +assert.ok(!source.includes('new Blob('));assert.ok(!source.includes('createObjectURL'));assert.ok(!source.includes('arrayBuffer()'));assert.ok(!source.includes('EXPORT_UI_LIMIT')); (async()=>{ - const good=environment(()=>Promise.resolve(response()),['front','back']); - const batch=good.ctx.startExportDownloads();await ticks(); - assert.equal(good.requests.length,1);assert.equal(good.jobs()[0].state,'ready');assert.equal(good.created.length,1);assert.equal(good.handed.length,0); - assert.ok(good.list.textContent.includes(''));assert.equal(good.ctx.pwned,undefined); - vm.runInContext(`downloadStartTime=new Date('2026-11-11T00:00:00Z')`,good.ctx); - good.jobs()[0].save.click();assert.equal(good.handed.length,1);assert.equal(good.jobs()[0].state,'handed');assert.ok(good.jobs()[0].status.textContent.includes('not verified')); - good.jobs()[0].cancel.click();await ticks();assert.equal(good.revoked.length,1);assert.equal(good.requests.length,2);assert.ok(good.requests[1].url.includes('2026-11-10T00%3A00%3A00.000Z')); - good.ctx.cancelExportDownloads();await batch;assert.equal(good.revoked.length,2);assert.equal(good.jobs()[1].state,'canceled'); - for(const [status,headers] of [[409,{'X-Export-Plan-Status':'gap'}],[409,{'X-Export-Plan-Status':'overlap'}],[422,{}],[413,{}],[504,{}],[401,{}]]){ - const raw=new TextEncoder().encode('server message');const env=environment(()=>Promise.resolve(response(status,raw,headers))); - await env.ctx.startExportDownloads();assert.equal(env.jobs()[0].state,'failed');assert.equal(env.created.length,0);assert.equal(env.handed.length,0);assert.ok(env.list.textContent.includes(''));assert.equal(env.ctx.pwned,undefined); - } - for(const value of [response(200,media,{'Content-Length':String(64*1024*1024+1)}),response(200,media,{'Content-Length':'13'}),response(200,media,{'Content-Type':'text/html'}),response(200,Uint8Array.from([1,2,3])),response(200,media,{'X-Export-Range-Policy':'unknown'})]){ - const env=environment(()=>Promise.resolve(value));await env.ctx.startExportDownloads();assert.equal(env.jobs()[0].state,'failed');assert.equal(env.created.length,0);assert.equal(env.handed.length,0); + let creates=0; + const good=environment((url,options)=>{ + if(url==='/api/exports/create')return Promise.resolve(response(202,view(++creates===1?'a':'b','queued'))); + if(url.startsWith('/api/exports/status'))return Promise.resolve(response(200,view(url.endsWith(id('a'))?'a':'b'))); + if(url==='/api/exports/cancel')return Promise.resolve(response(204)); + throw Error('Video must not be fetched by JavaScript: '+url); + },['front','back']); + const batch=good.run('startExportDownloads()');await good.flush(); + assert.equal(good.requests.length,1);assert.equal(good.jobs()[0].state,'queued');assert.equal(good.requests[0].options.method,'POST'); + assert.equal(good.requests[0].options.credentials,'same-origin');assert.equal(JSON.parse(good.requests[0].options.body).camera_id,'front'); + good.run("downloadStartTime=new Date('2026-12-01T00:00:00Z')"); + await good.fire(1500);assert.equal(good.jobs()[0].state,'ready');assert.ok(good.jobs()[0].status.textContent.includes('5120.0 MiB'));assert.equal(good.downloads.length,0); + assert.ok(good.find('download-progress-list').textContent.includes(''));assert.equal(good.context.pwned,undefined); + good.jobs()[0].save.click();assert.deepEqual(good.downloads,['/api/exports/download?id='+id('a')]);assert.equal(good.jobs()[0].state,'handed');assert.ok(good.jobs()[0].status.textContent.includes('not verified')); + await good.run('exportUiCancel(exportUiBatch.jobs[0])');await good.flush();assert.equal(good.jobs()[0].state,'released');assert.equal(creates,2); + assert.equal(JSON.parse(good.requests.findLast(r=>r.url==='/api/exports/create').options.body).start_time,'2026-11-21T00:00:00.000Z'); + await good.fire(1500);await good.run('cancelExportDownloads()');await batch;assert.equal(good.jobs()[1].state,'canceled');assert.equal(good.timers.size,0); + for(const [status,header,word] of [[401,{},'Authentication'],[503,{},'NVR_EXPORT_SCRATCH_DIR'],[429,{},'capacity'],[409,{'X-Export-Plan-Status':'gap'},'full interval'],[422,{},'profile']]){ + const env=environment(()=>Promise.resolve(response(status,'test',header))); + await env.run('startExportDownloads()');assert.equal(env.jobs()[0].state,'failed');assert.ok(env.jobs()[0].status.textContent.includes(word));assert.equal(env.downloads.length,0);assert.equal(env.requests.length,1);assert.equal(env.context.pwned,undefined); } - const canceled=environment(()=>new Promise(()=>{}),['front','back']);const pending=canceled.ctx.startExportDownloads();await ticks();canceled.ctx.cancelExportDownloads();await pending; - assert.equal(canceled.requests.length,1);assert.ok(canceled.requests[0].options.signal.aborted);assert.equal(canceled.jobs()[0].state,'canceled');assert.equal(canceled.jobs()[1].state,'canceled'); - const stalled=environment(()=>new Promise(()=>{}));const timed=stalled.ctx.startExportDownloads();await ticks(); - [...stalled.timers.values()].find(timer=>timer.delay===240000).fn();await timed;assert.equal(stalled.jobs()[0].state,'failed');assert.ok(stalled.jobs()[0].status.textContent.includes('deadline')); - let readerCanceled=false; - const streaming=environment(()=>Promise.resolve({...response(),body:{getReader(){return {read(){return new Promise(()=>{});},cancel(){readerCanceled=true;return Promise.resolve();},releaseLock(){}}}}})); - const receiving=streaming.ctx.startExportDownloads();await ticks();streaming.ctx.cancelExportDownloads();await receiving;assert.ok(readerCanceled);assert.equal(streaming.created.length,0); - const expired=environment(()=>Promise.resolve(response()));const expires=expired.ctx.startExportDownloads();await ticks();[...expired.timers.values()].find(timer=>timer.delay===120000).fn();await expires; - assert.equal(expired.jobs()[0].state,'expired');assert.equal(expired.revoked.length,1);assert.equal(expired.handed.length,0); - console.log('Export UI status, limits, text-only rendering and cancellation regressions passed'); + let finishCreate; + const late=environment((url)=>url==='/api/exports/create'?new Promise(resolve=>{finishCreate=resolve;}):Promise.resolve(response(204))); + const lateBatch=late.run('startExportDownloads()');await late.flush();await late.run('cancelExportDownloads()');assert.equal(late.jobs()[0].state,'canceling');finishCreate(response(202,view('c','queued')));await lateBatch; + assert.equal(late.jobs()[0].state,'canceled');assert.equal(late.requests[1].url,'/api/exports/cancel');assert.equal(JSON.parse(late.requests[1].options.body).id,id('c')); + const expired=environment(()=>Promise.resolve(response(202,view('d','ready',{expires_at:new Date(Date.now()-1000).toISOString()}))));await expired.run('startExportDownloads()');assert.equal(expired.jobs()[0].state,'expired');assert.equal(expired.downloads.length,0); + const malicious=environment(()=>Promise.resolve(response(202,view('e','ready',{download_url:'https://evil.invalid/'}))));await malicious.run('startExportDownloads()');assert.equal(malicious.jobs()[0].state,'failed');assert.equal(malicious.downloads.length,0); + const huge=environment(()=>Promise.resolve(response(202,'x'.repeat(8193))));await huge.run('startExportDownloads()');assert.equal(huge.jobs()[0].state,'failed');assert.equal(huge.timers.size,0); + let releases=0; + const releaseFail=environment(url=>Promise.resolve(url==='/api/exports/create'?response(202,view('f')):++releases===1?response(503,'busy'):response(204))); + const failBatch=releaseFail.run('startExportDownloads()');await releaseFail.flush();await releaseFail.run('cancelExportDownloads()');await failBatch; + assert.equal(releaseFail.jobs()[0].state,'release-failed');assert.ok(releaseFail.jobs()[0].status.textContent.includes('could not be confirmed'));assert.equal(releaseFail.jobs()[0].cancel.disabled,false); + await releaseFail.run('cancelExportDownloads()');assert.equal(releaseFail.jobs()[0].state,'canceled');assert.equal(releases,2); + console.log('PASS prepared-job UI: 5 GiB metadata, no media buffering, bounded JSON, auth/configuration errors, safe text, immutable ranges, cancel race, release confirmation and expiry'); })().catch(error=>{console.error(error);process.exitCode=1;}); From 2ca17a4dbe2cc81caff1b2f714a02dfa96f5c8d7 Mon Sep 17 00:00:00 2001 From: mouldybread <38598858+mouldybread@users.noreply.github.com> Date: Sat, 21 Nov 2026 11:43:03 +0800 Subject: [PATCH 4/8] Add audio-preserving prepared-media rendering primitives --- docs/EXPORT_RENDERING.md | 23 +++++ export_media.go | 175 +++++++++++++++++++++++++++++++++++++++ export_media_test.go | 94 +++++++++++++++++++++ 3 files changed, 292 insertions(+) create mode 100644 docs/EXPORT_RENDERING.md create mode 100644 export_media.go create mode 100644 export_media_test.go diff --git a/docs/EXPORT_RENDERING.md b/docs/EXPORT_RENDERING.md new file mode 100644 index 0000000..06763d5 --- /dev/null +++ b/docs/EXPORT_RENDERING.md @@ -0,0 +1,23 @@ +# Prepared export rendering backend: implementation status + +This is a staged backend implementation, not a completed integration. export_media.go adds copy/accurate rendering primitives and export_media_test.go adds media integration tests. The HTTP planner still calls the legacy renderer and the browser does not yet send a mode choice. These paths are NOT yet selectable through the app. Do not release the branch as complete. + +## Copy + +A single complete source is copied byte for byte into bounded prepared-output storage. It is not yet the planned zero-copy original-file HTTP delivery. Several complete sources are remuxed with explicit audio/video mapping, stream copy, and matching codec configuration/extradata checks. No nominal/average frame-rate equality test is used. Packet conversion is disabled for compatible concatenation, preserving compressed payloads. Additional tracks or incompatible configurations are refused, never silently selected/dropped or transcoded. + +Subsections in copy mode currently return a boundary error. Keyframe probing, actual-range preview and explicit consent are still required before adjusted-boundary copy is enabled. No intervals are widened automatically. + +## Accurate + +Audio/video windows use the same video timestamp origin and cut reference, preserving relative audio offset rather than independently resetting tracks. AAC is retained by re-encoding in accurate mode; its sample rate/channel count are validated. Leading/trailing missing audio in an accepted input window is represented as silence to maintain the window duration, without stretching audio. Mixed audio presence or mismatched audio configurations across parts are currently refused. + +8-bit yuv420p and full-range yuvj420p are supported within existing dimension bounds. Accurate full-range input is explicitly converted to limited-range yuv420p output. Video is re-encoded with libx264 CRF 18; accurate exports are not byte-identical or lossless. Copy does not perform this conversion. Rotation and higher-bit-depth profiles remain unsupported. Frame-rate strings are not used as a proxy for valid timestamps. Video is frame-granular, not exact camera wall-clock timing. + +Fragmented MP4 accurate output uses delayed moov/edit-list handling with negative timestamps enabled for AAC encoder priming; this avoids treating encoder delay as source timing. Output must be a fresh empty regular file. Sources are pinned and revalidated, output writes are bounded, output tracks/durations are probed, and all subprocesses are joined before return. Checks do not by themselves certify every frame, colour value or audio sample. + +## Evidence and remaining work + +Local FFmpeg command experiments (not execution of the Go implementation) produced decodable H.264/AAC single/multi-part accurate output, preserved a deliberately delayed audio onset at approximately 250 ms, and preserved compressed video/AAC packet hashes when copying complete identical-profile clips. Go integration tests cover the camera metadata profile, original-byte copying, copied-packet hashes, multi-part accurate output, external A/V decode, delayed audio onset, source digests, cancellation and output limits. Go is unavailable in the assistant environment, so these Go tests have NOT been executed or compilation-verified. + +Still required: HTTP job renderer wiring; API and UI explicit mode selection; keyframe-adjusted copy planning/consent; original descriptor delivery; real-camera and fractional/B-frame boundary timing validation; colour-range validation; large-file and cancellation-under-load tests; migration and execution of the real Chrome acceptance suite. diff --git a/export_media.go b/export_media.go new file mode 100644 index 0000000..a9f1908 --- /dev/null +++ b/export_media.go @@ -0,0 +1,175 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "os/exec" + "reflect" + "strconv" + "strings" + "time" +) + +var errPreparedMediaProfile=errors.New("unsupported or incompatible export tracks") +var errPreparedCopyBoundary=errors.New("stream copy requires complete recording boundaries; use accurate mode or explicitly choose a different interval") + +type preparedMediaStream struct { + Type string `json:"codec_type"` + Codec string `json:"codec_name"` + Tag string `json:"codec_tag_string"` + Profile string `json:"profile"` + Width int `json:"width"` + Height int `json:"height"` + Pixel string `json:"pix_fmt"` + Range string `json:"color_range"` + SampleRate string `json:"sample_rate"` + Channels int `json:"channels"` + Layout string `json:"channel_layout"` + TimeBase string `json:"time_base"` + ExtraHash string `json:"extradata_hash"` + Start string `json:"start_time"` + Duration string `json:"duration"` + Tags struct {Rotate string `json:"rotate"`} `json:"tags"` + Side []struct {Rotation float64 `json:"rotation"`} `json:"side_data_list"` +} +type preparedMediaProfile struct {Streams []preparedMediaStream `json:"streams"`} +func preparedMediaSeconds(value string)(float64,error){ + n,err:=strconv.ParseFloat(value,64) + if err!=nil || math.IsNaN(n) || math.IsInf(n,0) || math.Abs(n)>86400*366{return 0,errPreparedMediaProfile} + return n,nil +} +func decodePreparedMediaProfile(raw []byte)(preparedMediaProfile,error){ + var profile preparedMediaProfile + if len(raw)>65536 || json.Unmarshal(raw,&profile)!=nil{return profile,errPreparedMediaProfile} + if len(profile.Streams)<1 || len(profile.Streams)>2{return profile,errPreparedMediaProfile} + video:=profile.Streams[0] + if video.Type!="video" || video.Width<2 || video.Height<2 || video.Width%2!=0 || video.Height%2!=0 || video.Width>4096 || video.Height>4096 || int64(video.Width)*int64(video.Height)>3840*2160{return profile,errPreparedMediaProfile} + if video.Pixel!="yuv420p" && video.Pixel!="yuvj420p"{return profile,errPreparedMediaProfile} + if video.Range!="" && video.Range!="unknown" && video.Range!="tv" && video.Range!="pc"{return profile,errPreparedMediaProfile} + for _,stream:=range profile.Streams{ + if stream.Codec==""{return profile,errPreparedMediaProfile} + if stream.Tags.Rotate!=""{angle,err:=preparedMediaSeconds(stream.Tags.Rotate);if err!=nil || angle!=0{return profile,errPreparedMediaProfile}} + for _,side:=range stream.Side{if side.Rotation!=0{return profile,errPreparedMediaProfile}} + } + if len(profile.Streams)==2{ + audio:=profile.Streams[1];rate,err:=strconv.Atoi(audio.SampleRate) + if audio.Type!="audio" || audio.Codec!="aac" || err!=nil || rate<8000 || rate>96000 || audio.Channels<1 || audio.Channels>8{return profile,errPreparedMediaProfile} + } + return profile,nil +} +func inspectPreparedMediaProfile(parent context.Context,file *os.File)(preparedMediaProfile,error){ + ctx,cancel:=context.WithTimeout(parent,10*time.Second);defer cancel() + if err:=ctx.Err();err!=nil{return preparedMediaProfile{},err} + cmd:=exec.CommandContext(ctx,"ffprobe","-v","error","-protocol_whitelist","file,pipe","-format_whitelist","mov","-f","mov","-i","/proc/self/fd/3","-show_data_hash","sha256","-show_entries","stream=codec_type,codec_name,codec_tag_string,profile,width,height,pix_fmt,color_range,sample_rate,channels,channel_layout,time_base,extradata_hash,start_time,duration:stream_tags=rotate:stream_side_data=rotation","-of","json") + cmd.ExtraFiles=[]*os.File{file};cmd.WaitDelay=2*time.Second + var output limitedProbeOutput;cmd.Stdout=&output;cmd.Stderr=io.Discard + if err:=cmd.Run();err!=nil{if ctx.Err()!=nil{return preparedMediaProfile{},ctx.Err()};return preparedMediaProfile{},errPreparedMediaProfile} + return decodePreparedMediaProfile(output.Bytes()) +} + +// Copy compatibility compares the actual codec configuration, not nominal or +// average frame-rate strings. Differing track counts/configurations are refused +// instead of selecting one audio track or falling back to a silent conversion. +func compatiblePreparedCopy(a,b preparedMediaProfile)bool{ + if len(a.Streams)!=len(b.Streams){return false} + for i,left:=range a.Streams{ + right:=b.Streams[i] + left.Start="";left.Duration="";left.Tags.Rotate="";left.Side=nil + right.Start="";right.Duration="";right.Tags.Rotate="";right.Side=nil + if left.ExtraHash=="" || right.ExtraHash=="" || !reflect.DeepEqual(left,right){return false} + } + return true +} + +func preparedAccurateArguments(profiles []preparedMediaProfile,plan exportPlan)([]string,error){ + if len(profiles)!=len(plan.Parts) || len(profiles)==0{return nil,errExportPlanInvalid} + args:=[]string{"-hide_banner","-loglevel","error","-nostdin","-copyts","-filter_complex_threads","1"} + audio:=len(profiles[0].Streams)==2 + first:=profiles[0].Streams[0] + var filters,labels []string + for index,part:=range plan.Parts{ + profile:=profiles[index];video:=profile.Streams[0] + if video.Width!=first.Width || video.Height!=first.Height || (len(profile.Streams)==2)!=audio{return nil,errPreparedMediaProfile} + if audio && (profile.Streams[1].SampleRate!=profiles[0].Streams[1].SampleRate || profile.Streams[1].Channels!=profiles[0].Streams[1].Channels || profile.Streams[1].Layout!=profiles[0].Streams[1].Layout){return nil,errPreparedMediaProfile} + origin,err:=preparedMediaSeconds(video.Start);if err!=nil{return nil,err} + if audio{if _,err:=preparedMediaSeconds(profile.Streams[1].Start);err!=nil{return nil,err}} + args=append(args,"-threads","2","-noautorotate","-protocol_whitelist","file,pipe","-format_whitelist","mov","-f","mov","-i",fmt.Sprintf("/proc/self/fd/%d",index+4)) + span:=part.End.Sub(part.Start) + if span<=0{return nil,errExportPlanInvalid} + rangeFilter:="format=yuv420p" + if video.Pixel=="yuvj420p" || video.Range=="pc"{rangeFilter="scale=in_range=pc:out_range=tv,format=yuv420p"} + // Both tracks use the VIDEO origin and the same requested cut origin. Do not + // independently reset audio/video to their first retained packet/frame. + filters=append(filters,fmt.Sprintf("[%d:v:0]setpts=PTS-(%.9f)/TB,trim=start=%s:end=%s,setpts=PTS-(%s)/TB,%s[v%d]",index,origin,trimSeconds(part.In),trimSeconds(part.Out),trimSeconds(part.In),rangeFilter,index)) + labels=append(labels,fmt.Sprintf("[v%d]",index)) + if audio{ + rate:=profile.Streams[1].SampleRate + filters=append(filters,fmt.Sprintf("[%d:a:0]asetpts=PTS-(%.9f)/TB,atrim=start=%s:end=%s,asetpts=PTS-(%s)/TB,aresample=%s:async=0:first_pts=0,apad=whole_dur=%s,atrim=duration=%s[a%d]",index,origin,trimSeconds(part.In),trimSeconds(part.Out),trimSeconds(part.In),rate,trimSeconds(span),trimSeconds(span),index)) + labels=append(labels,fmt.Sprintf("[a%d]",index)) + } + } + if audio{filters=append(filters,fmt.Sprintf("%sconcat=n=%d:v=1:a=1[outv][outa]",strings.Join(labels,""),len(plan.Parts)))}else{filters=append(filters,fmt.Sprintf("%sconcat=n=%d:v=1:a=0[outv]",strings.Join(labels,""),len(plan.Parts)))} + args=append(args,"-filter_complex",strings.Join(filters,";"),"-map","[outv]") + if audio{args=append(args,"-map","[outa]","-c:a","aac","-b:a","128k")}else{args=append(args,"-an")} + args=append(args,"-sn","-dn","-c:v","libx264","-bf","0","-crf","18","-preset","veryfast","-pix_fmt","yuv420p","-color_range","tv","-threads","2","-fps_mode","vfr","-avoid_negative_ts","disabled","-use_editlist","1","-f","mp4","-movflags","frag_keyframe+delay_moov","pipe:1") + return args,nil +} + +// Backend rendering primitive, not yet selected by the HTTP/UI planner. mode +// is exactly "copy" or "accurate"; there is no automatic conversion fallback. +// Output must be a new empty file, never an existing recording. The caller owns +// its lifetime and must not expose it until this function validates success. +func renderPreparedMedia(ctx context.Context,pins *pinnedExport,plan exportPlan,mode string,output *os.File,limit int64)error{ + if err:=ctx.Err();err!=nil{return err} + if mode!="copy" && mode!="accurate"{return errExportPlanInvalid} + canonical,err:=buildExportPlan(plan.clips(),timelineWindow{Start:plan.Start,End:plan.End});if err!=nil{return err} + if pins==nil || output==nil || len(pins.files)!=len(canonical.Parts) || len(pins.identities)!=len(pins.files) || limit<1{return errExportPlanInvalid} + info,err:=output.Stat();if err!=nil || !info.Mode().IsRegular() || info.Size()!=0{return errors.New("export output must be a new empty regular file")} + if _,err:=output.Seek(0,io.SeekStart);err!=nil{return err} + if mode=="copy" && canonical.NeedsTrim{return errPreparedCopyBoundary} + if err:=pins.validate(ctx);err!=nil{return err} + profiles:=make([]preparedMediaProfile,len(pins.files)) + for i,file:=range pins.files{ + if !sameMediaFile(canonical.Parts[i].Clip.identity,pins.identities[i]){return errRecordingChanged} + profiles[i],err=inspectPreparedMediaProfile(ctx,file);if err!=nil{return err} + } + writer:=&boundedExportJobWriter{ctx:ctx,file:output,limit:limit} + if mode=="copy" && len(pins.files)==1{ + _,err=io.Copy(writer,io.NewSectionReader(pins.files[0],0,pins.identities[0].Size())) + }else{ + var args []string + if mode=="copy"{ + for i:=1;i250*time.Millisecond{return errors.New("export video duration differs from selected interval")} + if len(produced.Streams)==2{ + seconds,err:=preparedMediaSeconds(produced.Streams[1].Duration);if err!=nil{return err} + if math.Abs(seconds-canonical.End.Sub(canonical.Start).Seconds())>0.25{return errors.New("export audio duration differs from selected interval")} + } + if _,err:=output.Seek(0,io.SeekStart);err!=nil{return err} + return ctx.Err() +} diff --git a/export_media_test.go b/export_media_test.go new file mode 100644 index 0000000..303c99e --- /dev/null +++ b/export_media_test.go @@ -0,0 +1,94 @@ +package main + +import ( + "context" + "crypto/sha256" + "encoding/binary" + "encoding/json" + "errors" + "io" + "math" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" +) + +func TestPreparedMediaCameraProfile(t *testing.T){ + raw:=`{"streams":[{"codec_name":"h264","codec_type":"video","width":2688,"height":1520,"pix_fmt":"yuvj420p","r_frame_rate":"20/1","avg_frame_rate":"1200000/60001","start_time":"0.000000"},{"codec_name":"aac","codec_type":"audio","sample_rate":"16000","channels":1,"start_time":"0.000000"}]}` + profile,err:=decodePreparedMediaProfile([]byte(raw));if err!=nil || len(profile.Streams)!=2{t.Fatalf("camera profile refused: %+v %v",profile,err)} + for _,invalid:=range []string{strings.Replace(raw,"yuvj420p","yuv420p10le",1),strings.Replace(raw,`"height":1520`,`"height":1521`,1),strings.Replace(raw,`"channels":1`,`"channels":0`,1),strings.Replace(raw,`"codec_name":"aac"`,`"codec_name":"unknown"`,1)}{ + if _,err:=decodePreparedMediaProfile([]byte(invalid));err==nil{t.Fatal("unsupported input accepted",invalid)} + } +} + +func makePreparedMediaSource(t *testing.T,ctx context.Context,delayed bool)string{ + t.Helper();file:=filepath.Join(t.TempDir(),"source.mp4") + args:=[]string{"-nostdin","-v","error","-f","lavfi","-i","testsrc2=size=128x96:rate=20"} + if delayed{args=append(args,"-itsoffset","0.25")} + args=append(args,"-f","lavfi","-i","sine=frequency=440:sample_rate=16000","-t","2","-c:v","libx264","-threads","1","-g","20","-pix_fmt","yuvj420p","-color_range","pc","-c:a","aac","-ac","1","-y",file) + if raw,err:=exec.CommandContext(ctx,"ffmpeg",args...).CombinedOutput();err!=nil{t.Fatalf("fixture generation: %v %s",err,raw)} + return file +} +func preparedMediaTestPlan(t *testing.T,ctx context.Context,source string,count int,start,end time.Duration)(exportPlan,*pinnedExport){ + t.Helper();file,err:=os.Open(source);if err!=nil{t.Fatal(err)};defer file.Close() + info,err:=file.Stat();if err!=nil{t.Fatal(err)};duration,err:=probeVideoDuration(ctx,file);if err!=nil{t.Fatal(err)} + base:=time.Date(2026,11,21,0,0,0,0,time.UTC);clips:=make([]recordingInterval,count) + for i:=range clips{begin:=base.Add(time.Duration(i)*duration);clips[i]=recordingInterval{CameraID:"camera",Path:source,Start:begin,End:begin.Add(duration),Size:info.Size(),identity:info}} + plan,err:=buildExportPlan(clips,timelineWindow{Start:base.Add(start),End:base.Add(end)});if err!=nil{t.Fatal(err)} + pool:=newExportInputPool(1,128);pool.tempDir=t.TempDir();pins,err:=pool.prepare(ctx,plan.clips());if err!=nil{t.Fatal(err)};t.Cleanup(func(){if err:=pins.Close();err!=nil{t.Error(err)}}) + return plan,pins +} +func preparedMediaOutput(t *testing.T)*os.File{ + t.Helper();file,err:=os.CreateTemp(t.TempDir(),"export-*.mp4");if err!=nil{t.Fatal(err)};t.Cleanup(func(){file.Close()});return file +} +func preparedMediaDigest(t *testing.T,path string)[32]byte{t.Helper();raw,err:=os.ReadFile(path);if err!=nil{t.Fatal(err)};return sha256.Sum256(raw)} +func decodePreparedMedia(t *testing.T,ctx context.Context,path string){ + t.Helper();raw,err:=exec.CommandContext(ctx,"ffmpeg","-nostdin","-v","error","-xerror","-i",path,"-map","0:v:0","-map","0:a:0","-f","null","-").CombinedOutput();if err!=nil{t.Fatalf("external A/V decode: %v %s",err,raw)} +} +func preparedPacketHashes(t *testing.T,ctx context.Context,path string)[]string{ + t.Helper();raw,err:=exec.CommandContext(ctx,"ffprobe","-v","error","-show_packets","-show_data_hash","sha256","-show_entries","packet=stream_index,data_hash","-of","json",path).Output();if err!=nil{t.Fatal(err)} + var data struct{Packets []struct{Index int `json:"stream_index"`;Hash string `json:"data_hash"`} `json:"packets"`};if err:=json.Unmarshal(raw,&data);err!=nil{t.Fatal(err)} + hashes:=make([]string,2) + for _,packet:=range data.Packets{if packet.Index<0 || packet.Index>1 || packet.Hash==""{t.Fatal("unexpected packet")};hashes[packet.Index]+=packet.Hash+"\n"} + return hashes +} + +func TestPreparedMediaCopyAndAccurateAcrossClips(t *testing.T){ + requireFFmpeg(t);ctx,cancel:=context.WithTimeout(context.Background(),90*time.Second);defer cancel() + source:=makePreparedMediaSource(t,ctx,false);before:=preparedMediaDigest(t,source) + plan,pins:=preparedMediaTestPlan(t,ctx,source,1,0,2*time.Second) + original:=preparedMediaOutput(t) + if err:=renderPreparedMedia(ctx,pins,plan,"copy",original,16<<20);err!=nil{t.Fatal(err)} + if preparedMediaDigest(t,original.Name())!=before{t.Fatal("single complete copy changed original bytes")} + plan,pins=preparedMediaTestPlan(t,ctx,source,2,0,4*time.Second) + copied:=preparedMediaOutput(t);if err:=renderPreparedMedia(ctx,pins,plan,"copy",copied,16<<20);err!=nil{t.Fatal(err)} + expected:=preparedPacketHashes(t,ctx,source);actual:=preparedPacketHashes(t,ctx,copied.Name()) + if actual[0]!=expected[0]+expected[0] || actual[1]!=expected[1]+expected[1]{t.Fatal("stream copy changed/dropped compressed video or audio packets")} + decodePreparedMedia(t,ctx,copied.Name()) + plan,pins=preparedMediaTestPlan(t,ctx,source,2,500*time.Millisecond,3500*time.Millisecond) + refused:=preparedMediaOutput(t);if err:=renderPreparedMedia(ctx,pins,plan,"copy",refused,16<<20);!errors.Is(err,errPreparedCopyBoundary){t.Fatalf("partial copy silently changed boundaries: %v",err)} + accurate:=preparedMediaOutput(t);if err:=renderPreparedMedia(ctx,pins,plan,"accurate",accurate,16<<20);err!=nil{t.Fatal(err)} + decodePreparedMedia(t,ctx,accurate.Name());profile,err:=inspectPreparedMediaProfile(ctx,accurate);if err!=nil{t.Fatal(err)} + if len(profile.Streams)!=2 || profile.Streams[0].Pixel!="yuv420p" || profile.Streams[1].Codec!="aac" || profile.Streams[1].SampleRate!="16000" || profile.Streams[1].Channels!=1{t.Fatalf("export track/profile mismatch: %+v",profile)} + if preparedMediaDigest(t,source)!=before{t.Fatal("export modified source")} +} + +func TestPreparedAccurateAudioOffsetAndLimits(t *testing.T){ + requireFFmpeg(t);ctx,cancel:=context.WithTimeout(context.Background(),60*time.Second);defer cancel() + source:=makePreparedMediaSource(t,ctx,true);before:=preparedMediaDigest(t,source) + plan,pins:=preparedMediaTestPlan(t,ctx,source,1,0,2*time.Second) + output:=preparedMediaOutput(t);if err:=renderPreparedMedia(ctx,pins,plan,"accurate",output,16<<20);err!=nil{t.Fatal(err)} + raw,err:=exec.CommandContext(ctx,"ffmpeg","-nostdin","-v","error","-i",output.Name(),"-map","0:a:0","-ac","1","-ar","16000","-f","f32le","pipe:1").Output();if err!=nil{t.Fatal(err)} + first:=-1 + for i:=0;i+4<=len(raw);i+=4{value:=math.Float32frombits(binary.LittleEndian.Uint32(raw[i:i+4]));if math.Abs(float64(value))>0.02{first=i/4;break}} + if first<0 || math.Abs(float64(first)/16000-0.25)>0.04{t.Fatalf("audio lost its relative 250ms start offset: first sample %d",first)} + tiny:=preparedMediaOutput(t);if err:=renderPreparedMedia(ctx,pins,plan,"accurate",tiny,16);!errors.Is(err,errExportJobSize){t.Fatalf("output size not bounded: %v",err)} + stopped,stop:=context.WithCancel(ctx);stop();canceled:=preparedMediaOutput(t) + if err:=renderPreparedMedia(stopped,pins,plan,"accurate",canceled,16<<20);!errors.Is(err,context.Canceled){t.Fatal(err)} + data:=preparedMediaOutput(t);if _,err:=io.WriteString(data,"not empty");err!=nil{t.Fatal(err)} + if err:=renderPreparedMedia(ctx,pins,plan,"accurate",data,16<<20);err==nil{t.Fatal("existing nonempty output overwritten")} + if preparedMediaDigest(t,source)!=before{t.Fatal("source changed after failure/cancellation")} +} From 2109c1c635e932991a1f18c8bc9f83273e66ca95 Mon Sep 17 00:00:00 2001 From: mouldybread <38598858+mouldybread@users.noreply.github.com> Date: Sat, 21 Nov 2026 12:02:46 +0800 Subject: [PATCH 5/8] Wire export modes and require approval for keyframe copy ranges --- docs/EXPORT_API_MODES.md | 11 ++ export_http.go | 277 ++++++++++++++------------------------- export_keyframes.go | 84 ++++++++++++ export_keyframes_test.go | 38 ++++++ 4 files changed, 231 insertions(+), 179 deletions(-) create mode 100644 docs/EXPORT_API_MODES.md create mode 100644 export_keyframes.go create mode 100644 export_keyframes_test.go diff --git a/docs/EXPORT_API_MODES.md b/docs/EXPORT_API_MODES.md new file mode 100644 index 0000000..cf1a40a --- /dev/null +++ b/docs/EXPORT_API_MODES.md @@ -0,0 +1,11 @@ +# Export mode API integration (intermediate commit) + +The HTTP job planner now calls the copy/accurate renderers instead of the legacy video-only trim renderer. POST /api/exports/create accepts mode (copy by default, or accurate). It never automatically switches modes. Job output accounting is populated from the validated output file size, including direct renderer writes. + +For complete recordings, copy preserves original bytes (one file) or compatible compressed A/V packets (multiple files). Accurate explicitly re-encodes supported video and AAC audio and applies the existing renderer validation. + +For subsections, copy scans decoded keyframe timestamps and supports non-reordered video only. It rounds the video start earlier and end later to available keyframe/file boundaries, rechecks expanded coverage for gaps/overlaps, and responds with 409 JSON code copy_range_approval_required before admitting a job or reserving output storage. The response includes requested_start/end, export_start/end and audio_allowance_ms (one AAC packet). Approval must send accepted_start/end matching the freshly recomputed proposal along with the unchanged original request. Stale approval never silently approves a different video interval. Audio packet edges can extend outside the video-aligned interval by the disclosed allowance. + +Copy rendering uses pinned descriptors, explicit track mapping, bounded output, duration checks and an external decode before readiness. Reordered video, missing keyframe metadata, unsupported tracks, incompatible configurations and undecodable random-access output are refused; choose accurate explicitly. These checks are not a general proof of frame/sample completeness or calibrated wall-clock boundaries. + +This is not yet a finished application change: the corresponding UI mode/approval controls still need to be committed, and the real-browser acceptance harness still requires migration. Go compilation/tests have not run in the assistant environment. A local FFmpeg command experiment, not the Go implementation, produced a decodable non-reordered H.264/AAC keyframe-aligned subsection. Do not merge/release on that evidence alone. diff --git a/export_http.go b/export_http.go index 75735f8..7947e22 100644 --- a/export_http.go +++ b/export_http.go @@ -15,203 +15,122 @@ import ( "sync" "time" ) - -const ( - exportCreatePath="/api/exports/create" - exportCancelPath="/api/exports/cancel" - exportStatusPath="/api/exports/status" - exportDownloadPath="/api/exports/download" -) - -// Extend the existing authentication and shutdown mutation policy. The route -// handler is installed INSIDE auth.Handler, so event tokens never reach it. -func init() {mutationPaths[exportCreatePath]=true;mutationPaths[exportCancelPath]=true} - -type exportCreateRequest struct { +const(exportCreatePath="/api/exports/create";exportCancelPath="/api/exports/cancel";exportStatusPath="/api/exports/status";exportDownloadPath="/api/exports/download") +func init(){mutationPaths[exportCreatePath]=true;mutationPaths[exportCancelPath]=true} +type exportCreateRequest struct{ CameraID string `json:"camera_id"` Start string `json:"start_time"` End string `json:"end_time"` + Mode string `json:"mode,omitempty"` + AcceptedStart string `json:"accepted_start,omitempty"` + AcceptedEnd string `json:"accepted_end,omitempty"` } -type exportHTTPView struct { - exportJobView - DownloadURL string `json:"download_url,omitempty"` +type exportHTTPView struct{exportJobView;DownloadURL string `json:"download_url,omitempty"`} +type exportPlanHTTPError struct{status int;message string} +func(e *exportPlanHTTPError)Error()string{return e.message} +type exportCopyProposal struct{ + Code string `json:"code"` + RequestedStart string `json:"requested_start"` + RequestedEnd string `json:"requested_end"` + ExportStart string `json:"export_start"` + ExportEnd string `json:"export_end"` + AudioAllowanceMS int64 `json:"audio_allowance_ms"` } -type exportPlanHTTPError struct {status int;message string} -func (e *exportPlanHTTPError) Error()string{return e.message} +func(e *exportCopyProposal)Error()string{return "copy range approval required"} type exportHTTPPlanner func(context.Context,exportCreateRequest)(exportJobRenderer,error) - -type exportJobHTTP struct { - auth *authManager - store *exportJobStore - plan exportHTTPPlanner - ctx context.Context - cancel context.CancelFunc - mu sync.Mutex - closing bool - requests sync.WaitGroup - once sync.Once - done chan struct{} - closeErr error -} - -func newExportJobHTTP(auth *authManager,store *exportJobStore,plan exportHTTPPlanner)*exportJobHTTP { - ctx,cancel:=context.WithCancel(context.Background()) - return &exportJobHTTP{auth:auth,store:store,plan:plan,ctx:ctx,cancel:cancel,done:make(chan struct{})} -} - -// Opt-in scratch storage avoids silently putting GiB exports in the container -// writable layer. The operator must mount trusted private storage at this path. -func configuredExportJobHTTP(auth *authManager,n *NVR)(*exportJobHTTP,error) { - limits,err:=loadExportJobLimits(os.Getenv);if err!=nil{return nil,err} - directory:=os.Getenv("NVR_EXPORT_SCRATCH_DIR") - if directory=="" {return newExportJobHTTP(auth,nil,nil),nil} - if !filepath.IsAbs(directory){return nil,errors.New("NVR_EXPORT_SCRATCH_DIR must be absolute")} - directory=filepath.Clean(directory) +type exportJobHTTP struct{auth *authManager;store *exportJobStore;plan exportHTTPPlanner;ctx context.Context;cancel context.CancelFunc;mu sync.Mutex;closing bool;requests sync.WaitGroup;once sync.Once;done chan struct{};closeErr error} +func newExportJobHTTP(a *authManager,store *exportJobStore,plan exportHTTPPlanner)*exportJobHTTP{ctx,cancel:=context.WithCancel(context.Background());return &exportJobHTTP{auth:a,store:store,plan:plan,ctx:ctx,cancel:cancel,done:make(chan struct{})}} +func configuredExportJobHTTP(a *authManager,n *NVR)(*exportJobHTTP,error){ + limits,err:=loadExportJobLimits(os.Getenv);if err!=nil{return nil,err};directory:=os.Getenv("NVR_EXPORT_SCRATCH_DIR");if directory==""{return newExportJobHTTP(a,nil,nil),nil} + if !filepath.IsAbs(directory){return nil,errors.New("NVR_EXPORT_SCRATCH_DIR must be absolute")};directory=filepath.Clean(directory) resolved,err:=filepath.EvalSymlinks(directory);if err!=nil{return nil,errors.New("export scratch directory unavailable")} if resolved!=directory || directory=="/recordings" || strings.HasPrefix(directory,"/recordings/"){return nil,errors.New("export scratch must be outside recordings and not symlinked")} info,err:=os.Stat(directory);if err!=nil || !info.IsDir(){return nil,errors.New("export scratch must be an existing directory")} - n.mu.RLock() - overlap:=false - for _,cam:=range n.cameras {if cam==nil{continue};path,err:=filepath.Abs(cam.RecordDir);if err!=nil{overlap=true;break};path=filepath.Clean(path);if path==directory || strings.HasPrefix(directory,path+string(os.PathSeparator)) || strings.HasPrefix(path,directory+string(os.PathSeparator)){overlap=true;break}} - n.mu.RUnlock() - if overlap{return nil,errors.New("export scratch overlaps a camera directory")} - store,err:=newExportJobStore(context.Background(),directory,limits);if err!=nil{return nil,err} - service:=newExportJobHTTP(auth,store,nil) - service.plan=func(ctx context.Context,req exportCreateRequest)(exportJobRenderer,error){return planExistingExportJob(ctx,n,store.directory,req)} - return service,nil + n.mu.RLock();overlap:=false + for _,cam:=range n.cameras{if cam==nil{continue};path,err:=filepath.Abs(cam.RecordDir);if err!=nil{overlap=true;break};path=filepath.Clean(path);if path==directory || strings.HasPrefix(directory,path+string(os.PathSeparator)) || strings.HasPrefix(path,directory+string(os.PathSeparator)){overlap=true;break}} + n.mu.RUnlock();if overlap{return nil,errors.New("export scratch overlaps a camera directory")} + store,err:=newExportJobStore(context.Background(),directory,limits);if err!=nil{return nil,err};service:=newExportJobHTTP(a,store,nil) + service.plan=func(ctx context.Context,req exportCreateRequest)(exportJobRenderer,error){return planExistingExportJob(ctx,n,store.directory,req)};return service,nil } - -func exportJobRoute(path string)bool {return path==exportCreatePath || path==exportCancelPath || path==exportStatusPath || path==exportDownloadPath} - -// The identity comes only from a validated session, not a header, query or -// JSON owner. A new session or changed credentials cannot inherit old jobs. -func (s *exportJobHTTP) owner(r *http.Request)(string,error) { - if s.auth==nil{return "",errExportJobMissing} - cfg,stamp,err:=readAuthConfig(s.auth.path);if err!=nil{return "",err} - user:=s.auth.sessions.GetString(r.Context(),"user") - version:=s.auth.sessions.GetString(r.Context(),"credential_version") - token:=s.auth.sessions.Token(r.Context()) - if user!=cfg.Username || version!=stamp || token==""{return "",errExportJobMissing} - digest:=sha256.Sum256([]byte(user+"\x00"+version+"\x00"+token)) - return hex.EncodeToString(digest[:]),nil +func exportJobRoute(path string)bool{return path==exportCreatePath || path==exportCancelPath || path==exportStatusPath || path==exportDownloadPath} +func(s *exportJobHTTP)owner(r *http.Request)(string,error){ + if s.auth==nil{return "",errExportJobMissing};cfg,stamp,err:=readAuthConfig(s.auth.path);if err!=nil{return "",err} + user:=s.auth.sessions.GetString(r.Context(),"user");version:=s.auth.sessions.GetString(r.Context(),"credential_version");token:=s.auth.sessions.Token(r.Context()) + if user!=cfg.Username || version!=stamp || token==""{return "",errExportJobMissing};digest:=sha256.Sum256([]byte(user+"\x00"+version+"\x00"+token));return hex.EncodeToString(digest[:]),nil } - -func (s *exportJobHTTP) Handler(next http.Handler)http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter,r *http.Request){ - if !exportJobRoute(r.URL.Path){next.ServeHTTP(w,r);return} - w.Header().Set("Cache-Control","private, no-store");w.Header().Set("X-Content-Type-Options","nosniff");w.Header().Set("Referrer-Policy","no-referrer") - owner,err:=s.owner(r);if err!=nil{http.Error(w,"Unauthorized",401);return} - allowed:="GET, HEAD" - if r.URL.Path==exportCreatePath || r.URL.Path==exportCancelPath {allowed="POST"} - if (allowed=="POST" && r.Method!=http.MethodPost) || (allowed!="POST" && r.Method!=http.MethodGet && r.Method!=http.MethodHead){w.Header().Set("Allow",allowed);http.Error(w,"Method not allowed",405);return} - s.mu.Lock() - if s.closing || s.store==nil {s.mu.Unlock();http.Error(w,"Prepared exports unavailable; configure export scratch storage or retry after restart",503);return} - s.requests.Add(1);s.mu.Unlock();defer s.requests.Done() - ctx,cancel:=context.WithCancel(r.Context());defer cancel() - stopService:=context.AfterFunc(s.ctx,cancel);defer stopService() - r=r.WithContext(ctx) - // This also releases readers blocked behind a slow/disconnected client. - writeDone:=make(chan struct{}) - stopWrite:=context.AfterFunc(ctx,func(){_ =http.NewResponseController(w).SetWriteDeadline(time.Now());close(writeDone)}) - defer func(){if !stopWrite(){<-writeDone}}() - switch r.URL.Path { - case exportCreatePath:s.create(w,r,owner) - case exportCancelPath:s.cancelRequest(w,r,owner) - case exportStatusPath: - id,ok:=exportRequestID(w,r);if !ok{return} - view,err:=s.store.status(owner,id);if err!=nil{writeExportJobHTTPError(w,err);return} - writeExportJobView(w,r,200,view) - case exportDownloadPath: - id,ok:=exportRequestID(w,r);if !ok{return} - reader,release,err:=s.store.acquire(owner,id);if err!=nil{writeExportJobHTTPError(w,err);return};defer release() - w.Header().Set("Content-Type","video/mp4");w.Header().Set("Content-Disposition",`attachment; filename="export-`+id+`.mp4"`) - http.ServeContent(w,r,"export-"+id+".mp4",time.Time{},reader) - } - }) -} - -func exportRequestID(w http.ResponseWriter,r *http.Request)(string,bool) { - query:=r.URL.Query();values:=query["id"] - if len(query)!=1 || len(values)!=1 || !validExportJobID(values[0]){http.Error(w,"One valid export id is required",400);return "",false} - return values[0],true +func(s *exportJobHTTP)Handler(next http.Handler)http.Handler{return http.HandlerFunc(func(w http.ResponseWriter,r *http.Request){ + if !exportJobRoute(r.URL.Path){next.ServeHTTP(w,r);return} + w.Header().Set("Cache-Control","private, no-store");w.Header().Set("X-Content-Type-Options","nosniff");w.Header().Set("Referrer-Policy","no-referrer") + owner,err:=s.owner(r);if err!=nil{http.Error(w,"Unauthorized",401);return} + allowed:="GET, HEAD";if r.URL.Path==exportCreatePath || r.URL.Path==exportCancelPath{allowed="POST"} + if (allowed=="POST" && r.Method!="POST") || (allowed!="POST" && r.Method!="GET" && r.Method!="HEAD"){w.Header().Set("Allow",allowed);http.Error(w,"Method not allowed",405);return} + s.mu.Lock();if s.closing || s.store==nil{s.mu.Unlock();http.Error(w,"Prepared exports unavailable; configure export scratch storage or retry after restart",503);return};s.requests.Add(1);s.mu.Unlock();defer s.requests.Done() + ctx,cancel:=context.WithCancel(r.Context());defer cancel();stopService:=context.AfterFunc(s.ctx,cancel);defer stopService();r=r.WithContext(ctx) + writeDone:=make(chan struct{});stopWrite:=context.AfterFunc(ctx,func(){_=http.NewResponseController(w).SetWriteDeadline(time.Now());close(writeDone)});defer func(){if !stopWrite(){<-writeDone}}() + switch r.URL.Path{ + case exportCreatePath:s.create(w,r,owner) + case exportCancelPath:s.cancelRequest(w,r,owner) + case exportStatusPath:id,ok:=exportRequestID(w,r);if !ok{return};view,err:=s.store.status(owner,id);if err!=nil{writeExportJobHTTPError(w,err);return};writeExportJobView(w,r,200,view) + case exportDownloadPath:id,ok:=exportRequestID(w,r);if !ok{return};reader,release,err:=s.store.acquire(owner,id);if err!=nil{writeExportJobHTTPError(w,err);return};defer release();w.Header().Set("Content-Type","video/mp4");w.Header().Set("Content-Disposition",`attachment; filename="export-`+id+`.mp4"`);http.ServeContent(w,r,"export-"+id+".mp4",time.Time{},reader) + } +})} +func validExportJobID(id string)bool{if len(id)!=64{return false};raw,err:=hex.DecodeString(id);return err==nil && len(raw)==32} +func exportRequestID(w http.ResponseWriter,r *http.Request)(string,bool){q:=r.URL.Query();v:=q["id"];if len(q)!=1 || len(v)!=1 || !validExportJobID(v[0]){http.Error(w,"One valid export id is required",400);return "",false};return v[0],true} +func decodeExportJobJSON(w http.ResponseWriter,r *http.Request,value interface{})bool{ + if r.URL.RawQuery!=""{http.Error(w,"Query parameters not accepted",400);return false};kind,_,err:=mime.ParseMediaType(r.Header.Get("Content-Type"));if err!=nil || kind!="application/json"{http.Error(w,"Content-Type must be application/json",415);return false} + d:=json.NewDecoder(http.MaxBytesReader(w,r.Body,4096));d.DisallowUnknownFields();if err:=d.Decode(value);err!=nil{http.Error(w,"Invalid export request",400);return false};if err:=d.Decode(&struct{}{});err!=io.EOF{http.Error(w,"One export JSON object is required",400);return false};return true } -func validExportJobID(id string)bool {if len(id)!=64{return false};raw,err:=hex.DecodeString(id);return err==nil && len(raw)==32} -func decodeExportJobJSON(w http.ResponseWriter,r *http.Request,value interface{})bool { - if r.URL.RawQuery!=""{http.Error(w,"Query parameters not accepted",400);return false} - kind,_,err:=mime.ParseMediaType(r.Header.Get("Content-Type"));if err!=nil || kind!="application/json"{http.Error(w,"Content-Type must be application/json",415);return false} - decoder:=json.NewDecoder(http.MaxBytesReader(w,r.Body,4096));decoder.DisallowUnknownFields() - if err:=decoder.Decode(value);err!=nil{http.Error(w,"Invalid export request",400);return false} - if err:=decoder.Decode(&struct{}{});err!=io.EOF{http.Error(w,"One export JSON object is required",400);return false} +func writeExportJobView(w http.ResponseWriter,r *http.Request,status int,view exportJobView){body:=exportHTTPView{exportJobView:view};if view.State=="ready"{body.DownloadURL=exportDownloadPath+"?id="+view.ID};w.Header().Set("Content-Type","application/json");w.WriteHeader(status);if r.Method!="HEAD"{json.NewEncoder(w).Encode(body)}} +func writeExportJobHTTPError(w http.ResponseWriter,err error){status,message:=404,"Export unavailable, expired, or not ready";if errors.Is(err,errExportJobsBusy){status,message=429,"Export capacity exhausted; release prepared exports or wait for expiry";w.Header().Set("Retry-After","5")};if errors.Is(err,errExportJobsClosed){status,message=503,"Export service stopping"};http.Error(w,message,status)} +func validExportSelection(req exportCreateRequest)bool{ + start,e1:=time.Parse(time.RFC3339Nano,req.Start);end,e2:=time.Parse(time.RFC3339Nano,req.End) + if req.CameraID=="" || len(req.CameraID)>256 || e1!=nil || e2!=nil || !end.After(start) || (req.Mode!="copy" && req.Mode!="accurate"){return false} + if req.AcceptedStart!="" || req.AcceptedEnd!=""{a,e1:=time.Parse(time.RFC3339Nano,req.AcceptedStart);b,e2:=time.Parse(time.RFC3339Nano,req.AcceptedEnd);if req.Mode!="copy" || e1!=nil || e2!=nil || a.After(start) || b.Before(end) || !b.After(a){return false}} return true } -func writeExportJobView(w http.ResponseWriter,r *http.Request,status int,view exportJobView) { - body:=exportHTTPView{exportJobView:view} - if view.State=="ready"{body.DownloadURL=exportDownloadPath+"?id="+view.ID} - w.Header().Set("Content-Type","application/json");w.WriteHeader(status) - if r.Method!=http.MethodHead{json.NewEncoder(w).Encode(body)} -} -func writeExportJobHTTPError(w http.ResponseWriter,err error) { - status,message:=404,"Export unavailable, expired, or not ready" - if errors.Is(err,errExportJobsBusy){status,message=429,"Export capacity exhausted; release prepared exports or wait for expiry";w.Header().Set("Retry-After","5")} - if errors.Is(err,errExportJobsClosed){status,message=503,"Export service stopping"} - http.Error(w,message,status) -} -func (s *exportJobHTTP) create(w http.ResponseWriter,r *http.Request,owner string) { - var req exportCreateRequest;if !decodeExportJobJSON(w,r,&req){return} - start,e1:=time.Parse(time.RFC3339Nano,req.Start);end,e2:=time.Parse(time.RFC3339Nano,req.End) - if req.CameraID=="" || len(req.CameraID)>256 || e1!=nil || e2!=nil || !end.After(start){http.Error(w,"Valid camera_id and ordered RFC3339 start_time/end_time required",400);return} - if s.plan==nil{http.Error(w,"Export renderer unavailable",503);return} - render,err:=s.plan(r.Context(),req) - if err!=nil{ - var requestErr *exportPlanHTTPError - if errors.As(err,&requestErr){http.Error(w,requestErr.message,requestErr.status)}else if errors.Is(err,errExportPlanGap) || errors.Is(err,errExportPlanOverlap) || errors.Is(err,errExportInputLimit) || errors.Is(err,errExportPlanInvalid){writeExportPlanError(w,err)}else{writeRecordingIntervalError(w,r,err)} - return +func(s *exportJobHTTP)create(w http.ResponseWriter,r *http.Request,owner string){ + var req exportCreateRequest;if !decodeExportJobJSON(w,r,&req){return};if req.Mode==""{req.Mode="copy"};if !validExportSelection(req){http.Error(w,"Valid camera, interval, copy/accurate mode and matching approval fields required",400);return} + if s.plan==nil{http.Error(w,"Export renderer unavailable",503);return};render,err:=s.plan(r.Context(),req) + if err!=nil{var proposal *exportCopyProposal;var requestErr *exportPlanHTTPError + if errors.As(err,&proposal){w.Header().Set("Content-Type","application/json");w.WriteHeader(409);json.NewEncoder(w).Encode(proposal)}else if errors.As(err,&requestErr){http.Error(w,requestErr.message,requestErr.status)}else if errors.Is(err,errCopyReordered) || errors.Is(err,errPreparedMediaProfile){message:="Unsupported or incompatible export tracks";if errors.Is(err,errCopyReordered){message=errCopyReordered.Error()};http.Error(w,message,422)}else if errors.Is(err,errExportPlanGap) || errors.Is(err,errExportPlanOverlap) || errors.Is(err,errExportInputLimit) || errors.Is(err,errExportPlanInvalid){writeExportPlanError(w,err)}else{writeRecordingIntervalError(w,r,err)};return } - if r.Context().Err()!=nil{return} - view,err:=s.store.submit(owner,render);if err!=nil{writeExportJobHTTPError(w,err);return} - w.Header().Set("Location",exportStatusPath+"?id="+view.ID);writeExportJobView(w,r,202,view) -} -func (s *exportJobHTTP) cancelRequest(w http.ResponseWriter,r *http.Request,owner string) { - var req struct{ID string `json:"id"`};if !decodeExportJobJSON(w,r,&req){return} - if !validExportJobID(req.ID){http.Error(w,"Valid export id required",400);return} - if err:=s.store.cancelJob(owner,req.ID);err!=nil{writeExportJobHTTPError(w,err);return} - w.WriteHeader(http.StatusNoContent) + if r.Context().Err()!=nil{return};view,err:=s.store.submit(owner,render);if err!=nil{writeExportJobHTTPError(w,err);return};w.Header().Set("Location",exportStatusPath+"?id="+view.ID);writeExportJobView(w,r,202,view) } +func(s *exportJobHTTP)cancelRequest(w http.ResponseWriter,r *http.Request,owner string){var req struct{ID string `json:"id"`};if !decodeExportJobJSON(w,r,&req){return};if !validExportJobID(req.ID){http.Error(w,"Valid export id required",400);return};if err:=s.store.cancelJob(owner,req.ID);err!=nil{writeExportJobHTTPError(w,err);return};w.WriteHeader(204)} -// This integration deliberately retains the existing render restrictions. -// Stream-copy compatibility and audio-preserving accurate trim are next steps. -func planExistingExportJob(parent context.Context,n *NVR,scratch string,req exportCreateRequest)(exportJobRenderer,error) { +func copyProposal(req exportCreateRequest,requested,actual exportPlan,indices []copyFrameIndex)error{ + a,b:=actual.Start.UTC().Format(time.RFC3339Nano),actual.End.UTC().Format(time.RFC3339Nano) + approvedA,e1:=time.Parse(time.RFC3339Nano,req.AcceptedStart);approvedB,e2:=time.Parse(time.RFC3339Nano,req.AcceptedEnd) + if req.AcceptedStart!="" && e1==nil && e2==nil && approvedA.Equal(actual.Start) && approvedB.Equal(actual.End){return nil} + // Partial copy also requires approval of possible AAC packet-boundary overhang. + var allowance time.Duration;for _,index:=range indices{if index.audioAllowance>allowance{allowance=index.audioAllowance}} + return &exportCopyProposal{Code:"copy_range_approval_required",RequestedStart:requested.Start.UTC().Format(time.RFC3339Nano),RequestedEnd:requested.End.UTC().Format(time.RFC3339Nano),ExportStart:a,ExportEnd:b,AudioAllowanceMS:int64((allowance+time.Millisecond-1)/time.Millisecond)} +} +func planExistingExportJob(parent context.Context,n *NVR,_ string,req exportCreateRequest)(exportJobRenderer,error){ + if req.Mode==""{req.Mode="copy"};if !validExportSelection(req){return nil,errExportPlanInvalid} start,_:=time.Parse(time.RFC3339Nano,req.Start);end,_:=time.Parse(time.RFC3339Nano,req.End) - n.mu.RLock();cam:=n.cameras[req.CameraID];var camera timelineCamera;if cam!=nil{camera=timelineCamera{cam.ID,cam.Name,cam.RecordDir}};n.mu.RUnlock() - if cam==nil{return nil,&exportPlanHTTPError{404,"Camera not found"}} + n.mu.RLock();cam:=n.cameras[req.CameraID];var camera timelineCamera;if cam!=nil{camera=timelineCamera{cam.ID,cam.Name,cam.RecordDir}};n.mu.RUnlock();if cam==nil{return nil,&exportPlanHTTPError{404,"Camera not found"}} ctx,cancel:=context.WithTimeout(parent,recordingMetadataBudget);defer cancel() - window:=timelineWindow{Start:start.UTC(),End:end.UTC()} - clips,err:=collectRecordingIntervals(ctx,[]timelineCamera{camera},window,recordingDurations);if err!=nil{return nil,err} - plan,err:=buildExportPlan(clips,window);if err!=nil{return nil,err} - return func(ctx context.Context,w io.Writer)error { - pins,err:=exportInputPins.prepare(ctx,plan.clips());if err!=nil{return err};defer pins.Close() - if plan.NeedsTrim { - output,err:=renderTrimmedExport(ctx,pins,plan,scratch,maxTrimBytes);if err!=nil{return err};defer output.Close() - if _,err=io.Copy(w,output.file);err!=nil{return err} - }else{ - cmd,err:=pins.command(ctx);if err!=nil{return err};cmd.Stdout=w;cmd.Stderr=io.Discard - if err=cmd.Run();err!=nil{return err} - } - if err:=pins.validate(ctx);err!=nil{return err} - output,ok:=w.(*boundedExportJobWriter);if !ok{return errors.New("export output descriptor unavailable")} - verify,stop:=context.WithTimeout(ctx,5*time.Second);defer stop() - duration,err:=probeVideoDuration(verify,output.file);if err!=nil{return err} - difference:=duration-plan.End.Sub(plan.Start);if difference<0{difference=-difference} - if duration<=0 || difference>250*time.Millisecond{return errors.New("prepared export duration validation failed")} - return ctx.Err() + clips,err:=collectRecordingIntervals(ctx,[]timelineCamera{camera},timelineWindow{Start:start.UTC(),End:end.UTC()},recordingDurations);if err!=nil{return nil,err} + requested,err:=buildExportPlan(clips,timelineWindow{Start:start.UTC(),End:end.UTC()});if err!=nil{return nil,err};plan:=requested + pins,err:=exportInputPins.prepare(ctx,requested.clips());if err!=nil{return nil,err};defer pins.Close() + profiles:=make([]preparedMediaProfile,len(pins.files));for i,file:=range pins.files{profiles[i],err=inspectPreparedMediaProfile(ctx,file);if err!=nil{return nil,err}} + var indices []copyFrameIndex + if req.Mode=="copy" && requested.NeedsTrim{ + plan,indices,err=planKeyframeCopy(ctx,pins,requested);if err!=nil{return nil,err} + // Reinspect expanded coverage, so overlapping media outside the original + // selection cannot become an implicit source choice after approval. + expanded,err:=collectRecordingIntervals(ctx,[]timelineCamera{camera},timelineWindow{Start:plan.Start,End:plan.End},recordingDurations);if err!=nil{return nil,err};checked,err:=buildExportPlan(expanded,timelineWindow{Start:plan.Start,End:plan.End});if err!=nil{return nil,err} + if len(checked.Parts)!=len(plan.Parts){return nil,errRecordingChanged};for i:=range checked.Parts{if checked.Parts[i].Clip.Path!=plan.Parts[i].Clip.Path || !sameMediaFile(checked.Parts[i].Clip.identity,plan.Parts[i].Clip.identity){return nil,errRecordingChanged}} + if err:=copyProposal(req,requested,plan,indices);err!=nil{return nil,err} + }else if req.Mode=="copy" && req.AcceptedStart!=""{a,_:=time.Parse(time.RFC3339Nano,req.AcceptedStart);b,_:=time.Parse(time.RFC3339Nano,req.AcceptedEnd);if !a.Equal(plan.Start) || !b.Equal(plan.End){return nil,errExportPlanInvalid}} + if req.Mode=="copy"{for i:=1;i0{err=renderKeyframeCopy(ctx,held,plan,indices,output.file,output.limit)}else{err=renderPreparedMedia(ctx,held,plan,req.Mode,output.file,output.limit)} + if err!=nil{return err};info,err:=output.file.Stat();if err!=nil{return err};if info.Size()>output.limit{return errExportJobSize};output.written=info.Size();return ctx.Err() },nil } - -func (s *exportJobHTTP) Close(ctx context.Context)error { - s.once.Do(func(){ - s.mu.Lock();s.closing=true;s.cancel();if s.store!=nil{s.store.cancel()};s.mu.Unlock() - go func(){s.requests.Wait();if s.store!=nil{s.closeErr=s.store.Close(context.Background())};close(s.done)}() - }) - select{case <-s.done:return s.closeErr;case <-ctx.Done():return ctx.Err()} -} -func (s *exportJobHTTP) runLifecycle(ctx context.Context){<-ctx.Done();_ =s.Close(context.Background())} +func(s *exportJobHTTP)Close(ctx context.Context)error{s.once.Do(func(){s.mu.Lock();s.closing=true;s.cancel();if s.store!=nil{s.store.cancel()};s.mu.Unlock();go func(){s.requests.Wait();if s.store!=nil{s.closeErr=s.store.Close(context.Background())};close(s.done)}()});select{case <-s.done:return s.closeErr;case <-ctx.Done():return ctx.Err()}} +func(s *exportJobHTTP)runLifecycle(ctx context.Context){<-ctx.Done();_=s.Close(context.Background())} diff --git a/export_keyframes.go b/export_keyframes.go new file mode 100644 index 0000000..f9dfe0c --- /dev/null +++ b/export_keyframes.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "os" + "os/exec" + "sort" + "time" +) + +// Subsection stream copy is deliberately limited to non-reordered video. Open +// or undecodable GOPs are rejected by full output decode before publication. +var errCopyReordered=errors.New("subsection copy cannot safely handle this video timestamp profile; explicitly choose accurate mode") +type copyFrameIndex struct {origin float64;keys []time.Duration;audioAllowance time.Duration} +func inspectCopyFrames(parent context.Context,file *os.File)(copyFrameIndex,error){ + ctx,cancel:=context.WithTimeout(parent,10*time.Second);defer cancel() + select{case recordingDurations.slots<-struct{}{}:defer func(){<-recordingDurations.slots}();case <-ctx.Done():return copyFrameIndex{},ctx.Err()} + cmd:=exec.CommandContext(ctx,"ffprobe","-v","error","-protocol_whitelist","file,pipe","-format_whitelist","mov","-f","mov","-i","/proc/self/fd/3","-select_streams","v:0","-skip_frame","nokey","-show_frames","-show_streams","-show_entries","stream=start_time,has_b_frames:frame=best_effort_timestamp_time","-of","json") + cmd.ExtraFiles=[]*os.File{file};cmd.WaitDelay=2*time.Second;var raw limitedProbeOutput;cmd.Stdout=&raw;cmd.Stderr=io.Discard + if err:=cmd.Run();err!=nil{if ctx.Err()!=nil{return copyFrameIndex{},ctx.Err()};return copyFrameIndex{},errCopyReordered} + var data struct{Streams []struct{Start string `json:"start_time"`;Reordered *int `json:"has_b_frames"`} `json:"streams"`;Frames []struct{PTS string `json:"best_effort_timestamp_time"`} `json:"frames"`} + if json.Unmarshal(raw.Bytes(),&data)!=nil || len(data.Streams)!=1 || (data.Streams[0].Reordered==nil || *data.Streams[0].Reordered!=0) || len(data.Frames)==0{return copyFrameIndex{},errCopyReordered} + origin,err:=preparedMediaSeconds(data.Streams[0].Start);if err!=nil{return copyFrameIndex{},err} + index:=copyFrameIndex{origin:origin} + for _,frame:=range data.Frames{pts,err:=preparedMediaSeconds(frame.PTS);if err!=nil{return index,err};relative:=pts-origin;if relative<0{return index,errCopyReordered};value:=time.Duration(math.Round(relative*float64(time.Second)));if len(index.keys)>0 && value<=index.keys[len(index.keys)-1]{return index,errCopyReordered};index.keys=append(index.keys,value)} + return index,nil +} +func copyBoundary(keys []time.Duration,target,limit time.Duration,later bool)(time.Duration,error){ + if target<0 || target>limit{return 0,errExportPlanInvalid} + if target==limit{return limit,nil} + pos:=sort.Search(len(keys),func(i int)bool{return keys[i]>=target}) + if pos0 && !compatiblePreparedCopy(profiles[0],profiles[i]){return errPreparedMediaProfile}} + manifest,err:=os.CreateTemp("","simplenvr-keycopy-*.ffconcat");if err!=nil{return err};defer manifest.Close();defer os.Remove(manifest.Name()) + if err:=os.Remove(manifest.Name());err!=nil{return err} + if _,err:=io.WriteString(manifest,"ffconcat version 1.0\n");err!=nil{return err} + for i,part:=range plan.Parts{if _,err:=fmt.Fprintf(manifest,"file '/proc/self/fd/%d'\ninpoint %.9f\noutpoint %.9f\nduration %s\n",i+4,indices[i].origin+part.In.Seconds(),indices[i].origin+part.Out.Seconds(),trimSeconds(part.End.Sub(part.Start)));err!=nil{return err}} + if _,err:=manifest.Seek(0,io.SeekStart);err!=nil{return err} + args:=[]string{"-hide_banner","-loglevel","error","-nostdin","-protocol_whitelist","file,pipe","-format_whitelist","concat,mov","-f","concat","-safe","0","-auto_convert","0","-i","/proc/self/fd/3","-map","0:v:0"} + if len(profiles[0].Streams)==2{args=append(args,"-map","0:a:0")} + args=append(args,"-c","copy","-t",trimSeconds(plan.End.Sub(plan.Start)),"-avoid_negative_ts","disabled","-use_editlist","1","-f","mp4","-movflags","frag_keyframe+delay_moov","pipe:1") + writer:=&boundedExportJobWriter{ctx:ctx,file:output,limit:limit} + cmd:=ffmpegCommand(ctx,args...);cmd.ExtraFiles=append([]*os.File{manifest},pins.files...);cmd.Stdout=writer;cmd.Stderr=io.Discard + err=cmd.Run();if writer.exceeded{return errExportJobSize};if ctx.Err()!=nil{return ctx.Err()};if err!=nil{return err} + produced,err:=inspectPreparedMediaProfile(ctx,output);if err!=nil{return err};if len(produced.Streams)!=len(profiles[0].Streams){return errPreparedMediaProfile} + duration,err:=probeVideoDuration(ctx,output);if err!=nil{return err};delta:=duration-plan.End.Sub(plan.Start);if delta<0{delta=-delta};if delta>100*time.Millisecond{return errCopyReordered} + // Decode the candidate without changing it. No undecodable random-access + // start is advertised as a ready download. Time/resource limits still apply. + validate:=ffmpegCommand(ctx,"-hide_banner","-loglevel","error","-nostdin","-xerror","-protocol_whitelist","file,pipe","-format_whitelist","mov","-f","mov","-i","/proc/self/fd/3","-map","0:v:0","-map","0:a:0?","-f","null","-") + validate.ExtraFiles=[]*os.File{output};validate.Stdout=io.Discard;validate.Stderr=io.Discard + if err:=validate.Run();err!=nil{if ctx.Err()!=nil{return ctx.Err()};return errCopyReordered} + return pins.validate(ctx) +} diff --git a/export_keyframes_test.go b/export_keyframes_test.go new file mode 100644 index 0000000..7b8960c --- /dev/null +++ b/export_keyframes_test.go @@ -0,0 +1,38 @@ +package main + +import ( + "context" + "errors" + "io" + "strings" + "testing" + "time" +) +func TestCopyKeyframeBoundaries(t *testing.T){ + keys:=[]time.Duration{0,time.Second,2*time.Second,3*time.Second} + for _,tc:=range []struct{target time.Duration;later bool;want time.Duration}{{1500*time.Millisecond,false,time.Second},{1500*time.Millisecond,true,2*time.Second},{time.Second,false,time.Second},{3500*time.Millisecond,true,4*time.Second},{4*time.Second,true,4*time.Second}}{got,err:=copyBoundary(keys,tc.target,4*time.Second,tc.later);if err!=nil || got!=tc.want{t.Fatalf("%+v: %v %v",tc,got,err)}} + if _,err:=copyBoundary([]time.Duration{time.Second},500*time.Millisecond,4*time.Second,false);!errors.Is(err,errCopyReordered){t.Fatal("missing safe start accepted")} +} +func TestCopyApprovalBeforeJobAdmission(t *testing.T){ + _,service,h,cookie:=exportHTTPFixture(t) + calls:=0 + service.plan=func(ctx context.Context,req exportCreateRequest)(exportJobRenderer,error){calls++;if req.AcceptedStart==""{return nil,&exportCopyProposal{Code:"copy_range_approval_required",RequestedStart:req.Start,RequestedEnd:req.End,ExportStart:"2026-11-21T00:00:00Z",ExportEnd:"2026-11-21T00:00:11Z",AudioAllowanceMS:64}};return func(ctx context.Context,w io.Writer)error{_,err:=io.WriteString(w,"video");return err},nil} + first:=exportHTTPRequest(h,cookie,"POST",exportCreatePath,exportHTTPCreateBody) + if first.Code!=409 || !strings.Contains(first.Body.String(),"copy_range_approval_required"){t.Fatal(first.Code,first.Body.String())} + service.store.mu.Lock();count:=len(service.store.jobs);reserved:=service.store.reserved;service.store.mu.Unlock() + if count!=0 || reserved!=0{t.Fatal("proposal reserved a job or output storage")} + approved:=strings.TrimSuffix(exportHTTPCreateBody,"}")+`,"accepted_start":"2026-11-21T00:00:00Z","accepted_end":"2026-11-21T00:00:11Z"}` + next:=exportHTTPRequest(h,cookie,"POST",exportCreatePath,approved);if next.Code!=202 || calls!=2{t.Fatal(next.Code,next.Body.String(),calls)} + wrong:=strings.TrimSuffix(exportHTTPCreateBody,"}")+`,"mode":"accurate","accepted_start":"2026-11-21T00:00:00Z","accepted_end":"2026-11-21T00:00:11Z"}` + if got:=exportHTTPRequest(h,cookie,"POST",exportCreatePath,wrong).Code;got!=400{t.Fatal("accurate mode accepted copy approval",got)} +} +func TestCopyStaleApprovalRequiresNewConsent(t *testing.T){ + start:=time.Date(2026,11,21,0,0,0,0,time.UTC) + requested:=exportPlan{Start:start.Add(1500*time.Millisecond),End:start.Add(2500*time.Millisecond)} + actual:=exportPlan{Start:start.Add(time.Second),End:start.Add(3*time.Second)} + req:=exportCreateRequest{Start:requested.Start.Format(time.RFC3339Nano),End:requested.End.Format(time.RFC3339Nano),Mode:"copy",AcceptedStart:start.Format(time.RFC3339Nano),AcceptedEnd:actual.End.Format(time.RFC3339Nano)} + var proposal *exportCopyProposal + if err:=copyProposal(req,requested,actual,nil);!errors.As(err,&proposal){t.Fatal("stale interval was approved")} + req.AcceptedStart=actual.Start.Format(time.RFC3339Nano) + if err:=copyProposal(req,requested,actual,nil);err!=nil{t.Fatal(err)} +} From 2468f09bb0d0b164f131d6dae3e36c22d2a5e861 Mon Sep 17 00:00:00 2001 From: mouldybread <38598858+mouldybread@users.noreply.github.com> Date: Sat, 21 Nov 2026 12:08:32 +0800 Subject: [PATCH 6/8] Expose export modes and explicit copy-range approval in browser --- docs/EXPORT_API_MODES.md | 16 ++- export_mode_ui_test.go | 9 ++ static/js/export-ui.js | 275 ++++++--------------------------------- tests/export-mode-ui.cjs | 11 ++ 4 files changed, 67 insertions(+), 244 deletions(-) create mode 100644 export_mode_ui_test.go create mode 100644 tests/export-mode-ui.cjs diff --git a/docs/EXPORT_API_MODES.md b/docs/EXPORT_API_MODES.md index cf1a40a..41910bf 100644 --- a/docs/EXPORT_API_MODES.md +++ b/docs/EXPORT_API_MODES.md @@ -1,11 +1,15 @@ -# Export mode API integration (intermediate commit) +# Export modes: current integration status -The HTTP job planner now calls the copy/accurate renderers instead of the legacy video-only trim renderer. POST /api/exports/create accepts mode (copy by default, or accurate). It never automatically switches modes. Job output accounting is populated from the validated output file size, including direct renderer writes. +The HTTP planner calls the prepared copy/accurate renderers and the browser offers both modes. This status supersedes older staged progress notes saying those paths are not yet wired. The branch is still NOT release-ready: Go compilation/tests, real-browser harness migration and full runtime validation remain outstanding. -For complete recordings, copy preserves original bytes (one file) or compatible compressed A/V packets (multiple files). Accurate explicitly re-encodes supported video and AAC audio and applies the existing renderer validation. +Copy is the default and never silently falls back to re-encoding. Whole recordings are copied byte for byte into prepared storage (single source) or remuxed with compatible compressed A/V payloads (multiple sources). This is not yet zero-copy original-file HTTP delivery. -For subsections, copy scans decoded keyframe timestamps and supports non-reordered video only. It rounds the video start earlier and end later to available keyframe/file boundaries, rechecks expanded coverage for gaps/overlaps, and responds with 409 JSON code copy_range_approval_required before admitting a job or reserving output storage. The response includes requested_start/end, export_start/end and audio_allowance_ms (one AAC packet). Approval must send accepted_start/end matching the freshly recomputed proposal along with the unchanged original request. Stale approval never silently approves a different video interval. Audio packet edges can extend outside the video-aligned interval by the disclosed allowance. +Subsection copy is limited to non-reordered video with usable decoded keyframe timestamps. The server rounds video boundaries to keyframe/file boundaries, reinspects expanded coverage and returns a 409 JSON copy_range_approval_required proposal BEFORE job admission/storage reservation. The UI displays requested and proposed local dates/times and AAC packet allowance, and waits for Use proposed copy range. Canceling the proposal creates no job. Approval sends accepted_start/end with the immutable original range; the server recomputes the proposal and refuses stale boundaries. Changed proposals require another click. Unsupported/reordered subsection copy must be changed to Accurate explicitly. -Copy rendering uses pinned descriptors, explicit track mapping, bounded output, duration checks and an external decode before readiness. Reordered video, missing keyframe metadata, unsupported tracks, incompatible configurations and undecodable random-access output are refused; choose accurate explicitly. These checks are not a general proof of frame/sample completeness or calibrated wall-clock boundaries. +AAC packet edges can extend outside the video-aligned interval by the disclosed packet allowance. These are metadata-derived/frame-packet granular boundaries, not calibrated camera-clock claims. Copy candidates are duration-probed and externally decoded before readiness; this does not certify every source frame/sample or colour value. -This is not yet a finished application change: the corresponding UI mode/approval controls still need to be committed, and the real-browser acceptance harness still requires migration. Go compilation/tests have not run in the assistant environment. A local FFmpeg command experiment, not the Go implementation, produced a decodable non-reordered H.264/AAC keyframe-aligned subsection. Do not merge/release on that evidence alone. +Accurate mode explicitly re-encodes video (libx264 CRF 18) and AAC audio. It shares the video timestamp/cut origin, preserves supported audio offset, and converts full-range input to limited-range yuv420p. It is not lossless or byte-identical. Missing leading/trailing audio may be represented as silence to maintain the selected window, without audio stretching. Mixed audio configurations, extra tracks, rotation and higher bit depths remain unsupported. + +The UI uses normal authenticated download URLs, not video Blobs. Cancellation/release requires server acknowledgement. Scratch configuration, server limits, session isolation and lease lifetime remain documented in LARGE_MEDIA_EXPORTS.md. + +New focused mode/approval UI regressions passed in Node in UTC and America/Chicago using a DOM fixture; syntax checks also passed. Go tests are added but have not been executed here. Old real-browser harnesses still contain Blob-era assumptions and need migration while retaining their media, hash, duration, playback and cancellation assertions. Large-file, slow-client, full-resolution real-camera and end-to-end API/media/browser validation must pass before completion is claimed. diff --git a/export_mode_ui_test.go b/export_mode_ui_test.go new file mode 100644 index 0000000..e5117cd --- /dev/null +++ b/export_mode_ui_test.go @@ -0,0 +1,9 @@ +package main + +import "testing" + +func TestPreparedExportModeUI(t *testing.T) { + for _,zone:=range []string{"UTC","America/Chicago"} { + t.Run(zone,func(t *testing.T){runFrontendRegression(t,"tests/export-mode-ui.cjs",zone)}) + } +} diff --git a/static/js/export-ui.js b/static/js/export-ui.js index 9de7348..7bfc46f 100644 --- a/static/js/export-ui.js +++ b/static/js/export-ui.js @@ -1,239 +1,38 @@ -// Server-prepared export UI; only bounded control JSON is fetched. -let exportUiBatch = null; -let exportInitialRange = null; -const EXPORT_RANGE_STEPS = [1000,10000,60000,600000]; - -function exportUiNode(tag,text='') {const node=document.createElement(tag);node.textContent=String(text);return node;} -function exportRangeCandidate(start,end,boundary,delta) { - if(!Number.isFinite(start) || !Number.isFinite(end) || end<=start || !['start','end'].includes(boundary) || !Number.isFinite(delta))return null; - const value=(boundary==='start'?start:end)+delta; - if(!Number.isFinite(new Date(value).getTime()) || (boundary==='start'?value>=end:value<=start))return null; - return value; -} -function exportRangeStep() {const value=Number(document.getElementById('download-range-step')?.value);return EXPORT_RANGE_STEPS.includes(value)?value:60000;} -function adjustDownloadBoundary(boundary,direction) { - if(exportUiBatch?.running || ![-1,1].includes(direction))return false; - const value=exportRangeCandidate(downloadStartTime?.getTime(),downloadEndTime?.getTime(),boundary,direction*exportRangeStep()); - if(value===null)return false; - if(boundary==='start')downloadStartTime=new Date(value);else downloadEndTime=new Date(value); - updateDownloadTimeDisplay();return true; -} -function installExportRangeControls() { - const button=document.getElementById('export-start-btn');if(button)button.textContent='Prepare exports on server'; - if(document.getElementById('download-range-step'))return; - const old=document.getElementById('download-time-display'); - const toolbar=old?.parentElement?.parentElement;if(!toolbar)return; - const group=exportUiNode('div');group.style.cssText='display:flex;flex-direction:column;gap:12px;width:100%'; - for(const boundary of ['start','end']) { - const row=exportUiNode('div');row.style.cssText='display:flex;align-items:center;gap:10px;flex-wrap:wrap'; - row.appendChild(exportUiNode('span',boundary==='start'?'Start':'End')); - for(const direction of [-1,1]) { - const button=exportUiNode('button',direction<0?'Earlier':'Later');button.type='button';button.className='btn'; - button.id=`download-${boundary}-${direction<0?'earlier':'later'}`; - button.setAttribute('aria-label',`Move ${boundary} ${direction<0?'earlier':'later'}`); - button.addEventListener('click',()=>adjustDownloadBoundary(boundary,direction));row.appendChild(button); - if(direction===-1){const time=exportUiNode('output');time.id=`download-${boundary}-display`;row.appendChild(time);} - } - group.appendChild(row); - } - const label=exportUiNode('label','Adjustment step: '),step=exportUiNode('select');step.id='download-range-step'; - for(const [index,text] of ['1 second','10 seconds','1 minute','10 minutes'].entries()){ - const option=exportUiNode('option',text);option.value=String(EXPORT_RANGE_STEPS[index]);step.appendChild(option); - } - step.value='60000';step.addEventListener('change',updateDownloadTimeDisplay);label.appendChild(step);group.appendChild(label); - const reset=exportUiNode('button','Reset to initial selection');reset.type='button';reset.className='btn';reset.id='download-range-reset'; - reset.addEventListener('click',resetDownloadTimeRange);group.appendChild(reset); - for(const id of ['download-time-display','download-duration-display']){const node=exportUiNode('div');node.id=id;group.appendChild(node);} - group.appendChild(exportUiNode('p','Times are local with time-zone labels. Exports require complete coverage. Existing media compatibility restrictions still apply. Prepare on the server, then click Download prepared video.')); - toolbar.replaceChildren(group); -} -function exportBoundaryLabel(time) { - return time.toLocaleString(undefined,{year:'numeric',month:'2-digit',day:'2-digit',hour:'2-digit',minute:'2-digit',second:'2-digit',timeZoneName:'short',hour12:false}); -} -// Loaded after playback.js; these entry points own the export dialog. -function updateDownloadTimeDisplay() { - const start=downloadStartTime?.getTime(),end=downloadEndTime?.getTime(),valid=Number.isFinite(start)&&Number.isFinite(end)&&end>start; - const busy=!!exportUiBatch?.running; - for(const boundary of ['start','end']) { - const display=document.getElementById(`download-${boundary}-display`); - if(display)display.textContent=valid?exportBoundaryLabel(boundary==='start'?downloadStartTime:downloadEndTime):'Select a valid range'; - for(const direction of [-1,1]) { - const button=document.getElementById(`download-${boundary}-${direction<0?'earlier':'later'}`); - if(button)button.disabled=busy || exportRangeCandidate(start,end,boundary,direction*exportRangeStep())===null; - } - } - const summary=document.getElementById('download-time-display');if(summary)summary.textContent=valid?'Selected interval; end is exclusive.':''; - const duration=document.getElementById('download-duration-display'); - if(duration)duration.textContent=valid?`Duration: ${(end-start)/1000} seconds (${((end-start)/60000).toFixed(2)} minutes)`:'Invalid or empty interval'; - const step=document.getElementById('download-range-step');if(step)step.disabled=busy; - const reset=document.getElementById('download-range-reset');if(reset)reset.disabled=busy || !exportInitialRange; -} -function openDownloadModal() { - const modal=document.getElementById('download-modal');if(!modal)return; - installExportRangeControls(); - if(!exportUiBatch?.running){ - if(!Number.isFinite(currentTimePosition)){if(typeof showError==='function')showError('Select a time on the timeline first');return;} - const start=currentTimePosition-600000,end=currentTimePosition+600000; - if(!Number.isFinite(new Date(start).getTime()) || !Number.isFinite(new Date(end).getTime()))return; - exportInitialRange={start,end};downloadStartTime=new Date(start);downloadEndTime=new Date(end);downloadCameras=[...selectedCameras]; - renderDownloadCameraSelection(); - } - updateDownloadTimeDisplay();modal.classList.add('active');modal.style.display='block'; -} -function resetDownloadTimeRange() { - if(exportUiBatch?.running || !exportInitialRange)return; - downloadStartTime=new Date(exportInitialRange.start);downloadEndTime=new Date(exportInitialRange.end);updateDownloadTimeDisplay(); -} - -const EXPORT_JOB_POLL_MS=1500; -const EXPORT_JOB_REQUEST_MS=45000; -const EXPORT_JOB_WAIT_MS=2*60*60*1000; -function exportUiState(job,state,message){job.state=state;job.row.dataset.state=state;job.status.textContent=String(message).slice(0,1200);} -function exportUiText(text){return String(text ?? '').replace(/[\u0000-\u001f\u007f]/g,' ').slice(0,600);} -function exportUiError(message){return new Error(message);} -async function exportUiRead(response,signal,limit=8192){ - if(!response.body || typeof response.body.getReader!=='function')throw exportUiError('Streaming JSON responses are unavailable.'); - const reader=response.body.getReader(),chunks=[];let size=0,done=false; - const canceled=()=>{try{Promise.resolve(reader.cancel()).catch(()=>{});}catch(_){}}; - signal.addEventListener('abort',canceled,{once:true}); - try{ - while(true){ - if(signal.aborted)throw exportUiError('Export control request timed out.'); - const part=await reader.read();if(part.done){done=true;break;} - if(part.value.byteLength>limit-size)throw exportUiError('Export control response exceeded its size limit.'); - size+=part.value.byteLength;chunks.push(part.value); - } - if(signal.aborted)throw exportUiError('Export control request timed out.'); - const bytes=new Uint8Array(size);let offset=0;for(const chunk of chunks){bytes.set(chunk,offset);offset+=chunk.byteLength;} - return new TextDecoder().decode(bytes); - }finally{signal.removeEventListener('abort',canceled);if(!done)canceled();try{reader.releaseLock();}catch(_){}} -} -function exportUiHTTPMessage(response){ - if(response.status===401 || response.status===403)return 'Authentication expired or access denied; sign in again.'; - if(response.status===503)return 'Prepared exports unavailable: check NVR_EXPORT_SCRATCH_DIR, recording metadata, and whether the server is restarting.'; - if(response.status===429)return 'Server export capacity is full. Release earlier prepared files or wait for expiry, then retry.'; - if(response.status===404)return 'Export or camera unavailable; the job may have expired or the server restarted.'; - const plan=response.headers.get('X-Export-Plan-Status'); - if(plan==='gap')return 'The full interval is not covered by recordings; adjust Start or End. No gaps were silently skipped.'; - if(plan==='overlap')return 'Overlapping recordings require an explicit clip choice.'; - if(response.status===413)return 'Server export limit exceeded; reduce the selected interval.'; - if(response.status===422)return 'The recording profile is not supported by the current trim renderer.'; - return `Export control request failed (HTTP ${response.status}).`; -} -// Only bounded JSON/control bodies use fetch. Video uses a normal download URL. -async function exportUiRequest(path,body){ - const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),EXPORT_JOB_REQUEST_MS); - try{ - const options={credentials:'same-origin',cache:'no-store',redirect:'error',signal:controller.signal}; - if(body!==undefined){options.method='POST';options.headers={'Content-Type':'application/json'};options.body=JSON.stringify(body);} - const response=await fetch(path,options); - if(response.redirected)throw exportUiError('Unexpected redirect; sign in again.'); - if(response.status===204)return {status:204}; - const text=await exportUiRead(response,controller.signal); - if(response.status!==200 && response.status!==202)throw exportUiError(exportUiHTTPMessage(response)+(text?' Server: '+exportUiText(text):'')); - if((response.headers.get('Content-Type') || '').split(';')[0].trim().toLowerCase()!=='application/json')throw exportUiError('Expected export JSON, not a video or login page.'); - let value;try{value=JSON.parse(text);}catch(_){throw exportUiError('Invalid export JSON response.');} - return {status:response.status,value}; - }finally{clearTimeout(timer);} -} -function exportUiView(value,expected){ - if(!value || typeof value!=='object' || Array.isArray(value) || !/^[0-9a-f]{64}$/.test(value.id || '') || (expected && value.id!==expected) || !['queued','preparing','ready','failed','canceled','expired'].includes(value.state) || typeof value.message!=='string' || !Number.isSafeInteger(value.bytes) || value.bytes<0 || !Number.isFinite(Date.parse(value.expires_at)))throw exportUiError('Invalid export status response.'); - if(value.state==='ready' && (value.bytes===0 || value.download_url!==`/api/exports/download?id=${value.id}`))throw exportUiError('Invalid prepared download response.'); - return value; -} -function exportUiRemoveDownload(job){if(job.save){job.save.disabled=true;job.save.remove();job.save=null;}job.url=null;clearTimeout(job.expiryTimer);} -function exportUiFinish(job,state,text){exportUiRemoveDownload(job);job.finished=true;exportUiState(job,state,text);if(job.finishReady){const finish=job.finishReady;job.finishReady=null;finish();}} -function exportUiWait(job){return new Promise(resolve=>{job.wake=()=>{clearTimeout(job.pollTimer);job.wake=null;resolve();};job.pollTimer=setTimeout(job.wake,EXPORT_JOB_POLL_MS);});} -async function exportUiCancel(job){ - if(job.canceling || ['canceled','expired','released'].includes(job.state))return; - job.reason='canceled';job.wake?.(); - if(!job.serverId){ - if(job.creating){exportUiState(job,'canceling','Cancellation requested; waiting for the creation response to identify the server job.');return;} - exportUiFinish(job,'canceled','Canceled before a server job was created.');job.cancel.disabled=true;return; - } - job.canceling=true;job.cancel.disabled=true;exportUiRemoveDownload(job); - exportUiState(job,'canceling','Requesting cancellation/release on the server. Existing browser transfers may continue.'); - try{ - const response=await exportUiRequest('/api/exports/cancel',{id:job.serverId}); - if(response.status!==204)throw exportUiError('Unexpected cancellation response.'); - exportUiFinish(job,job.handed?'released':'canceled',job.handed?'Server confirmed release. Any already-started download may continue; disk saving is not verified.':'Server confirmed export cancellation/release.');job.cancel.disabled=true; - }catch(error){ - exportUiFinish(job,'release-failed',`Cancellation could not be confirmed. ${error.message} The job may remain until server expiry.`);job.cancel.textContent='Retry cancellation/release';job.cancel.disabled=false; - }finally{job.canceling=false;} -} -function cancelExportDownloads(){return exportUiBatch?Promise.all(exportUiBatch.jobs.map(job=>exportUiCancel(job))):Promise.resolve();} -function exportUiOffer(job,view){ - return new Promise(resolve=>{ - job.finishReady=resolve;job.url=`/api/exports/download?id=${job.serverId}`; - const expires=Date.parse(view.expires_at),remaining=expires-Date.now(); - if(remaining<=0){exportUiFinish(job,'expired','Prepared export expired; prepare it again.');return;} - job.save=exportUiNode('button','Download prepared video');job.save.type='button';job.save.className='btn btn-primary'; - job.save.addEventListener('click',()=>{ - if(job.finished || job.canceling || job.handed)return; - if(Date.now()>=expires){exportUiFinish(job,'expired','Prepared export expired; prepare it again.');job.cancel.disabled=true;return;} - const anchor=exportUiNode('a');anchor.href=job.url;anchor.download=`export-${job.serverId}.mp4`; - try{document.body.appendChild(anchor);anchor.click();}catch(_){exportUiState(job,'ready','Browser download request failed. Try again before expiry.');return;}finally{anchor.remove();} - job.handed=true;job.save.disabled=true; - exportUiState(job,'handed','Browser download requested; check your browser downloads. Disk saving is not verified. Release after the browser accepts it; existing transfers retain their server file lease.'); - }); - job.row.appendChild(job.save);job.cancel.textContent='Release prepared file / continue';job.cancel.disabled=false; - exportUiState(job,'ready',`Ready on server: ${(view.bytes/(1024*1024)).toFixed(1)} MiB. Click Download prepared video before ${new Date(expires).toLocaleString()}. Nothing has been saved by this page; disk saving is not verified.`); - job.expiryTimer=setTimeout(()=>{exportUiFinish(job,'expired','Prepared export reached its expiry time; request a new export if needed. An already-started browser transfer may continue.');job.cancel.disabled=true;},Math.min(remaining,2147483647)); - job.save.focus?.(); - }); -} -async function exportUiRunJob(job){ - if(job.reason==='canceled'){job.cancel.disabled=true;return;} - try{ - job.creating=true;exportUiState(job,'preparing','Requesting server preparation. Video will not be buffered in this page.'); - let result; - try{result=await exportUiRequest('/api/exports/create',{camera_id:job.id,start_time:job.start,end_time:job.end});}finally{job.creating=false;} - if(result.status!==202)throw exportUiError('Expected server job acceptance.'); - let view=exportUiView(result.value);job.serverId=view.id; - if(job.reason==='canceled'){await exportUiCancel(job);return;} - const deadline=Date.now()+EXPORT_JOB_WAIT_MS; - while(true){ - if(job.reason==='canceled')return; - if(view.state==='ready'){await exportUiOffer(job,view);return;} - if(['failed','canceled','expired'].includes(view.state)){exportUiFinish(job,view.state,exportUiText(view.message));job.cancel.disabled=true;return;} - exportUiState(job,view.state,view.state==='queued'?'Queued on server.':'Preparing on server; large exports can take time.'); - if(Date.now()>=deadline)throw exportUiError('Stopped waiting after two hours. The job may continue until its server deadline or expiry.'); - await exportUiWait(job);if(job.reason==='canceled')return; - result=await exportUiRequest(`/api/exports/status?id=${job.serverId}`); - if(job.reason==='canceled')return; - if(result.status!==200)throw exportUiError('Unexpected export status response.'); - view=exportUiView(result.value,job.serverId); - } - }catch(error){ - if(job.reason==='canceled' && job.serverId){await exportUiCancel(job);return;} - exportUiFinish(job,'failed',`${error.message} ${job.serverId?'The server job may still exist; use cancellation below or wait for expiry.':'Creation outcome may be unknown if the response was lost; no automatic retry was made.'}`); - job.cancel.disabled=!job.serverId;if(job.serverId)job.cancel.textContent='Cancel server job'; - } -} -function startExportDownloads(){ - if(exportUiBatch?.running)return exportUiBatch.promise; - const host=document.getElementById('download-status'),list=document.getElementById('download-progress-list'); - if(!host || !list)return Promise.resolve(); - // Keep unresolved release failures visible rather than orphaning their controls. - if(exportUiBatch?.jobs.some(job=>job.state==='release-failed' || (job.serverId && job.state==='failed' && !job.cancel.disabled)))return exportUiBatch.promise; - host.style.display='block';list.replaceChildren(); - const start=downloadStartTime?.getTime(),end=downloadEndTime?.getTime(),ids=[...new Set(downloadCameras)]; - if(!Number.isFinite(start) || !Number.isFinite(end) || end<=start || ids.length<1 || ids.length>8){list.appendChild(exportUiNode('p','Choose a valid range and one to eight cameras. No request started.'));return Promise.resolve();} - const cameras=ids.map(id=>allCameras.find(cam=>cam.id===id)); - if(cameras.some(cam=>!cam || typeof cam.id!=='string' || cam.id.length>256)){list.appendChild(exportUiNode('p','Selected camera unavailable; reload the camera list.'));return Promise.resolve();} - const stop=exportUiNode('button','Cancel all exports');stop.type='button';stop.addEventListener('click',()=>{void cancelExportDownloads();});list.appendChild(stop); - list.appendChild(exportUiNode('p','Prepare on the server, then download with your browser. Server storage limits apply; this page does not buffer video or impose a 64 MiB file limit. The next camera starts after release or expiry of the current prepared export. Existing rendering compatibility restrictions still apply.')); - const jobs=cameras.map(cam=>{ - const row=exportUiNode('div');row.className='export-job';row.dataset.cameraId=cam.id; - const status=exportUiNode('p','Queued'),cancel=exportUiNode('button','Cancel');cancel.type='button';status.setAttribute('aria-live','polite'); - row.appendChild(exportUiNode('h4',String(cam.name ?? cam.id).slice(0,256)));row.appendChild(status);row.appendChild(cancel);list.appendChild(row); - const job={id:cam.id,start:new Date(start).toISOString(),end:new Date(end).toISOString(),row,status,cancel,state:'queued',reason:null,serverId:null,creating:false,canceling:false,finished:false,handed:false,url:null,save:null,finishReady:null,wake:null}; - cancel.addEventListener('click',()=>{void exportUiCancel(job);});return job; - }); - const batch={jobs,running:true,promise:null};exportUiBatch=batch;updateDownloadTimeDisplay(); - const button=document.getElementById('export-start-btn');if(button)button.disabled=true; - batch.promise=(async()=>{try{for(const job of jobs)await exportUiRunJob(job);}finally{batch.running=false;if(button)button.disabled=false;updateDownloadTimeDisplay();}})();return batch.promise; -} -// Do not auto-release on navigation: downloading can itself trigger pagehide, -// and aborting creation can lose the server ID. Server TTL bounds abandoned jobs. +// Job-based exports: only bounded control JSON is fetched, never video. +let exportUiBatch=null,exportInitialRange=null; +const EXPORT_RANGE_STEPS=[1000,10000,60000,600000]; +function exportUiNode(tag,text=''){const e=document.createElement(tag);e.textContent=String(text);return e;} +function exportRangeCandidate(a,b,which,delta){if(!Number.isFinite(a)||!Number.isFinite(b)||b<=a||!['start','end'].includes(which)||!Number.isFinite(delta))return null;const next=(which==='start'?a:b)+delta;return !Number.isFinite(new Date(next).getTime())||(which==='start'?next>=b:next<=a)?null:next;} +function exportRangeStep(){const n=Number(document.getElementById('download-range-step')?.value);return EXPORT_RANGE_STEPS.includes(n)?n:60000;} +function adjustDownloadBoundary(which,direction){if(exportUiBatch?.running||![-1,1].includes(direction))return false;const n=exportRangeCandidate(downloadStartTime?.getTime(),downloadEndTime?.getTime(),which,direction*exportRangeStep());if(n===null)return false;if(which==='start')downloadStartTime=new Date(n);else downloadEndTime=new Date(n);updateDownloadTimeDisplay();return true;} +function exportBoundaryLabel(d){return d.toLocaleString(undefined,{year:'numeric',month:'2-digit',day:'2-digit',hour:'2-digit',minute:'2-digit',second:'2-digit',timeZoneName:'short',hour12:false});} +function installExportRangeControls(){ + const prepare=document.getElementById('export-start-btn');if(prepare)prepare.textContent='Prepare exports on server';if(document.getElementById('download-range-step'))return; + const bar=document.getElementById('download-time-display')?.parentElement?.parentElement;if(!bar)return;const group=exportUiNode('div');group.style.cssText='display:flex;flex-direction:column;gap:10px;width:100%'; + for(const which of ['start','end']){const row=exportUiNode('div',which==='start'?'Start: ':'End: ');for(const dir of [-1,1]){const b=exportUiNode('button',dir<0?'Earlier':'Later');b.type='button';b.id=`download-${which}-${dir<0?'earlier':'later'}`;b.setAttribute('aria-label',`Move ${which} ${dir<0?'earlier':'later'}`);b.addEventListener('click',()=>adjustDownloadBoundary(which,dir));row.appendChild(b);if(dir<0){const out=exportUiNode('output');out.id=`download-${which}-display`;row.appendChild(out);}}group.appendChild(row);} + const label=exportUiNode('label','Adjustment step: '),step=exportUiNode('select');step.id='download-range-step';['1 second','10 seconds','1 minute','10 minutes'].forEach((text,i)=>{const o=exportUiNode('option',text);o.value=String(EXPORT_RANGE_STEPS[i]);step.appendChild(o);});step.value='60000';step.addEventListener('change',updateDownloadTimeDisplay);label.appendChild(step);group.appendChild(label); + const modeLabel=exportUiNode('label','Export mode: '),mode=exportUiNode('select');mode.id='download-export-mode';for(const [value,text]of [['copy','Original quality / stream copy'],['accurate','Accurate cut — re-encode video and audio']]){const o=exportUiNode('option',text);o.value=value;mode.appendChild(o);}mode.value='copy';modeLabel.appendChild(mode);group.appendChild(modeLabel); + group.appendChild(exportUiNode('p','Copy preserves encoded media. A proposed subsection range and possible AAC packet overhang require approval before a job starts. Subsection copy with reordered video is refused; choose Accurate explicitly. Accurate cuts re-encode and may convert full-range colour to limited range. Timing is frame/packet granular, not calibrated camera wall-clock time.')); + const reset=exportUiNode('button','Reset to initial selection');reset.id='download-range-reset';reset.type='button';reset.addEventListener('click',resetDownloadTimeRange);group.appendChild(reset);for(const id of ['download-time-display','download-duration-display']){const e=exportUiNode('div');e.id=id;group.appendChild(e);}bar.replaceChildren(group); +} +function updateDownloadTimeDisplay(){const a=downloadStartTime?.getTime(),b=downloadEndTime?.getTime(),valid=Number.isFinite(a)&&Number.isFinite(b)&&b>a,busy=!!exportUiBatch?.running;for(const which of ['start','end']){const e=document.getElementById(`download-${which}-display`);if(e)e.textContent=valid?exportBoundaryLabel(which==='start'?downloadStartTime:downloadEndTime):'Invalid range';for(const dir of [-1,1]){const e=document.getElementById(`download-${which}-${dir<0?'earlier':'later'}`);if(e)e.disabled=busy||exportRangeCandidate(a,b,which,dir*exportRangeStep())===null;}}const summary=document.getElementById('download-time-display');if(summary)summary.textContent='End is exclusive.';const duration=document.getElementById('download-duration-display');if(duration)duration.textContent=valid?`Duration: ${(b-a)/1000} seconds`:'Invalid range';for(const id of ['download-range-step','download-range-reset','download-export-mode']){const e=document.getElementById(id);if(e)e.disabled=busy;}} +function openDownloadModal(){const modal=document.getElementById('download-modal');if(!modal)return;installExportRangeControls();if(!exportUiBatch?.running){if(!Number.isFinite(currentTimePosition)){if(typeof showError==='function')showError('Select a time first');return;}exportInitialRange={start:currentTimePosition-600000,end:currentTimePosition+600000};downloadStartTime=new Date(exportInitialRange.start);downloadEndTime=new Date(exportInitialRange.end);downloadCameras=[...selectedCameras];renderDownloadCameraSelection();}updateDownloadTimeDisplay();modal.classList.add('active');modal.style.display='block';} +function resetDownloadTimeRange(){if(exportUiBatch?.running||!exportInitialRange)return;downloadStartTime=new Date(exportInitialRange.start);downloadEndTime=new Date(exportInitialRange.end);updateDownloadTimeDisplay();} +const EXPORT_JOB_POLL_MS=1500,EXPORT_JOB_REQUEST_MS=45000,EXPORT_JOB_WAIT_MS=7200000; +function exportUiState(job,state,message){job.state=state;job.row.dataset.state=state;job.status.textContent=String(message).slice(0,1500);} +function exportUiText(text){return String(text??'').replace(/[\u0000-\u001f\u007f]/g,' ').slice(0,600);} +async function exportUiRead(response,signal){if(!response.body?.getReader)throw Error('Streaming JSON responses unavailable');const reader=response.body.getReader(),chunks=[];let size=0,done=false;const cancel=()=>{try{Promise.resolve(reader.cancel()).catch(()=>{});}catch(_){}};signal.addEventListener('abort',cancel,{once:true});try{while(true){if(signal.aborted)throw Error('Export control request timed out');const p=await reader.read();if(p.done){done=true;break;}if(p.value.byteLength>8192-size)throw Error('Export control response exceeded its size limit');size+=p.value.byteLength;chunks.push(p.value);}if(signal.aborted)throw Error('Export control request timed out');const bytes=new Uint8Array(size);let n=0;for(const c of chunks){bytes.set(c,n);n+=c.byteLength;}return new TextDecoder().decode(bytes);}finally{signal.removeEventListener('abort',cancel);if(!done)cancel();try{reader.releaseLock();}catch(_){}}} +function exportUiHTTPMessage(r){if(r.status===401||r.status===403)return 'Authentication expired or access denied; sign in again.';if(r.status===503)return 'Prepared exports unavailable; check NVR_EXPORT_SCRATCH_DIR, recording metadata, or server restart.';if(r.status===429)return 'Server capacity is full; release exports or wait for expiry.';if(r.headers.get('X-Export-Plan-Status')==='gap')return 'The full interval is not covered; no gaps were silently skipped.';if(r.status===422)return 'Unsupported recording profile for this mode; choose Accurate explicitly if subsection copy is refused.';return `Export request failed (HTTP ${r.status}).`;} +async function exportUiRequest(path,body){const controller=new AbortController(),timer=setTimeout(()=>controller.abort(),EXPORT_JOB_REQUEST_MS);try{const options={credentials:'same-origin',cache:'no-store',redirect:'error',signal:controller.signal};if(body!==undefined){options.method='POST';options.headers={'Content-Type':'application/json'};options.body=JSON.stringify(body);}const r=await fetch(path,options);if(r.redirected)throw Error('Unexpected redirect; sign in again');if(r.status===204)return {status:204};const text=await exportUiRead(r,controller.signal),json=(r.headers.get('Content-Type')||'').split(';')[0].trim().toLowerCase()==='application/json';let value;if(json&&[200,202,409].includes(r.status)){try{value=JSON.parse(text);}catch(_){if(r.status===200||r.status===202)throw Error('Invalid export JSON');}}if(r.status===409&&json&&value?.code==='copy_range_approval_required')return {status:409,value};if(r.status!==200&&r.status!==202)throw Error(exportUiHTTPMessage(r)+(text?' Server: '+exportUiText(text):''));if(!json)throw Error('Expected export JSON, not video or a login page');return {status:r.status,value};}finally{clearTimeout(timer);}} +function exportUiView(v,id){if(!v||typeof v!=='object'||Array.isArray(v)||!/^[0-9a-f]{64}$/.test(v.id||'')||(id&&v.id!==id)||!['queued','preparing','ready','failed','canceled','expired'].includes(v.state)||typeof v.message!=='string'||!Number.isSafeInteger(v.bytes)||v.bytes<0||!Number.isFinite(Date.parse(v.expires_at)))throw Error('Invalid export status response');if(v.state==='ready'&&(!v.bytes||v.download_url!==`/api/exports/download?id=${v.id}`))throw Error('Invalid prepared download response');return v;} +function exportUiRemoveDownload(j){if(j.save){j.save.disabled=true;j.save.remove();j.save=null;}j.url=null;clearTimeout(j.expiryTimer);} +function exportUiFinish(j,state,text){exportUiRemoveDownload(j);j.finished=true;exportUiState(j,state,text);j.resolveConsent?.(false);if(j.finishReady){const done=j.finishReady;j.finishReady=null;done();}} +function exportUiWait(j){return new Promise(resolve=>{j.wake=()=>{clearTimeout(j.pollTimer);j.wake=null;resolve();};j.pollTimer=setTimeout(j.wake,EXPORT_JOB_POLL_MS);});} +async function exportUiCancel(j){if(j.canceling||['canceled','expired','released'].includes(j.state))return;j.reason='canceled';j.wake?.();if(!j.serverId){if(j.creating){exportUiState(j,'canceling','Waiting for creation response to identify the job to cancel.');return;}exportUiFinish(j,'canceled','Canceled before a job was created.');j.cancel.disabled=true;return;}j.canceling=true;j.cancel.disabled=true;exportUiRemoveDownload(j);try{const r=await exportUiRequest('/api/exports/cancel',{id:j.serverId});if(r.status!==204)throw Error('Unexpected release response');exportUiFinish(j,j.handed?'released':'canceled','Server confirmed cancellation/release. Existing transfers may continue; disk saving is not verified.');j.cancel.disabled=true;}catch(e){exportUiFinish(j,'release-failed',`Cancellation could not be confirmed. ${e.message} The job may remain until expiry.`);j.cancel.textContent='Retry cancellation/release';j.cancel.disabled=false;}finally{j.canceling=false;}} +function cancelExportDownloads(){return exportUiBatch?Promise.all(exportUiBatch.jobs.map(exportUiCancel)):Promise.resolve();} +function exportUiConsent(j,p){const a=Date.parse(p.export_start),b=Date.parse(p.export_end);if(j.mode!=='copy'||Date.parse(p.requested_start)!==Date.parse(j.start)||Date.parse(p.requested_end)!==Date.parse(j.end)||!Number.isFinite(a)||!Number.isFinite(b)||a>Date.parse(j.start)||b128)throw Error('Invalid copy range proposal');return new Promise(resolve=>{const box=exportUiNode('div'),button=exportUiNode('button','Use proposed copy range');button.type='button';box.appendChild(exportUiNode('p',`Requested: ${exportBoundaryLabel(new Date(j.start))} to ${exportBoundaryLabel(new Date(j.end))}. Proposed video interval: ${exportBoundaryLabel(new Date(a))} to ${exportBoundaryLabel(new Date(b))}. AAC packet boundaries may include up to ${p.audio_allowance_ms} ms outside that video interval. Copy does not re-encode. Approve this interval, or Cancel and choose Accurate.`));box.appendChild(button);j.row.appendChild(box);exportUiState(j,'awaiting-consent','Approval required; no server job has been created.');j.resolveConsent=accept=>{box.remove();j.resolveConsent=null;resolve(accept);};button.addEventListener('click',()=>j.resolveConsent?.(true));});} +function exportUiOffer(j,v){return new Promise(resolve=>{j.finishReady=resolve;j.url=`/api/exports/download?id=${j.serverId}`;const expires=Date.parse(v.expires_at),remaining=expires-Date.now();if(remaining<=0){exportUiFinish(j,'expired','Prepared export expired');return;}j.save=exportUiNode('button','Download prepared video');j.save.type='button';j.save.className='btn btn-primary';j.save.addEventListener('click',()=>{if(j.finished||j.canceling||j.handed)return;if(Date.now()>=expires){exportUiFinish(j,'expired','Prepared export expired');return;}const link=exportUiNode('a');link.href=j.url;link.download=`export-${j.serverId}.mp4`;try{document.body.appendChild(link);link.click();}catch(_){exportUiState(j,'ready','Browser download request failed; retry before expiry');return;}finally{link.remove();}j.handed=true;j.save.disabled=true;exportUiState(j,'handed','Browser download requested; disk saving is not verified. Release after the browser accepts it.');});j.row.appendChild(j.save);j.cancel.textContent='Release prepared file / continue';j.cancel.disabled=false;const span=j.approved?`Approved video interval: ${exportBoundaryLabel(new Date(j.approved.export_start))} to ${exportBoundaryLabel(new Date(j.approved.export_end))}. `:'';exportUiState(j,'ready',`${j.mode==='copy'?'Original quality / copy':'Accurate cut / re-encoded'}: Ready on server, ${(v.bytes/1048576).toFixed(1)} MiB. ${span}Download before ${new Date(expires).toLocaleString()}. Nothing saved by this page; disk saving is not verified.`);j.expiryTimer=setTimeout(()=>{exportUiFinish(j,'expired','Prepared export reached expiry; an already-started transfer may continue.');j.cancel.disabled=true;},Math.min(remaining,2147483647));});} +async function exportUiRunJob(j){if(j.reason==='canceled')return;try{let result,accepted={};for(let attempt=0;attempt<4;attempt++){j.creating=true;exportUiState(j,'preparing','Requesting server preparation');try{result=await exportUiRequest('/api/exports/create',{camera_id:j.id,start_time:j.start,end_time:j.end,mode:j.mode,...accepted});}finally{j.creating=false;}if(result.status!==409)break;if(j.reason==='canceled'){exportUiFinish(j,'canceled','Canceled before a server job was created');j.cancel.disabled=true;return;}if(attempt===3)throw Error('Source boundaries keep changing; no automatic approval was made');if(!await exportUiConsent(j,result.value))return;if(j.reason==='canceled')return;j.approved=result.value;accepted={accepted_start:result.value.export_start,accepted_end:result.value.export_end};}if(result.status!==202)throw Error('Expected job acceptance');let v=exportUiView(result.value);j.serverId=v.id;if(j.reason==='canceled'){await exportUiCancel(j);return;}const deadline=Date.now()+EXPORT_JOB_WAIT_MS;while(true){if(j.reason==='canceled')return;if(v.state==='ready'){await exportUiOffer(j,v);return;}if(['failed','canceled','expired'].includes(v.state)){exportUiFinish(j,v.state,exportUiText(v.message));j.cancel.disabled=true;return;}exportUiState(j,v.state,v.state==='queued'?'Queued on server':'Preparing on server');if(Date.now()>=deadline)throw Error('Stopped waiting after two hours; the server job may still exist');await exportUiWait(j);if(j.reason==='canceled')return;result=await exportUiRequest(`/api/exports/status?id=${j.serverId}`);if(j.reason==='canceled')return;if(result.status!==200)throw Error('Unexpected status response');v=exportUiView(result.value,j.serverId);}}catch(e){if(j.reason==='canceled'&&j.serverId){await exportUiCancel(j);return;}exportUiFinish(j,'failed',`${e.message} ${j.serverId?'Use cancellation below or wait for expiry.':'Creation may be unknown if the response was lost; no automatic retry.'}`);j.cancel.disabled=!j.serverId;if(j.serverId)j.cancel.textContent='Cancel server job';}} +function startExportDownloads(){if(exportUiBatch?.running)return exportUiBatch.promise;const host=document.getElementById('download-status'),list=document.getElementById('download-progress-list');if(!host||!list)return Promise.resolve();if(exportUiBatch?.jobs.some(j=>j.state==='release-failed'||(j.serverId&&j.state==='failed'&&!j.cancel.disabled)))return exportUiBatch.promise;host.style.display='block';list.replaceChildren();const a=downloadStartTime?.getTime(),b=downloadEndTime?.getTime(),ids=[...new Set(downloadCameras)],mode=document.getElementById('download-export-mode')?.value||'copy';if(!Number.isFinite(a)||!Number.isFinite(b)||b<=a||ids.length<1||ids.length>8||!['copy','accurate'].includes(mode)){list.appendChild(exportUiNode('p','Choose a valid range and one to eight cameras'));return Promise.resolve();}const cameras=ids.map(id=>allCameras.find(c=>c.id===id));if(cameras.some(c=>!c||typeof c.id!=='string'||c.id.length>256)){list.appendChild(exportUiNode('p','Selected camera unavailable'));return Promise.resolve();}const stop=exportUiNode('button','Cancel all exports');stop.type='button';stop.addEventListener('click',()=>{void cancelExportDownloads();});list.appendChild(stop);list.appendChild(exportUiNode('p','Files are prepared on server storage, not buffered in this page. Server size limits apply. The next camera starts after release or expiry. Copy never silently becomes a re-encode.'));const jobs=cameras.map(c=>{const row=exportUiNode('div');row.className='export-job';row.dataset.cameraId=c.id;const status=exportUiNode('p','Queued'),cancel=exportUiNode('button','Cancel');cancel.type='button';status.setAttribute('aria-live','polite');row.appendChild(exportUiNode('h4',String(c.name??c.id).slice(0,256)));row.appendChild(status);row.appendChild(cancel);list.appendChild(row);const j={id:c.id,start:new Date(a).toISOString(),end:new Date(b).toISOString(),mode,row,status,cancel,state:'queued',reason:null,serverId:null,creating:false,canceling:false,finished:false,handed:false,url:null,save:null,finishReady:null,wake:null,resolveConsent:null,approved:null};cancel.addEventListener('click',()=>{void exportUiCancel(j);});return j;});const batch={jobs,running:true,promise:null};exportUiBatch=batch;updateDownloadTimeDisplay();const button=document.getElementById('export-start-btn');if(button)button.disabled=true;batch.promise=(async()=>{try{for(const j of jobs)await exportUiRunJob(j);}finally{batch.running=false;if(button)button.disabled=false;updateDownloadTimeDisplay();}})();return batch.promise;} +// Keep handed-off downloads alive across page navigation; server TTL cleans jobs. if(document.readyState==='loading')document.addEventListener('DOMContentLoaded',installExportRangeControls);else installExportRangeControls(); diff --git a/tests/export-mode-ui.cjs b/tests/export-mode-ui.cjs new file mode 100644 index 0000000..fd07812 --- /dev/null +++ b/tests/export-mode-ui.cjs @@ -0,0 +1,11 @@ +const assert=require('node:assert/strict');const {environment,view,response}=require('./export-ui-fixture.cjs'); +const proposal={code:'copy_range_approval_required',requested_start:'2026-11-21T00:00:00.000Z',requested_end:'2026-11-21T00:01:00.000Z',export_start:'2026-11-20T23:59:59Z',export_end:'2026-11-21T00:01:01Z',audio_allowance_ms:64}; +function approve(env){const search=e=>{if(e.tagName==='button'&&e.textContent==='Use proposed copy range')return e;for(const child of e.children){const found=search(child);if(found)return found;}return null;};const button=search(env.find('download-progress-list'));assert.ok(button);button.click();} +(async()=>{let creates=0;const env=environment((url)=>Promise.resolve(url==='/api/exports/create'?(++creates===1?response(409,proposal):response(202,view('a'))):response(204))); + env.run('installExportRangeControls()');assert.equal(env.find('download-export-mode').value,'copy');const batch=env.run('startExportDownloads()');await env.flush();assert.equal(env.jobs()[0].state,'awaiting-consent');assert.equal(creates,1);assert.equal(env.jobs()[0].serverId,null);assert.ok(env.find('download-progress-list').textContent.includes('64 ms'));assert.equal(env.find('download-export-mode').disabled,true); + approve(env);await env.flush();assert.equal(creates,2);const body=JSON.parse(env.requests[1].options.body);assert.equal(body.mode,'copy');assert.equal(body.start_time,proposal.requested_start);assert.equal(body.accepted_start,proposal.export_start);assert.equal(body.accepted_end,proposal.export_end);assert.equal(env.jobs()[0].state,'ready');await env.run('cancelExportDownloads()');await batch; + const canceled=environment(()=>Promise.resolve(response(409,proposal)));const canceledBatch=canceled.run('startExportDownloads()');await canceled.flush();await canceled.run('cancelExportDownloads()');await canceledBatch;assert.equal(canceled.requests.length,1);assert.equal(canceled.jobs()[0].state,'canceled'); + const accurate=environment(()=>Promise.resolve(response(202,view('b'))));accurate.run('installExportRangeControls()');accurate.find('download-export-mode').value='accurate';const accurateBatch=accurate.run('startExportDownloads()');await accurate.flush();assert.equal(JSON.parse(accurate.requests[0].options.body).mode,'accurate');assert.ok(accurate.jobs()[0].status.textContent.includes('re-encoded'));await accurate.run('cancelExportDownloads()');await accurateBatch; + const bad=environment(()=>Promise.resolve(response(409,{...proposal,export_start:'2026-11-21T00:00:01Z'})));await bad.run('startExportDownloads()');assert.equal(bad.jobs()[0].state,'failed');assert.equal(bad.requests.length,1);assert.equal(bad.downloads.length,0); + console.log('PASS explicit modes, immutable range approval, no pre-approval job, consent cancellation, invalid proposal refusal'); +})().catch(e=>{console.error(e);process.exitCode=1;}); From e0cd671c37714fa0a3b51e9b7b3cc5406e3ffac9 Mon Sep 17 00:00:00 2001 From: mouldybread <38598858+mouldybread@users.noreply.github.com> Date: Sat, 21 Nov 2026 12:33:09 +0800 Subject: [PATCH 7/8] Run full Go verification on export branch and add prepared protocol coverage --- .github/workflows/prepared-export-go.yml | 51 ++++++++++++++++++++ docs/PREPARED_EXPORT_VERIFICATION.md | 11 +++++ export_keyframes_native_test.go | 35 ++++++++++++++ tests/prepared_export_protocol.py | 51 ++++++++++++++++++++ tests/test_prepared_export_protocol.py | 59 ++++++++++++++++++++++++ 5 files changed, 207 insertions(+) create mode 100644 .github/workflows/prepared-export-go.yml create mode 100644 docs/PREPARED_EXPORT_VERIFICATION.md create mode 100644 export_keyframes_native_test.go create mode 100644 tests/prepared_export_protocol.py create mode 100644 tests/test_prepared_export_protocol.py diff --git a/.github/workflows/prepared-export-go.yml b/.github/workflows/prepared-export-go.yml new file mode 100644 index 0000000..2785f4f --- /dev/null +++ b/.github/workflows/prepared-export-go.yml @@ -0,0 +1,51 @@ +name: Prepared export Go verification + +on: + push: + branches: [fix/large-media-exports] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + go: + name: Full Go build, race suite and vet + runs-on: ubuntu-24.04 + timeout-minutes: 25 + env: + GOFLAGS: -mod=readonly + SIMPLENVR_REQUIRE_FFMPEG: '1' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache: false + - uses: actions/setup-node@v4 + with: + node-version: '22' + - run: bash .github/scripts/ci-media-apt.sh update + - run: bash .github/scripts/ci-media-apt.sh install ffmpeg + - name: Prepared protocol regression tests + run: python3 -m unittest discover -s tests -p 'test_prepared_export_protocol.py' -v + - name: Verify dependencies and build the exact checkout + run: | + go mod download + go mod verify + git diff --exit-code -- go.mod go.sum + go build ./... + - name: Full race-enabled Go suite including Node and native media tests + run: bash tests/core-ci-evidence.sh "$RUNNER_TEMP/prepared-export-go" + - name: Static analysis + if: always() + run: go vet ./... + - name: Retain Go execution evidence even on failure + if: always() + uses: actions/upload-artifact@v4 + with: + name: prepared-export-go-${{ github.sha }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/prepared-export-go + if-no-files-found: warn + retention-days: 14 diff --git a/docs/PREPARED_EXPORT_VERIFICATION.md b/docs/PREPARED_EXPORT_VERIFICATION.md new file mode 100644 index 0000000..671336b --- /dev/null +++ b/docs/PREPARED_EXPORT_VERIFICATION.md @@ -0,0 +1,11 @@ +# Prepared export verification status + +The prepared-export Go workflow triggers on pushes to fix/large-media-exports, pull requests and manual dispatch. It installs the Go version from go.mod, Node 22 and FFmpeg; verifies dependency files; builds the exact checkout; runs the existing complete race-enabled core evidence command (not a filtered export-only subset); and runs go vet. It retains the Go execution evidence with the commit SHA and run attempt. A missing dependency/build/test is not treated as a pass. + +New native Go tests exercise non-reordered H264/AAC keyframe planning, actual copy rendering, external decode, duration and source immutability, plus reordered-video refusal. The required-media environment makes missing FFmpeg/Node a failure rather than an optional local skip. + +Six standard-library protocol regressions passed locally: no POST retry after an uncertain create, queued/preparing/ready transitions, terminal failure refusal, status/control-size rejection, release acknowledgement plus HTTP 404, and job/download URL binding. These tests do NOT prove the live backend or browser. + +Go and Docker are unavailable in the assistant execution environment. The Go/native suites have not run there; CI must produce the actual results after this commit is pushed. This workflow is not a release-qualification claim. + +The live-browser migration is the next separate commit: mount private export scratch outside recordings, preserve existing native playback/diagnostic checks, use prepared create/status/download/release APIs, exercise explicit copy proposal approval/cancellation, validate accurate AAC output, retain saved-download hashes/duration/external decode and active-render cancellation/reaping. Its migration draft has only passed Python/shell syntax checks here; no live-container browser pass is claimed. The older routed-fixture browser-export-acceptance.py also remains a separate legacy harness and must not be reported as migrated. diff --git a/export_keyframes_native_test.go b/export_keyframes_native_test.go new file mode 100644 index 0000000..204dfdb --- /dev/null +++ b/export_keyframes_native_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" + "os/exec" + "path/filepath" + "testing" + "time" +) + +func TestPreparedKeyframeCopyNativeRoundTrip(t *testing.T) { + requireFFmpeg(t) + ctx,cancel:=context.WithTimeout(context.Background(),40*time.Second);defer cancel() + source:=filepath.Join(t.TempDir(),"source.mp4") + args:=[]string{"-nostdin","-v","error","-f","lavfi","-i","testsrc2=size=128x96:rate=20","-f","lavfi","-i","sine=frequency=440:sample_rate=16000","-t","2","-c:v","libx264","-threads","1","-bf","0","-g","20","-sc_threshold","0","-pix_fmt","yuv420p","-c:a","aac","-y",source} + if raw,err:=exec.CommandContext(ctx,"ffmpeg",args...).CombinedOutput();err!=nil{t.Fatalf("fixture: %v %s",err,raw)} + before:=preparedMediaDigest(t,source) + requested,pins:=preparedMediaTestPlan(t,ctx,source,1,500*time.Millisecond,1500*time.Millisecond) + actual,indices,err:=planKeyframeCopy(ctx,pins,requested);if err!=nil{t.Fatal(err)} + if !actual.Start.Equal(requested.MediaStart) || !actual.End.Equal(requested.MediaEnd){t.Fatalf("unexpected keyframe interval: %+v",actual)} + output:=preparedMediaOutput(t) + if err:=renderKeyframeCopy(ctx,pins,actual,indices,output,16<<20);err!=nil{t.Fatal(err)} + decodePreparedMedia(t,ctx,output.Name()) + duration,err:=probeVideoDuration(ctx,output);if err!=nil{t.Fatal(err)} + if delta:=duration-2*time.Second;delta < -100*time.Millisecond || delta > 100*time.Millisecond{t.Fatal("wrong duration",duration)} + if preparedMediaDigest(t,source)!=before{t.Fatal("source changed")} +} + +func TestPreparedKeyframeCopyRefusesReorderedVideo(t *testing.T) { + requireFFmpeg(t) + ctx,cancel:=context.WithTimeout(context.Background(),30*time.Second);defer cancel() + source:=makePreparedMediaSource(t,ctx,false) + requested,pins:=preparedMediaTestPlan(t,ctx,source,1,500*time.Millisecond,1500*time.Millisecond) + if _,_,err:=planKeyframeCopy(ctx,pins,requested);err!=errCopyReordered{t.Fatalf("reordered subsection copy not refused: %v",err)} +} diff --git a/tests/prepared_export_protocol.py b/tests/prepared_export_protocol.py new file mode 100644 index 0000000..96c5c5e --- /dev/null +++ b/tests/prepared_export_protocol.py @@ -0,0 +1,51 @@ +"""Prepared-export control protocol assertions shared by acceptance checks.""" +import asyncio +import re +import time + + +def job_view(value, expected=None): + assert isinstance(value, dict) and re.fullmatch(r"[0-9a-f]{64}", value.get("id", "")), value + assert expected is None or value["id"] == expected, value + assert value.get("state") in ("queued", "preparing", "ready", "failed", "canceled", "expired"), value + if value["state"] == "ready": + assert type(value.get("bytes")) is int and value["bytes"] > 0, value + assert value.get("download_url") == "/api/exports/download?id=" + value["id"], value + return value + + +async def control(request, origin, path, body=None, expected=200): + if body is None: + response = await request.get(origin + path, timeout=90000) + else: + response = await request.post(origin + path, data=body, timeout=90000) + assert response.status == expected, (path, response.status, (await response.text())[:1000]) + if expected == 204: + return None + assert response.headers.get("content-type", "").split(";")[0] == "application/json", path + raw = await response.body() + assert len(raw) <= 8192, "control response exceeded bound" + return await response.json() + + +async def ready(request, origin, value, budget=90): + value = job_view(value) + deadline = time.monotonic() + budget + while value["state"] != "ready": + assert value["state"] in ("queued", "preparing"), value + assert time.monotonic() < deadline, "job readiness deadline exceeded" + await asyncio.sleep(0.15) + value = job_view(await control(request, origin, "/api/exports/status?id=" + value["id"]), value["id"]) + return value + + +async def create_ready(request, origin, body): + # No automatic POST retry: an uncertain creation must not duplicate a job. + return await ready(request, origin, await control(request, origin, "/api/exports/create", body, 202)) + + +async def release(request, origin, value): + await control(request, origin, "/api/exports/cancel", {"id": value["id"]}, 204) + response = await request.get(origin + value["download_url"]) + assert response.status == 404, "released file still admits a new reader" + assert response.headers.get("content-type", "").split(";")[0] != "video/mp4" diff --git a/tests/test_prepared_export_protocol.py b/tests/test_prepared_export_protocol.py new file mode 100644 index 0000000..2187c88 --- /dev/null +++ b/tests/test_prepared_export_protocol.py @@ -0,0 +1,59 @@ +import unittest +from unittest.mock import AsyncMock, patch +from types import SimpleNamespace +from prepared_export_protocol import control, create_ready, job_view, release + + +def response(status, value=None, raw=b"{}"): + return SimpleNamespace(status=status, headers={"content-type":"application/json"}, + text=AsyncMock(return_value="response"), body=AsyncMock(return_value=raw), json=AsyncMock(return_value=value)) + + +def view(state="ready"): + return {"id":"a"*64,"state":state,"bytes":42,"download_url":"/api/exports/download?id="+"a"*64} + + +class ProtocolTests(unittest.IsolatedAsyncioTestCase): + async def test_creation_not_retried_on_lost_response(self): + request=SimpleNamespace(post=AsyncMock(side_effect=OSError("connection lost"))) + with self.assertRaises(OSError): + await create_ready(request,"https://nvr",{"mode":"copy"}) + self.assertEqual(request.post.await_count,1) + + async def test_queued_preparing_ready(self): + request=SimpleNamespace(post=AsyncMock(return_value=response(202,view("queued"))), + get=AsyncMock(side_effect=[response(200,view("preparing")),response(200,view())])) + with patch("prepared_export_protocol.asyncio.sleep",new=AsyncMock()): + got=await create_ready(request,"https://nvr",{"mode":"accurate"}) + self.assertEqual(got["state"],"ready") + self.assertEqual(request.post.await_count,1) + self.assertEqual(request.get.await_count,2) + + async def test_failure_is_not_ready(self): + request=SimpleNamespace(post=AsyncMock(return_value=response(202,view("failed")))) + with self.assertRaises(AssertionError): + await create_ready(request,"https://nvr",{}) + + async def test_wrong_status_and_oversized_control_fail(self): + for answer in (response(503),response(200,{},b"x"*8193)): + with self.subTest(status=answer.status): + with self.assertRaises(AssertionError): + await control(SimpleNamespace(get=AsyncMock(return_value=answer)),"https://nvr","/status") + + async def test_release_requires_ack_and_http_404(self): + for status in (200,404): + request=SimpleNamespace(post=AsyncMock(return_value=response(204)),get=AsyncMock(return_value=response(status))) + if status==404: + await release(request,"https://nvr",view()) + else: + with self.assertRaises(AssertionError): await release(request,"https://nvr",view()) + bad=SimpleNamespace(post=AsyncMock(return_value=response(503))) + with self.assertRaises(AssertionError): await release(bad,"https://nvr",view()) + + def test_ready_url_and_job_identity_are_bound(self): + for bad in ({**view(),"download_url":"blob:dead"},{**view(),"id":"bad"},{**view(),"bytes":0}): + with self.assertRaises(AssertionError):job_view(bad) + with self.assertRaises(AssertionError):job_view(view(),"b"*64) + + +if __name__=="__main__":unittest.main() From 8cb9ff7a2c7346724ea0ee102ab14bbaad691913 Mon Sep 17 00:00:00 2001 From: mouldybread <38598858+mouldybread@users.noreply.github.com> Date: Sat, 21 Nov 2026 12:41:57 +0800 Subject: [PATCH 8/8] Migrate live browser export acceptance to prepared jobs and run branch CI --- .github/workflows/prepared-export-browser.yml | 45 +++ docs/EXPORT_API_MODES.md | 16 +- docs/PREPARED_EXPORT_VERIFICATION.md | 12 +- tests/runtime-browser-acceptance.py | 334 ++++++++++-------- tests/runtime-browser-acceptance.sh | 21 +- 5 files changed, 268 insertions(+), 160 deletions(-) create mode 100644 .github/workflows/prepared-export-browser.yml diff --git a/.github/workflows/prepared-export-browser.yml b/.github/workflows/prepared-export-browser.yml new file mode 100644 index 0000000..59df41c --- /dev/null +++ b/.github/workflows/prepared-export-browser.yml @@ -0,0 +1,45 @@ +name: Prepared export browser verification + +on: + push: + branches: [fix/large-media-exports] + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + browser: + name: Prepared jobs and native playback — attempt ${{ matrix.attempt }} + runs-on: ubuntu-24.04 + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + attempt: [1, 2] + steps: + - uses: actions/checkout@v4 + - run: bash .github/scripts/ci-media-apt.sh update + - run: bash .github/scripts/ci-media-apt.sh install ffmpeg python3-venv openssl + - name: Build the exact checkout runtime image + run: docker build -t simplenvr-export-test . + - name: Install pinned Chrome tooling + run: | + python3 -m venv "$RUNNER_TEMP/export-browser-venv" + . "$RUNNER_TEMP/export-browser-venv/bin/activate" + python -m pip install 'playwright==1.56.0' + python -m playwright install --with-deps chrome + - name: Protocol regressions and real HTTPS/browser acceptance + run: | + . "$RUNNER_TEMP/export-browser-venv/bin/activate" + python -m unittest discover -s tests -p 'test_prepared_export_protocol.py' -v + bash tests/runtime-browser-acceptance.sh simplenvr-export-test "$RUNNER_TEMP/prepared-export-browser" + - name: Retain media, network, browser and container evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: prepared-export-browser-${{ github.sha }}-${{ github.run_attempt }}-${{ matrix.attempt }} + path: ${{ runner.temp }}/prepared-export-browser + if-no-files-found: warn + retention-days: 14 diff --git a/docs/EXPORT_API_MODES.md b/docs/EXPORT_API_MODES.md index 41910bf..c345d6d 100644 --- a/docs/EXPORT_API_MODES.md +++ b/docs/EXPORT_API_MODES.md @@ -1,15 +1,13 @@ -# Export modes: current integration status +# Export modes: integration and verification status -The HTTP planner calls the prepared copy/accurate renderers and the browser offers both modes. This status supersedes older staged progress notes saying those paths are not yet wired. The branch is still NOT release-ready: Go compilation/tests, real-browser harness migration and full runtime validation remain outstanding. +The HTTP planner and browser are wired to prepared copy/accurate exports. Copy is the default and never silently switches to re-encoding. Whole single-source copies preserve original bytes in prepared storage; compatible multiple sources preserve compressed A/V payloads when remuxed. This is not zero-copy original-file HTTP delivery. -Copy is the default and never silently falls back to re-encoding. Whole recordings are copied byte for byte into prepared storage (single source) or remuxed with compatible compressed A/V payloads (multiple sources). This is not yet zero-copy original-file HTTP delivery. +Subsection copy supports non-reordered video with usable decoded keyframe timestamps. The server proposes video boundaries at keyframe/file boundaries, reinspects expanded coverage and returns copy_range_approval_required before admitting a job. The UI shows the requested/proposed intervals and AAC packet allowance and waits for explicit approval. Canceling a proposal creates no job. Approval retains the original selection and sends accepted_start/end for fresh server checks. Changed proposals require another click. Unsupported copy must be changed to Accurate explicitly. -Subsection copy is limited to non-reordered video with usable decoded keyframe timestamps. The server rounds video boundaries to keyframe/file boundaries, reinspects expanded coverage and returns a 409 JSON copy_range_approval_required proposal BEFORE job admission/storage reservation. The UI displays requested and proposed local dates/times and AAC packet allowance, and waits for Use proposed copy range. Canceling the proposal creates no job. Approval sends accepted_start/end with the immutable original range; the server recomputes the proposal and refuses stale boundaries. Changed proposals require another click. Unsupported/reordered subsection copy must be changed to Accurate explicitly. +AAC packet edges may extend outside the video-aligned interval by the disclosed allowance. Timing is metadata/frame/packet granular, not calibrated camera-clock time. Candidate copy outputs are duration-probed and externally decoded before readiness; this is not a proof of every source frame/sample or colour value. -AAC packet edges can extend outside the video-aligned interval by the disclosed packet allowance. These are metadata-derived/frame-packet granular boundaries, not calibrated camera-clock claims. Copy candidates are duration-probed and externally decoded before readiness; this does not certify every source frame/sample or colour value. +Accurate explicitly re-encodes video with libx264 CRF 18 and audio with AAC. Both tracks share the video/cut origin; supported audio offsets are preserved. Full-range input becomes limited-range yuv420p, so accurate exports are not lossless or byte-identical. Leading/trailing silence can maintain the selected window without stretching audio. Mixed audio configurations, extra tracks, rotation and higher bit depths remain unsupported. -Accurate mode explicitly re-encodes video (libx264 CRF 18) and AAC audio. It shares the video timestamp/cut origin, preserves supported audio offset, and converts full-range input to limited-range yuv420p. It is not lossless or byte-identical. Missing leading/trailing audio may be represented as silence to maintain the selected window, without audio stretching. Mixed audio configurations, extra tracks, rotation and higher bit depths remain unsupported. +The live-container browser acceptance harness has been migrated to prepared jobs, real downloads and server release. Its container now mounts private export scratch. Branch-triggered Go and two-attempt Chrome workflows test the same pushed commit. See PREPARED_EXPORT_VERIFICATION.md for assertions and the distinction between local helper checks and actual CI results. The older routed-fixture browser script remains separate and unmigrated. -The UI uses normal authenticated download URLs, not video Blobs. Cancellation/release requires server acknowledgement. Scratch configuration, server limits, session isolation and lease lifetime remain documented in LARGE_MEDIA_EXPORTS.md. - -New focused mode/approval UI regressions passed in Node in UTC and America/Chicago using a DOM fixture; syntax checks also passed. Go tests are added but have not been executed here. Old real-browser harnesses still contain Blob-era assumptions and need migration while retaining their media, hash, duration, playback and cancellation assertions. Large-file, slow-client, full-resolution real-camera and end-to-end API/media/browser validation must pass before completion is claimed. +Release readiness is NOT established until both workflows pass on the final commit and failures are resolved. Nothing in this change merges, deploys or certifies real-camera/full-resolution/multi-GiB operation. Scratch settings, quotas, session isolation and leases remain described in LARGE_MEDIA_EXPORTS.md. diff --git a/docs/PREPARED_EXPORT_VERIFICATION.md b/docs/PREPARED_EXPORT_VERIFICATION.md index 671336b..8f3e60b 100644 --- a/docs/PREPARED_EXPORT_VERIFICATION.md +++ b/docs/PREPARED_EXPORT_VERIFICATION.md @@ -1,11 +1,11 @@ -# Prepared export verification status +# Prepared export verification -The prepared-export Go workflow triggers on pushes to fix/large-media-exports, pull requests and manual dispatch. It installs the Go version from go.mod, Node 22 and FFmpeg; verifies dependency files; builds the exact checkout; runs the existing complete race-enabled core evidence command (not a filtered export-only subset); and runs go vet. It retains the Go execution evidence with the commit SHA and run attempt. A missing dependency/build/test is not treated as a pass. +Both prepared-export workflows trigger on pushes to fix/large-media-exports, pull requests and manual dispatch. A push runs the full Go build/race/vet workflow and two independent Chrome acceptance attempts against the same event commit. Require both workflows to pass on that exact commit before completion is claimed. No tests were replaced by an export-only Go subset. -New native Go tests exercise non-reordered H264/AAC keyframe planning, actual copy rendering, external decode, duration and source immutability, plus reordered-video refusal. The required-media environment makes missing FFmpeg/Node a failure rather than an optional local skip. +The live HTTPS browser harness now uses create/status/download/cancel endpoints. It checks anonymous access refusal, whole-copy compressed video packet hashes, external decode/duration, HEAD and Range, missing-coverage refusal, accurate AAC output and duration, copy proposal cancellation and explicit approval, active accurate-render cancellation and child reaping, browser-saved SHA-256/duration/decode, and HTTP 404 after server release. It no longer expects Blob URL revocation or fetches video into page JavaScript for hashing. Test-directed range/seek state is retained; application functions and response bodies are not replaced. -Six standard-library protocol regressions passed locally: no POST retry after an uncertain create, queued/preparing/ready transitions, terminal failure refusal, status/control-size rejection, release acknowledgement plus HTTP 404, and job/download URL binding. These tests do NOT prove the live backend or browser. +The container has a private scratch bind mount at /exports, outside /recordings, with explicit small test quotas. Non-reordered fixtures exercise supported subsection copy; reordered inputs remain covered by Go refusal tests. Native playback, real login/navigation diagnostics and the expanded playback-edge suite are retained. Reports include the commit, image identity, browser version, media and network evidence. Passing synthetic acceptance is not production-camera or multi-GiB qualification. -Go and Docker are unavailable in the assistant execution environment. The Go/native suites have not run there; CI must produce the actual results after this commit is pushed. This workflow is not a release-qualification claim. +Local checks: Python/shell/YAML syntax; all 30 embedded browser-JavaScript expressions parsed by Node; media helper check copied 160 video packets without hash changes and externally decoded a 16-second fixture. The earlier six protocol regressions passed. These are not a live-backend or browser pass. Go and Docker are unavailable locally; actual Go/Chrome/container results must come from CI after this commit is pushed. -The live-browser migration is the next separate commit: mount private export scratch outside recordings, preserve existing native playback/diagnostic checks, use prepared create/status/download/release APIs, exercise explicit copy proposal approval/cancellation, validate accurate AAC output, retain saved-download hashes/duration/external decode and active-render cancellation/reaping. Its migration draft has only passed Python/shell syntax checks here; no live-container browser pass is claimed. The older routed-fixture browser-export-acceptance.py also remains a separate legacy harness and must not be reported as migrated. +The older routed-response tests/browser-export-acceptance.py is a separate legacy harness, not the real-runtime acceptance entrypoint. It has not been migrated or reported as passing here. Do not confuse that script with the migrated tests/runtime-browser-acceptance.py executed by both browser workflows. diff --git a/tests/runtime-browser-acceptance.py b/tests/runtime-browser-acceptance.py index ef5564b..6028a41 100644 --- a/tests/runtime-browser-acceptance.py +++ b/tests/runtime-browser-acceptance.py @@ -1,28 +1,45 @@ -"""Live HTTPS application/runtime-image acceptance; no routed responses or DOM stubs.""" +"""Real HTTPS application acceptance: native playback and prepared export jobs. +No application functions, media events or HTTP response bodies are replaced. +""" import argparse import asyncio import hashlib import json import os import subprocess -from pathlib import Path from collections import deque -from urllib.parse import urlencode, urlsplit +from pathlib import Path +from urllib.parse import urlsplit from playwright.async_api import async_playwright, Error as PlaywrightError from browser_ci_diagnostics import BrowserNetworkEvidence, observe_login_post +from prepared_export_protocol import create_ready, release BASE = "2026-11-10T12:00:" CAMERA = "id_browser" -def export_path(start, end): - return "/api/download?" + urlencode({"camera_id": CAMERA, "start_time": start, "end_time": end}) +def media_metadata(path): + subprocess.run(["ffmpeg", "-nostdin", "-v", "error", "-xerror", "-i", str(path), "-f", "null", "-"], check=True, timeout=30, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + result = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "stream=codec_type,codec_name,duration:format=duration", "-of", "json", str(path)], check=True, timeout=10, capture_output=True) + return json.loads(result.stdout) + + +def duration(metadata): + videos = [stream for stream in metadata["streams"] if stream["codec_type"] == "video"] + assert len(videos) == 1 and videos[0]["codec_name"] == "h264", metadata + return float(videos[0].get("duration", metadata["format"]["duration"])) def decode(path): - subprocess.run(["ffmpeg", "-nostdin", "-v", "error", "-i", str(path), "-f", "null", "-"], check=True, timeout=30, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) - result = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "json", str(path)], check=True, timeout=10, capture_output=True) - return float(json.loads(result.stdout)["format"]["duration"]) + # Preserve the baseline helper's float-returning interface for edge suites. + return duration(media_metadata(path)) + + +def packet_hashes(path): + result = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_packets", "-show_data_hash", "sha256", "-show_entries", "packet=data_hash", "-of", "json", str(path)], check=True, timeout=15, capture_output=True) + packets = json.loads(result.stdout)["packets"] + assert packets and all(p.get("data_hash", "").startswith("SHA256:") for p in packets) + return [p["data_hash"] for p in packets] def ffmpeg_pids(container): @@ -79,7 +96,6 @@ async def private_media_preflight(context, origin): async def submit_login_once(page, origin, report, network_events): root = origin + "/" - # Observe the real form POST; never resubmit credentials on a network error. response = await observe_login_post(page, origin, report) location = response.headers.get("location", "") report["login_response"] = {"status": response.status, "redirect_path": urlsplit(location).path} @@ -91,18 +107,16 @@ async def submit_login_once(page, origin, report, network_events): changed = any(event.get("event") == "requestfailed" and event.get("path") == "/" and event.get("method") == "GET" and event.get("resource_type") == "document" and event.get("failure") == "net::ERR_NETWORK_CHANGED" for event in network_events) if not changed: raise - # Only this observed error, after a proven successful POST, gets one - # GET recovery. A second failure propagates; there is no retry loop. report["login_navigation_recovery"] = {"method": "GET", "path": "/", "attempt": 1, "reason": "net::ERR_NETWORK_CHANGED", "completed": False} recovered = await page.goto(root, wait_until="load") - assert recovered is not None and recovered.status == 200 and page.url == root, "Login GET recovery did not return the authenticated root document" + assert recovered is not None and recovered.status == 200 and page.url == root report["login_navigation_recovery"]["completed"] = True roots = [event for event in network_events if event.get("event") == "response" and event.get("path") == "/" and event.get("method") == "GET" and event.get("resource_type") == "document"] - assert page.url == root and roots and roots[-1]["status"] == 200, "Authenticated root navigation did not complete successfully" + assert page.url == root and roots and roots[-1]["status"] == 200 async def application_ready(page, origin): - assert page.url == origin + "/", "Application readiness requires the authenticated root document" + assert page.url == origin + "/" await page.wait_for_function("typeof allCameras !== 'undefined' && allCameras.some(c => c.id === 'id_browser')") picker = page.locator("#timeline-date-picker") await picker.fill("2026-11-10") @@ -112,36 +126,173 @@ async def application_ready(page, origin): assert await page.locator("#video-grid video").count() == 1 -async def prepare_ui(page, start, end): +async def prepare_ui(page, start, end, mode="accurate"): if await page.locator("#download-modal").is_visible(): await page.locator("#download-modal .close").click() await page.locator("#download-btn").click() - # There are no short-range text inputs in this UI. Select a deterministic - # range through its real application state, without replacing functions. - await page.evaluate("([start,end]) => {downloadCameras=['id_browser']; downloadStartTime=new Date(start); downloadEndTime=new Date(end);}", [start, end]) + await page.evaluate("([start,end]) => {downloadCameras=['id_browser'];downloadStartTime=new Date(start);downloadEndTime=new Date(end);updateDownloadTimeDisplay();}", [start, end]) + await page.locator("#download-export-mode").select_option(mode) await page.locator("#export-start-btn").click() +async def native_playback(page, report): + checks = report["checks_passed"] + report["before_first_seek"] = await playback_snapshot(page) + await page.evaluate("() => {videoPlayers[0].muted=true;seekToTime(new Date('2026-11-10T12:00:01Z'));}") + await page.wait_for_function("videoPlayers[0]?.error || (videoPlayers[0]?.readyState >= 2 && videoPlayers[0].currentTime > 1.3 && !videoPlayers[0].paused)") + state = await playback_snapshot(page) + report["first_playback"] = state + video = state["players"][0] + assert video["error"] is None and video["readyState"] >= 2 and video["currentTime"] > 1.3 and not video["paused"], state + frames = await page.evaluate("() => new Promise((resolve,reject) => {const timer=setTimeout(()=>reject(new Error('No native decoded-frame callback within 5 seconds')),5000);videoPlayers[0].requestVideoFrameCallback((_,m) => {clearTimeout(timer);resolve({frames:m.presentedFrames,width:m.width,height:m.height});});})") + assert frames["frames"] > 0 and frames["width"] == 256 and frames["height"] == 144 + report["native_frame_witness"] = frames + checks.append("native-h264-playback-decoded-frames") + await page.evaluate("togglePlayPause()") + paused = await page.evaluate("videoPlayers[0].currentTime") + await page.wait_for_timeout(400) + assert await page.evaluate("videoPlayers[0].paused && !isPlaying") + assert abs(await page.evaluate("videoPlayers[0].currentTime") - paused) < 0.1 + await page.evaluate("togglePlayPause()") + await page.wait_for_function("before => !videoPlayers[0].paused && videoPlayers[0].currentTime > before + 0.2", arg=paused) + checks.append("application-pause-resume-with-native-clock") + await page.evaluate("() => {videoPlayers[0].currentTime=7.6;}") + await page.wait_for_function("videoPlayers[0]?.currentSrc.includes('20261110_120008') && videoPlayers[0].currentTime > 0.1 && !videoPlayers[0].paused") + checks.append("native-ended-contiguous-clip-transition") + await page.evaluate("() => {videoPlayers[0].src='/recordings/id_browser/missing.mp4';}") + await page.wait_for_function("!isPlaying && videoPlayers[0].error !== null") + await page.evaluate("seekToTime(new Date('2026-11-10T12:00:01Z'))") + await page.wait_for_function("videoPlayers[0]?.readyState >= 2 && !videoPlayers[0].paused && videoPlayers[0].currentTime > 1.2 && !videoPlayers[0].error") + checks.append("native-404-media-failure-and-application-retry") + await page.evaluate("togglePlayPause()") + + +async def exports(page, context, p, args, report): + checks, request = report["checks_passed"], context.request + selection = {"camera_id": CAMERA, "start_time": BASE + "00Z", "end_time": BASE + "16Z", "mode": "copy"} + anonymous = await p.request.new_context(ignore_https_errors=True) + try: + for path in ("/api/exports/status?id=" + "a"*64, "/api/exports/download?id=" + "a"*64, "/recordings/id_browser/20261110_120000_fixture.mp4"): + response = await anonymous.get(args.origin + path) + assert response.status == 401 + assert response.headers.get("content-type", "").split(";")[0] != "video/mp4" + for path, body in (("/api/exports/create", selection), ("/api/exports/cancel", {"id": "a"*64})): + assert (await anonymous.post(args.origin + path, data=body)).status == 401 + finally: + await anonymous.dispose() + checks.append("unauthenticated-prepared-routes-and-private-media-refused") + + whole = await create_ready(request, args.origin, selection) + response = await request.get(args.origin + whole["download_url"]) + assert response.status == 200 and response.headers["content-type"].split(";")[0] == "video/mp4" + raw = await response.body() + assert len(raw) == whole["bytes"] + saved = args.evidence / "runtime-whole-copy.mp4" + saved.write_bytes(raw) + assert abs(duration(await asyncio.to_thread(media_metadata, saved)) - 16) < 0.3 + sources = [] + for stamp in ("120000", "120008"): + source = args.evidence / (stamp + "-source.mp4") + fetched = await request.get(args.origin + f"/recordings/id_browser/20261110_{stamp}_fixture.mp4") + assert fetched.status == 200 + source.write_bytes(await fetched.body()) + sources.extend(await asyncio.to_thread(packet_hashes, source)) + assert await asyncio.to_thread(packet_hashes, saved) == sources, "whole-copy video payload changed" + head = await request.head(args.origin + whole["download_url"]) + assert head.status == 200 and int(head.headers["content-length"]) == len(raw) + part = await request.get(args.origin + whole["download_url"], headers={"Range": "bytes=0-11"}) + assert part.status == 206 and await part.body() == raw[:12] + assert part.headers["content-range"] == f"bytes 0-11/{len(raw)}" + await release(request, args.origin, whole) + checks.append("prepared-copy-packet-hashes-decode-head-range-and-release") + + response = await request.post(args.origin + "/api/exports/create", data={**selection, "start_time": BASE + "15Z", "end_time": BASE + "20Z"}) + assert response.status == 409 and response.headers.get("x-export-plan-status") == "gap" + assert response.headers.get("content-type", "").split(";")[0] != "video/mp4" + checks.append("prepared-gap-refused-without-download") + + audio = await create_ready(request, args.origin, {**selection, "mode": "accurate", "start_time": "2026-11-10T12:30:00.5Z", "end_time": "2026-11-10T12:30:03.5Z"}) + response = await request.get(args.origin + audio["download_url"]) + assert response.status == 200 + audio_path = args.evidence / "accurate-audio.mp4" + audio_path.write_bytes(await response.body()) + metadata = await asyncio.to_thread(media_metadata, audio_path) + assert abs(duration(metadata) - 3) < 0.3 + tracks = [s for s in metadata["streams"] if s["codec_type"] == "audio"] + assert len(tracks) == 1 and tracks[0]["codec_name"] == "aac" + assert abs(float(tracks[0]["duration"]) - 3) < 0.25 + await release(request, args.origin, audio) + checks.append("prepared-accurate-audio-track-duration-and-decode") + + start, end = BASE + "00.5Z", BASE + "15.5Z" + await prepare_ui(page, start, end, "copy") + await page.wait_for_selector('.export-job[data-state="awaiting-consent"]', timeout=90000) + assert await page.evaluate("exportUiBatch.jobs[0].serverId === null && exportUiBatch.jobs[0].url === null") + assert "AAC packet" in await page.locator(".export-job").inner_text() + await page.locator(".export-job").get_by_role("button", name="Cancel", exact=True).click() + await page.wait_for_function("!exportUiBatch.running && exportUiBatch.jobs[0].state === 'canceled'") + checks.append("real-copy-proposal-cancel-before-job") + + await prepare_ui(page, start, end, "copy") + await page.wait_for_selector('.export-job[data-state="awaiting-consent"]', timeout=90000) + async with page.expect_response(lambda r: urlsplit(r.url).path == "/api/exports/create" and r.status == 202) as accepted: + await page.get_by_role("button", name="Use proposed copy range", exact=True).click() + assert (await accepted.value).status == 202 + await page.wait_for_selector('.export-job[data-state="ready"]', timeout=90000) + assert await page.evaluate("exportUiBatch.jobs[0].mode") == "copy" + await page.get_by_role("button", name="Release prepared file / continue", exact=True).click() + await page.wait_for_function("!exportUiBatch.running") + checks.append("real-copy-proposal-explicit-approval-and-release") + + await prepare_ui(page, start, end, "accurate") + pids = await wait_pids(args.container, True) + await page.locator(".export-job").get_by_role("button", name="Cancel", exact=True).click() + await page.wait_for_selector('.export-job[data-state="canceled"]') + await page.wait_for_function("!exportUiBatch.running") + await wait_pids(args.container, False) + assert await page.evaluate("exportUiBatch.jobs[0].url") is None + report["canceled_runtime_ffmpeg_pids"] = pids + checks.append("ui-cancel-active-paced-runtime-render-and-child-reap") + + await prepare_ui(page, start, end, "accurate") + await page.wait_for_selector('.export-job[data-state="ready"]', timeout=90000) + assert "re-encoded" in await page.locator(".export-job").inner_text() + url = await page.evaluate("exportUiBatch.jobs[0].url") + assert url.startswith("/api/exports/download?id=") and not url.startswith("blob:") + response = await request.get(args.origin + url) + assert response.status == 200 + received_hash = hashlib.sha256(await response.body()).hexdigest() + async with page.expect_download() as pending: + await page.get_by_role("button", name="Download prepared video", exact=True).click() + download = await pending.value + assert await download.failure() is None + saved = args.evidence / "browser-saved-trim.mp4" + await download.save_as(saved) + assert hashlib.sha256(saved.read_bytes()).hexdigest() == received_hash + span = duration(await asyncio.to_thread(media_metadata, saved)) + assert abs(span - 15) < 0.3, span + assert "not verified" in await page.locator(".export-job").inner_text() + await page.get_by_role("button", name="Release prepared file / continue", exact=True).click() + await page.wait_for_function("!exportUiBatch.running && exportUiBatch.jobs[0].url === null") + assert (await request.get(args.origin + url)).status == 404 + await wait_pids(args.container, False) + checks.append("real-ui-save-hash-duration-decode-and-server-release") + report.update({"saved_trim_sha256": received_hash, "saved_trim_duration_seconds": span}) + + async def run(args): args.evidence.mkdir(parents=True, exist_ok=True) - report = {"commit": args.commit, "image_id": args.image_id, "backend": "real HTTPS runtime container", "routed_responses": False, "checks_passed": [], "status": "failed", "limits": "Synthetic H264/AAC files, headless Chromium, test-directed seek/range selection. No live RTSP, OS save dialog, late transport truncation, metadata-delay or autoplay-denial acceptance."} - checks = report["checks_passed"] - password = os.environ["SIMPLENVR_E2E_PASSWORD"] + report = {"commit": args.commit, "image_id": args.image_id, "backend": "real HTTPS runtime container", "routed_responses": False, "checks_passed": [], "status": "failed", "runtime_qualified": False, "cleanup_authorized": False, "limits": "Synthetic non-reordered H264/AAC; headless Chrome; deterministic state-directed ranges. No live RTSP, OS save dialogs, multi-GiB or full-resolution camera qualification."} async with BrowserNetworkEvidence(args.evidence, args.origin) as diagnostics, async_playwright() as p: browser = await p.chromium.launch(channel="chrome", headless=True) - report["browser"] = browser.version - report["browser_channel"] = "chrome" - report["playwright_version"] = diagnostics.playwright_version - report["network_diagnostics"] = "browser-network-evidence.json" + report.update({"browser": browser.version, "browser_channel": "chrome", "playwright_version": diagnostics.playwright_version, "network_diagnostics": "browser-network-evidence.json"}) context = await browser.new_context(ignore_https_errors=True, accept_downloads=True, timezone_id="UTC") - page = None - errors = [] - media_network = deque(maxlen=100) - def remember_response(response): + page, errors, media_network = None, [], deque(maxlen=100) + def remember(response): path = urlsplit(response.url).path if path.startswith('/recordings/') or path in ('/api/timeline/recordings', '/auth/login', '/'): media_network.append({"event":"response","path":path,"method":response.request.method,"status":response.status,"resource_type":response.request.resource_type}) - def remember_failure(request): + def failed(request): path = urlsplit(request.url).path if path.startswith('/recordings/') or path in ('/api/timeline/recordings', '/auth/login', '/'): media_network.append({"event":"requestfailed","path":path,"method":request.method,"resource_type":request.resource_type,"failure":request.failure}) @@ -149,117 +300,25 @@ def remember_failure(request): context.set_default_timeout(30000) page = await context.new_page() await diagnostics.attach(context, page) - report["codec_support"] = await page.evaluate("""() => { - const video=document.createElement('video'); - return {h264:video.canPlayType('video/mp4; codecs="avc1.64000a"'),aac:video.canPlayType('audio/mp4; codecs="mp4a.40.2"')}; - }""") - assert report["codec_support"]["h264"], "Selected Chrome channel lacks required H264 support: " + json.dumps(report["codec_support"]) - page.on("pageerror", lambda error: errors.append(str(error)[:1000]) if len(errors)<30 else None) - page.on("response", remember_response) - page.on("requestfailed", remember_failure) + report["codec_support"] = await page.evaluate("() => {const v=document.createElement('video');return {h264:v.canPlayType('video/mp4; codecs=\"avc1.64000a\"'),aac:v.canPlayType('audio/mp4; codecs=\"mp4a.40.2\"')};}") + assert report["codec_support"]["h264"] + page.on("pageerror", lambda e: errors.append(str(e)[:1000]) if len(errors) < 30 else None) + page.on("response", remember) + page.on("requestfailed", failed) await page.goto(args.origin + "/auth/login") await page.locator('[name="username"]').fill("ci-browser") - await page.locator('[name="password"]').fill(password) + await page.locator('[name="password"]').fill(os.environ["SIMPLENVR_E2E_PASSWORD"]) await submit_login_once(page, args.origin, report, media_network) - checks.append("real-login-form-session") + report["checks_passed"].append("real-login-form-session") await application_ready(page, args.origin) - checks.append("real-camera-template-and-timeline") + report["checks_passed"].append("real-camera-template-and-timeline") report["private_media_preflight"] = await private_media_preflight(context, args.origin) - checks.append("authenticated-private-media-range-preflight") - report["before_first_seek"] = await playback_snapshot(page) - - await page.evaluate("() => {videoPlayers[0].muted=true; seekToTime(new Date('2026-11-10T12:00:01Z'));}") - await page.wait_for_function("videoPlayers[0]?.error || (videoPlayers[0]?.readyState >= 2 && videoPlayers[0].currentTime > 1.3 && !videoPlayers[0].paused)") - state = await playback_snapshot(page) - report["first_playback"] = state - video = state["players"][0] - assert video["error"] is None and video["readyState"] >= 2 and video["currentTime"] > 1.3 and not video["paused"], state - frames = await page.evaluate("() => new Promise((resolve,reject) => {const timer=setTimeout(()=>reject(new Error('No native decoded-frame callback within 5 seconds')),5000);videoPlayers[0].requestVideoFrameCallback((_,m) => {clearTimeout(timer);resolve({frames:m.presentedFrames,width:m.width,height:m.height});});})") - assert frames["frames"] > 0 and frames["width"] == 256 and frames["height"] == 144 - checks.append("native-h264-playback-decoded-frames") - await page.evaluate("togglePlayPause()") - paused = await page.evaluate("videoPlayers[0].currentTime") - await page.wait_for_timeout(400) - assert await page.evaluate("videoPlayers[0].paused && !isPlaying") - assert abs(await page.evaluate("videoPlayers[0].currentTime") - paused) < 0.1 - await page.evaluate("togglePlayPause()") - await page.wait_for_function("before => !videoPlayers[0].paused && videoPlayers[0].currentTime > before + 0.2", arg=paused) - checks.append("application-pause-resume-with-native-clock") - await page.evaluate("() => {videoPlayers[0].currentTime=7.6;}") - await page.wait_for_function("videoPlayers[0]?.currentSrc.includes('20261110_120008') && videoPlayers[0].currentTime > 0.1 && !videoPlayers[0].paused") - checks.append("native-ended-contiguous-clip-transition") - # A real 404 and native media error, not a dispatched fake event. - await page.evaluate("() => {videoPlayers[0].src='/recordings/id_browser/missing.mp4';}") - await page.wait_for_function("!isPlaying && videoPlayers[0].error !== null") - await page.evaluate("seekToTime(new Date('2026-11-10T12:00:01Z'))") - await page.wait_for_function("videoPlayers[0]?.readyState >= 2 && !videoPlayers[0].paused && videoPlayers[0].currentTime > 1.2 && !videoPlayers[0].error") - checks.append("native-404-media-failure-and-application-retry") - await page.evaluate("togglePlayPause()") - - anonymous = await p.request.new_context(ignore_https_errors=True) - try: - for path in [export_path(BASE + "00Z", BASE + "16Z"), "/recordings/id_browser/20261110_120000_fixture.mp4"]: - response = await anonymous.get(args.origin + path) - assert response.status == 401 - assert response.headers.get("content-type", "").split(";")[0] != "video/mp4" - finally: - await anonymous.dispose() - checks.append("unauthenticated-export-and-private-media-refused") - - response = await context.request.get(args.origin + export_path(BASE + "00Z", BASE + "16Z"), timeout=90000) - assert response.status == 200, (response.status, await response.text()) - assert response.headers.get("x-export-range-policy") == "whole-clips-untrimmed" - whole = args.evidence / "runtime-whole-copy.mp4" - whole.write_bytes(await response.body()) - duration = await asyncio.to_thread(decode, whole) - assert abs(duration - 16) < 0.3, duration - checks.append("runtime-whole-clip-export-external-decode") - - for start, end, status, header, value in [ - (BASE + "15Z", BASE + "20Z", 409, "x-export-plan-status", "gap"), - ("2026-11-10T12:30:00.5Z", "2026-11-10T12:30:03.5Z", 422, "x-export-trim-status", "unsupported"), - ]: - response = await context.request.get(args.origin + export_path(start, end), timeout=30000) - assert response.status == status, (response.status, await response.text()) - assert response.headers.get(header) == value - assert response.headers.get("content-type", "").split(";")[0] != "video/mp4" - checks.append("runtime-gap-and-audio-trim-refusals") - - start, end = BASE + "00.5Z", BASE + "15.5Z" - await prepare_ui(page, start, end) - pids = await wait_pids(args.container, True) - await page.locator(".export-job").get_by_role("button", name="Cancel", exact=True).click() - await page.wait_for_selector('.export-job[data-state="canceled"]') - await page.wait_for_function("!exportUiBatch.running") - await wait_pids(args.container, False) - assert await page.evaluate("exportUiBatch.jobs[0].url") is None - report["canceled_runtime_ffmpeg_pids"] = pids - checks.append("ui-cancel-active-paced-runtime-render-and-child-reap") - - await prepare_ui(page, start, end) - await page.wait_for_selector('.export-job[data-state="ready"]', timeout=90000) - assert "Frame-granular trim" in await page.locator(".export-job").inner_text() - url = await page.evaluate("exportUiBatch.jobs[0].url") - received_hash = await page.evaluate("async url => {const bytes=await (await fetch(url)).arrayBuffer();return Array.from(new Uint8Array(await crypto.subtle.digest('SHA-256',bytes))).map(x=>x.toString(16).padStart(2,'0')).join('');}", url) - async with page.expect_download() as pending: - await page.get_by_role("button", name="Download prepared video", exact=True).click() - download = await pending.value - assert await download.failure() is None - saved = args.evidence / "browser-saved-trim.mp4" - await download.save_as(saved) - assert hashlib.sha256(saved.read_bytes()).hexdigest() == received_hash - duration = await asyncio.to_thread(decode, saved) - assert abs(duration - 15) < 0.3, duration - assert "not verified" in await page.locator(".export-job").inner_text() - await page.get_by_role("button", name="Release prepared file / continue", exact=True).click() - await page.wait_for_function("!exportUiBatch.running && exportUiBatch.jobs[0].url === null") - assert await page.evaluate("async url => {try{await fetch(url);return false;}catch(_){return true;}}", url) - await wait_pids(args.container, False) - checks.append("real-ui-trim-save-hash-duration-decode-and-object-url-release") - report.update({"saved_trim_sha256": received_hash, "saved_trim_duration_seconds": duration, "native_frame_witness": frames}) + report["checks_passed"].append("authenticated-private-media-range-preflight") + await native_playback(page, report) + await exports(page, context, p, args, report) assert not errors, errors report["status"] = "passed-with-navigation-recovery" if report.get("login_navigation_recovery") else "passed" - except Exception as error: + except BaseException as error: report["failure"] = type(error).__name__ + ": " + str(error)[:2000] if page is not None: try: @@ -268,17 +327,16 @@ def remember_failure(request): report["diagnostic_failure"] = str(diagnostic_error)[:1000] raise finally: - report["page_errors"] = errors - report["media_network"] = list(media_network) - print(json.dumps({key:report[key] for key in ["status", "checks_passed", "browser", "browser_channel", "playwright_version", "network_diagnostics", "codec_support", "login_submission", "login_response", "login_navigation_failure", "login_navigation_recovery", "failure", "failure_playback_snapshot", "private_media_preflight", "page_errors", "media_network"] if key in report}, indent=2), flush=True) + report["page_errors"], report["media_network"] = errors, list(media_network) (args.evidence / "runtime-browser-report.json").write_text(json.dumps(report, indent=2) + "\n") + print(json.dumps(report, indent=2), flush=True) await context.close() await browser.close() if __name__ == "__main__": parser = argparse.ArgumentParser() - for name in ["origin", "container", "commit", "image-id"]: + for name in ("origin", "container", "commit", "image-id"): parser.add_argument("--" + name, required=True) parser.add_argument("--evidence", type=Path, required=True) asyncio.run(run(parser.parse_args())) diff --git a/tests/runtime-browser-acceptance.sh b/tests/runtime-browser-acceptance.sh index a5232f1..5f42cba 100644 --- a/tests/runtime-browser-acceptance.sh +++ b/tests/runtime-browser-acceptance.sh @@ -5,9 +5,11 @@ evidence=${2:?evidence directory required} work=$(mktemp -d "${RUNNER_TEMP:-/tmp}/simplenvr-runtime-browser.XXXXXX") state=$work/state bin=$work/bin +scratch=$work/exports name=simplenvr-runtime-browser-$$ -mkdir -p "$state/id_browser" "$bin" "$evidence" +mkdir -p "$state/id_browser" "$bin" "$scratch" "$evidence" chmod 755 "$work" "$bin" +chmod 700 "$scratch" cleanup() { docker logs "$name" > "$evidence/runtime-container.log" 2>&1 || true docker rm -f "$name" >/dev/null 2>&1 || true @@ -16,12 +18,14 @@ cleanup() { trap cleanup EXIT timeout --kill-after=2 20 python3 -m unittest discover -s tests -p 'test_browser_playback*.py' -v \ 2>&1 | tee "$evidence/playback-edge-unit-tests.log" -# Generate fixtures using the actual runtime image codecs, not host FFmpeg. +# Deterministic non-reordered video exercises the supported subsection-copy path. +# The Go suite separately retains reordered-video refusal and accurate rendering. for stamp in 120000 120008; do docker run --rm --network none --entrypoint /usr/bin/ffmpeg \ --mount "type=bind,src=$state,dst=/recordings" "$image" \ -nostdin -v error -f lavfi -i testsrc2=size=256x144:rate=10 \ - -t 8 -c:v libx264 -threads 1 -g 10 -pix_fmt yuv420p -an -movflags +faststart \ + -t 8 -c:v libx264 -threads 1 -bf 0 -g 10 -sc_threshold 0 \ + -pix_fmt yuv420p -an -movflags +faststart \ -y "/recordings/id_browser/20261110_${stamp}_fixture.mp4" done docker run --rm --network none --entrypoint /usr/bin/ffmpeg \ @@ -36,8 +40,7 @@ cam={'id':'id_browser','name':'Browser fixture','rtsp_url':'rtsp://unused-fixtur with open(sys.argv[1],'w') as f: json.dump({'cameras':{'id_browser':cam},'camera_order':['id_browser'],'events':[]},f) PY -# Pace real FFmpeg input so cancellation can observe an active runtime render. -# Preserve all application arguments and execute the real binary, not a fake MP4. +# Pace real FFmpeg, preserving application arguments and process ownership. cat > "$bin/ffmpeg" <<'SH' #!/bin/sh exec /usr/bin/ffmpeg -re "$@" @@ -47,13 +50,17 @@ SIMPLENVR_E2E_PASSWORD=$(python3 -c 'import secrets; print(secrets.token_urlsafe export SIMPLENVR_E2E_PASSWORD printf '%s\n' "$SIMPLENVR_E2E_PASSWORD" | docker run --rm -i \ --mount "type=bind,src=$state,dst=/recordings" "$image" auth set-password ci-browser -sudo chown -R 0:0 "$state" -sudo chmod 700 "$state" +sudo chown -R 0:0 "$state" "$scratch" +sudo chmod 700 "$state" "$scratch" docker run -d --name "$name" --read-only --cap-drop ALL --cap-add NET_BIND_SERVICE \ --security-opt no-new-privileges --tmpfs /certs:rw,nosuid,nodev,noexec \ --tmpfs /tmp:rw,nosuid,nodev,noexec,size=64m \ -p 127.0.0.1::443 \ --env PATH=/fixture-bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + --env NVR_EXPORT_SCRATCH_DIR=/exports \ + --env NVR_EXPORT_MAX_BYTES=33554432 --env NVR_EXPORT_TOTAL_BYTES=67108864 \ + --env NVR_EXPORT_MAX_JOBS=8 --env NVR_EXPORT_PARALLEL=1 \ + --mount "type=bind,src=$scratch,dst=/exports" \ --mount "type=bind,src=$state,dst=/recordings" \ --mount "type=bind,src=$bin,dst=/fixture-bin,readonly" "$image" >/dev/null ready=0