Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
124 changes: 124 additions & 0 deletions continuous_recording_health.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
package main

import (
"context"
"errors"
"fmt"
"log"
"os"
"os/exec"
"path/filepath"
"regexp"
"sync"
)

const recordingStderrLimit = 8192

// exec.Cmd joins its stderr-copy goroutine before Run returns. Read only then.
type recordingStderrTail struct { data []byte }
func (b *recordingStderrTail) Write(p []byte) (int,error) {
n:=len(p)
if n>=recordingStderrLimit { b.data=append(b.data[:0],p[n-recordingStderrLimit:]...);return n,nil }
if extra:=len(b.data)+n-recordingStderrLimit;extra>0 { copy(b.data,b.data[extra:]);b.data=b.data[:len(b.data)-extra] }
b.data=append(b.data,p...);return n,nil
}

type continuousRecordingNotice struct {
Code string
Message string
Detail string
PublicationMessage string
Retrying bool
}

// Error and the public notice are built only from constants and integers.
// Underlying errors remain available to errors.Is/As, never to rendering/logs.
type continuousRecordingError struct {
notice continuousRecordingNotice
causes []error
}
func (e *continuousRecordingError) Error() string {
text:=e.notice.Code+": "+e.notice.Message
if e.notice.Detail!="" {text+=" ("+e.notice.Detail+")"}
if e.notice.PublicationMessage!="" {text+=" "+e.notice.PublicationMessage}
return text
}
func (e *continuousRecordingError) Unwrap() []error {return e.causes}

var describeFailure = regexp.MustCompile(`(?m)method DESCRIBE failed: ([1-5][0-9]{2})(?:[ (]|$)`)

func continuousAttemptResult(parent,segment context.Context,runErr,publicationErr error,tempPath string,stderr []byte) error {
if parent.Err()!=nil {return parent.Err()}
if runErr==nil && segment.Err()==nil && publicationErr==nil {return nil}
notice:=continuousRecordingNotice{Code:"recorder_failed",Message:"Recording failed. Check the camera or stream server."}
if runErr!=nil {
if match:=describeFailure.FindSubmatch(stderr);len(match)>1 {
notice.Code="upstream_rtsp_unavailable"
notice.Message="Couldn't open the upstream RTSP stream. Check the camera or stream server. If this stream comes from go2rtc, check its configuration and camera source."
notice.Detail="DESCRIBE "+string(match[1])
} else {
var exit *exec.ExitError
if errors.As(runErr,&exit) && exit.ExitCode()>=0 {notice.Detail=fmt.Sprintf("FFmpeg exit %d",exit.ExitCode())}
}
} else if segment.Err()!=nil {
notice.Code="recording_timeout";notice.Message="Recording exceeded its capture deadline."
}
if publicationErr!=nil {
// Only a direct lstat of the temporary output can be the secondary
// missing-output symptom. Destination, rename and later validation errors
// are never suppressed by a broad os.IsNotExist check. Verify the parent
// still exists; a missing storage directory is a storage failure.
var pe *os.PathError
absent:=errors.As(publicationErr,&pe) && pe.Op=="lstat" && pe.Path==tempPath && errors.Is(pe.Err,os.ErrNotExist)
parentInfo,parentErr:=os.Stat(filepath.Dir(tempPath))
absent=absent && parentErr==nil && parentInfo.IsDir()
if absent && runErr==nil && segment.Err()==nil {
notice.Code="recording_output_missing";notice.Message="Recorder finished without creating its output file."
} else if !absent || (runErr==nil && segment.Err()==nil) {
if runErr==nil && segment.Err()==nil {notice.Code="recording_publication_failed";notice.Message="Recording could not be validated or published. Check recording storage."} else {notice.PublicationMessage="Recording storage or publication also failed; check recording storage."}
}
}
return &continuousRecordingError{notice:notice,causes:[]error{runErr,segment.Err(),publicationErr}}
}

type continuousHealthWorker struct {ctx context.Context;notice continuousRecordingNotice}
var continuousHealth = struct {sync.Mutex; workers map[*Camera]*continuousHealthWorker}{workers:make(map[*Camera]*continuousHealthWorker)}

func beginContinuousHealth(ctx context.Context,cam *Camera) *continuousHealthWorker {
worker:=&continuousHealthWorker{ctx:ctx}
continuousHealth.Lock();continuousHealth.workers[cam]=worker;continuousHealth.Unlock()
return worker
}
func endContinuousHealth(cam *Camera,worker *continuousHealthWorker) {
continuousHealth.Lock();defer continuousHealth.Unlock()
if continuousHealth.workers[cam]==worker {delete(continuousHealth.workers,cam)}
}
func updateContinuousHealth(cam *Camera,worker *continuousHealthWorker,err error) {
continuousHealth.Lock();defer continuousHealth.Unlock()
if continuousHealth.workers[cam]!=worker || worker.ctx.Err()!=nil {return}
if err==nil {worker.notice=continuousRecordingNotice{};return}
var failure *continuousRecordingError
if errors.As(err,&failure) {worker.notice=failure.notice} else {worker.notice=continuousRecordingNotice{Code:"recorder_failed",Message:"Recording failed. Check the camera, stream server, or recording storage."}}
worker.notice.Retrying=true
}
func snapshotContinuousHealth(cam *Camera) continuousRecordingNotice {
continuousHealth.Lock();defer continuousHealth.Unlock()
worker:=continuousHealth.workers[cam]
if worker==nil || worker.ctx.Err()!=nil {return continuousRecordingNotice{}}
return worker.notice
}

func runContinuousRecorder(ctx context.Context,cam *Camera,attempt func(context.Context,*Camera,int) error,wait func(context.Context) bool) {
worker:=beginContinuousHealth(ctx,cam);defer endContinuousHealth(cam,worker)
for ctx.Err()==nil {
err:=attempt(ctx,cam,600)
if ctx.Err()!=nil {return}
updateContinuousHealth(cam,worker,err)
if err!=nil {
// Log the safe snapshot, never an arbitrary subprocess or storage error.
notice:=snapshotContinuousHealth(cam)
if notice.Code!="" {log.Printf("Recording error for camera %s: %s",cam.ID,(&continuousRecordingError{notice:notice}).Error())}
if !wait(ctx) {return}
}
}
}
130 changes: 130 additions & 0 deletions continuous_recording_health_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
package main

import (
"bytes"
"context"
"encoding/json"
"errors"
"html/template"
"log"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"sync"
"testing"
)

const upstreamFixture = "[rtsp @ fixture] method DESCRIBE failed: 404 (Not Found)\nrtsp://dummy:SECRET@example.invalid/private?token=HIDDEN\n"

func fakeContinuousTools(t *testing.T,body string) {
t.Helper();dir:=t.TempDir()
for name,source:=range map[string]string{"ffmpeg":"#!/bin/sh\n"+body,"ffprobe":"#!/bin/sh\nprintf 'h264\\n1\\n'\n"} {
if err:=os.WriteFile(filepath.Join(dir,name),[]byte(source),0700);err!=nil {t.Fatal(err)}
}
t.Setenv("PATH",dir+string(os.PathListSeparator)+os.Getenv("PATH"))
}
func noticeFrom(t *testing.T,err error) continuousRecordingNotice {
t.Helper();var failure *continuousRecordingError
if !errors.As(err,&failure) {t.Fatalf("expected safe recording failure, got %v",err)}
return failure.notice
}
func noDiagnosticSecrets(t *testing.T,text string) {
t.Helper()
for _,bad:=range []string{"SECRET","HIDDEN","example.invalid","rtsp://dummy","lstat",".mp4.tmp"} {if strings.Contains(text,bad) {t.Errorf("diagnostic leaked %q: %s",bad,text)}}
}

func TestContinuousStderrTailBounded(t *testing.T) {
var tail recordingStderrTail
tail.Write(bytes.Repeat([]byte("x"),recordingStderrLimit*3))
tail.Write([]byte(upstreamFixture))
if len(tail.data)!=recordingStderrLimit || !bytes.HasSuffix(tail.data,[]byte(upstreamFixture)) {t.Fatal("incorrect tail")}
}

func TestContinuousUpstreamFailureAndCameraResponse(t *testing.T) {
fakeContinuousTools(t,"printf '%s\\n' '[rtsp @ fixture] method DESCRIBE failed: 404 (Not Found)' 'rtsp://dummy:SECRET@example.invalid/private?token=HIDDEN' >&2\nexit 8\n")
cam:=&Camera{ID:"affected",Name:"HALL2",RecordDir:t.TempDir(),RecordingMode:ModeContinuous,IsRecording:true}
ctx,cancel:=context.WithCancel(context.Background());defer cancel()
worker:=beginContinuousHealth(ctx,cam);defer endContinuousHealth(cam,worker)
err:=recordSegment(ctx,cam,1)
notice:=noticeFrom(t,err)
if notice.Code!="upstream_rtsp_unavailable" || notice.Detail!="DESCRIBE 404" || notice.PublicationMessage!="" {t.Fatalf("%+v",notice)}
noDiagnosticSecrets(t,err.Error())
updateContinuousHealth(cam,worker,err);updateContinuousHealth(cam,worker,err)
other:=&Camera{ID:"other",Name:"other",RecordDir:t.TempDir(),RecordingMode:ModeContinuous,IsRecording:true}
if snapshotContinuousHealth(other).Code!="" {t.Fatal("unaffected camera has error")}
oldNVR,oldTemplates:=nvr,templates
defer func(){nvr=oldNVR;templates=oldTemplates}()
nvr=&NVR{cameras:map[string]*Camera{cam.ID:cam,other.ID:other},cameraOrder:[]string{cam.ID,other.ID}}
var parseErr error
templates,parseErr=template.ParseFiles("templates/cameras.html")
if parseErr!=nil {t.Fatal(parseErr)}
w:=httptest.NewRecorder();camerasHandler(w,httptest.NewRequest("GET","https://nvr.test/api/cameras",nil))
if w.Code!=200 || strings.Count(w.Body.String(),`class="camera-recording-error"`)!=1 || !strings.Contains(w.Body.String(),"DESCRIBE 404") {t.Fatalf("%d %s",w.Code,w.Body.String())}
noDiagnosticSecrets(t,w.Body.String())
raw,_:=json.Marshal(cam)
if strings.Contains(string(raw),"upstream_rtsp") {t.Fatal("transient health persisted")}
var logs bytes.Buffer;oldWriter:=log.Writer();log.SetOutput(&logs);defer log.SetOutput(oldWriter)
runContinuousRecorder(ctx,cam,func(context.Context,*Camera,int) error{return err},func(context.Context) bool{return false})
if !strings.Contains(logs.String(),"DESCRIBE 404") {t.Fatal("safe diagnosis not logged")}
noDiagnosticSecrets(t,logs.String())
}

func TestContinuousPublicationOutcomes(t *testing.T) {
for _,tc:=range []struct{name,body,code string;partial bool}{
{"success missing output","exit 0\n","recording_output_missing",false},
{"generic failure","echo 'rtsp://dummy:SECRET@example.invalid/private' >&2\nexit 8\n","recorder_failed",false},
{"valid partial salvage","for output do :; done\nprintf 'partial bytes' > \"$output\"\nexit 8\n","recorder_failed",true},
{"normal publication","for output do :; done\nprintf 'normal bytes' > \"$output\"\nexit 0\n","",false},
} {t.Run(tc.name,func(t *testing.T){
fakeContinuousTools(t,tc.body)
cam:=&Camera{ID:"test",Name:"test",RecordDir:t.TempDir()}
err:=recordSegment(context.Background(),cam,1)
if tc.code=="" {if err!=nil {t.Fatal(err)}} else if got:=noticeFrom(t,err);got.Code!=tc.code {t.Fatalf("%+v",got)}
if err!=nil {noDiagnosticSecrets(t,err.Error())}
files,_:=filepath.Glob(filepath.Join(cam.RecordDir,"*_partial.mp4"))
if tc.partial && len(files)!=1 {t.Fatalf("partial not preserved: %v",files)}
if tc.code=="" {files,_=filepath.Glob(filepath.Join(cam.RecordDir,"*.mp4"));if len(files)!=1 {t.Fatal("normal output not published")}}
})}
}

func TestContinuousStorageFailuresRemainVisible(t *testing.T) {
parent:=context.Background();dir:=t.TempDir();temp:=filepath.Join(dir,"output.mp4.tmp")
for _,pubErr:=range []error{
&os.PathError{Op:"lstat",Path:filepath.Join(dir,"destination.mp4"),Err:os.ErrNotExist},
&os.LinkError{Op:"rename",Old:temp,New:filepath.Join(dir,"destination.mp4"),Err:os.ErrNotExist},
&os.PathError{Op:"open",Path:temp,Err:os.ErrPermission},
} {
err:=continuousAttemptResult(parent,parent,errors.New("process failure"),pubErr,temp,[]byte(upstreamFixture))
if noticeFrom(t,err).PublicationMessage=="" {t.Fatal("publication error hidden")}
if !errors.Is(err,pubErr) {t.Fatal("lost underlying publication error")}
noDiagnosticSecrets(t,err.Error())
}
missing:=filepath.Join(dir,"absent-parent","output.mp4.tmp")
err:=continuousAttemptResult(parent,parent,errors.New("process failure"),&os.PathError{Op:"lstat",Path:missing,Err:os.ErrNotExist},missing,[]byte(upstreamFixture))
if noticeFrom(t,err).PublicationMessage=="" {t.Fatal("missing storage directory hidden")}
err=continuousAttemptResult(parent,parent,nil,os.ErrPermission,temp,nil)
if noticeFrom(t,err).Code!="recording_publication_failed" {t.Fatal("publication-only failure misclassified")}
}

func TestContinuousWorkerRecoveryCancellationAndReplacement(t *testing.T) {
cam:=&Camera{ID:"test"};ctx,cancel:=context.WithCancel(context.Background());defer cancel()
old:=beginContinuousHealth(ctx,cam);defer endContinuousHealth(cam,old)
failure:=&continuousRecordingError{notice:continuousRecordingNotice{Code:"upstream_rtsp_unavailable",Message:"safe"}}
updateContinuousHealth(cam,old,failure)
if !snapshotContinuousHealth(cam).Retrying {t.Fatal("failure not retrying")}
updateContinuousHealth(cam,old,nil)
if snapshotContinuousHealth(cam).Code!="" {t.Fatal("normal publication did not clear")}
newer:=beginContinuousHealth(ctx,cam);defer endContinuousHealth(cam,newer)
updateContinuousHealth(cam,newer,failure)
updateContinuousHealth(cam,old,nil);endContinuousHealth(cam,old)
if snapshotContinuousHealth(cam).Code=="" {t.Fatal("old worker cleared replacement")}
replacement:=&Camera{ID:cam.ID}
if snapshotContinuousHealth(replacement).Code!="" {t.Fatal("replacement inherited old status")}
var wg sync.WaitGroup
for i:=0;i<4;i++ {wg.Add(1);go func(){defer wg.Done();for j:=0;j<100;j++ {updateContinuousHealth(cam,newer,failure);snapshotContinuousHealth(cam)}}()}
wg.Wait();cancel()
if snapshotContinuousHealth(cam).Code!="" {t.Fatal("canceled worker still retrying")}
err:=continuousAttemptResult(ctx,ctx,failure,os.ErrNotExist,"unused",nil)
if !errors.Is(err,context.Canceled) {t.Fatal("intentional cancellation became failure")}
}
5 changes: 4 additions & 1 deletion handlers.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ func camerasHandler(w http.ResponseWriter,r *http.Request) {
RetentionDisplay int
ModeDisplay string
DiskUsage string
RecordingHealth continuousRecordingNotice
}
nvr.mu.RLock()
cameras:=make([]CameraView,0,len(nvr.cameraOrder))
Expand All @@ -98,7 +99,9 @@ func camerasHandler(w http.ResponseWriter,r *http.Request) {
if cam==nil {continue}
days:=cam.RetentionDays;if days==0 {days=7}
mode:="Continuous";if cam.RecordingMode==ModeEventBased {mode="Event Based"} else if cam.RecordingMode==ModeNoRecording {mode="No Recording"}
cameras=append(cameras,CameraView{ID:cam.ID,Name:cam.Name,RTSPUrl:cam.RTSPUrl,RecordDir:cam.RecordDir,IsRecording:cam.IsRecording,RecordingMode:cam.RecordingMode,RetentionDisplay:days,ModeDisplay:mode})
health:=continuousRecordingNotice{}
if cam.IsRecording && cam.RecordingMode==ModeContinuous {health=snapshotContinuousHealth(cam)}
cameras=append(cameras,CameraView{ID:cam.ID,Name:cam.Name,RTSPUrl:cam.RTSPUrl,RecordDir:cam.RecordDir,IsRecording:cam.IsRecording,RecordingMode:cam.RecordingMode,RetentionDisplay:days,ModeDisplay:mode,RecordingHealth:health})
}
nvr.mu.RUnlock()
for i:=range cameras {cameras[i].DiskUsage=calculateDiskUsage(cameras[i].RecordDir)}
Expand Down
17 changes: 7 additions & 10 deletions recorder.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,9 +43,7 @@ func newBufferManager(ctx context.Context,cam *Camera) *BufferManager {

func recordCamera(ctx context.Context,cam *Camera) {
defer cam.recordWG.Done()
for ctx.Err()==nil {
if err:=recordSegment(ctx,cam,600);err!=nil && ctx.Err()==nil {log.Printf("Recording error for camera %s: %v",cam.ID,err);if !waitRecordingRetry(ctx) {return}}
}
runContinuousRecorder(ctx,cam,recordSegment,waitRecordingRetry)
}

func waitRecordingRetry(ctx context.Context) bool {
Expand Down Expand Up @@ -234,6 +232,7 @@ func (bm *BufferManager) concatenateEventRecording(job *bufferEvent,eventFile st
job.keepStaging=true
if err:=job.recoveryCheckpoint("assembling","");err!=nil {return ""}
execution.requireCheckpoint(job,"assembling")
job.keepStaging=true
if err:=job.writeEventConcat(bm.ctx,eventFile);err!=nil {return ""}
name:=fmt.Sprintf("%s_%s_%s.mp4",job.preStart.UTC().Format("20060102_150405"),sanitizeFilename(job.event.Label),sanitizeFilename(job.event.ID))
final:=filepath.Join(bm.camera.RecordDir,name);temp:=final+".tmp"
Expand Down Expand Up @@ -270,16 +269,14 @@ func recordSegment(ctx context.Context,cam *Camera,durationSeconds int) error {
filePath:=filepath.Join(cam.RecordDir,filename);tempPath:=filePath+".tmp"
segmentCtx,cancel:=context.WithTimeout(ctx,time.Duration(durationSeconds)*time.Second+5*time.Second);defer cancel()
cmd:=ffmpegCommand(segmentCtx,"-hide_banner","-loglevel","error","-nostats","-rtsp_transport","tcp","-fflags","+genpts+igndts+ignidx+discardcorrupt","-i",cam.RTSPUrl,"-t",strconv.Itoa(durationSeconds),"-c","copy","-movflags","faststart","-f","mp4","-y",tempPath)
stderr:=&recordingStderrTail{}
cmd.Stderr=stderr
runErr:=cmd.Run()
if runErr!=nil || segmentCtx.Err()!=nil {filePath=partialRecordingName(filePath)}
finalizeCtx,finalizeCancel:=context.WithTimeout(context.WithoutCancel(ctx),recordingFinalizeGrace);defer finalizeCancel()
if err:=publishRecording(finalizeCtx,tempPath,filePath);err!=nil {
removeEmptyRecording(tempPath)
return errors.Join(runErr,segmentCtx.Err(),err)
}
if ctx.Err()!=nil {return ctx.Err()}
if segmentCtx.Err()!=nil {return segmentCtx.Err()}
return runErr
publicationErr:=publishRecording(finalizeCtx,tempPath,filePath)
if publicationErr!=nil {removeEmptyRecording(tempPath)}
return continuousAttemptResult(ctx,segmentCtx,runErr,publicationErr,tempPath,stderr.data)
}

func sanitizeFilename(name string) string {
Expand Down
3 changes: 3 additions & 0 deletions templates/cameras.html
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
<span class="camera-disk-usage">Disk Usage: {{ .DiskUsage }}</span>
<span class="camera-status">Status: <span class="{{ if .IsRecording }}status-recording{{ else }}status-idle{{ end }}">{{ if .IsRecording }}Recording enabled{{ else }}Recording stopped{{ end }}</span></span>
<span class="camera-status-explanation">Recording state reflects intent, not camera connectivity or successful footage capture.</span>
{{ if .RecordingHealth.Code }}
<span class="camera-recording-error" role="status" data-error-code="{{ .RecordingHealth.Code }}">{{ .RecordingHealth.Message }}{{ if .RecordingHealth.Detail }} ({{ .RecordingHealth.Detail }}){{ end }}{{ if .RecordingHealth.PublicationMessage }} {{ .RecordingHealth.PublicationMessage }}{{ end }}{{ if .RecordingHealth.Retrying }} Retrying…{{ end }}</span>
{{ end }}
<span class="recording-mode">Recording Mode: <span>{{ .ModeDisplay }}</span></span>
</div>
<div class="camera-retention">
Expand Down
Loading