diff --git a/continuous_recording_health.go b/continuous_recording_health.go new file mode 100644 index 0000000..27ff0ad --- /dev/null +++ b/continuous_recording_health.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "sync" +) + +const recordingStderrLimit = 8192 + +// exec.Cmd joins its stderr-copy goroutine before Run returns. Read only then. +type recordingStderrTail struct { data []byte } +func (b *recordingStderrTail) Write(p []byte) (int,error) { + n:=len(p) + if n>=recordingStderrLimit { b.data=append(b.data[:0],p[n-recordingStderrLimit:]...);return n,nil } + if extra:=len(b.data)+n-recordingStderrLimit;extra>0 { copy(b.data,b.data[extra:]);b.data=b.data[:len(b.data)-extra] } + b.data=append(b.data,p...);return n,nil +} + +type continuousRecordingNotice struct { + Code string + Message string + Detail string + PublicationMessage string + Retrying bool +} + +// Error and the public notice are built only from constants and integers. +// Underlying errors remain available to errors.Is/As, never to rendering/logs. +type continuousRecordingError struct { + notice continuousRecordingNotice + causes []error +} +func (e *continuousRecordingError) Error() string { + text:=e.notice.Code+": "+e.notice.Message + if e.notice.Detail!="" {text+=" ("+e.notice.Detail+")"} + if e.notice.PublicationMessage!="" {text+=" "+e.notice.PublicationMessage} + return text +} +func (e *continuousRecordingError) Unwrap() []error {return e.causes} + +var describeFailure = regexp.MustCompile(`(?m)method DESCRIBE failed: ([1-5][0-9]{2})(?:[ (]|$)`) + +func continuousAttemptResult(parent,segment context.Context,runErr,publicationErr error,tempPath string,stderr []byte) error { + if parent.Err()!=nil {return parent.Err()} + if runErr==nil && segment.Err()==nil && publicationErr==nil {return nil} + notice:=continuousRecordingNotice{Code:"recorder_failed",Message:"Recording failed. Check the camera or stream server."} + if runErr!=nil { + if match:=describeFailure.FindSubmatch(stderr);len(match)>1 { + notice.Code="upstream_rtsp_unavailable" + notice.Message="Couldn't open the upstream RTSP stream. Check the camera or stream server. If this stream comes from go2rtc, check its configuration and camera source." + notice.Detail="DESCRIBE "+string(match[1]) + } else { + var exit *exec.ExitError + if errors.As(runErr,&exit) && exit.ExitCode()>=0 {notice.Detail=fmt.Sprintf("FFmpeg exit %d",exit.ExitCode())} + } + } else if segment.Err()!=nil { + notice.Code="recording_timeout";notice.Message="Recording exceeded its capture deadline." + } + if publicationErr!=nil { + // Only a direct lstat of the temporary output can be the secondary + // missing-output symptom. Destination, rename and later validation errors + // are never suppressed by a broad os.IsNotExist check. Verify the parent + // still exists; a missing storage directory is a storage failure. + var pe *os.PathError + absent:=errors.As(publicationErr,&pe) && pe.Op=="lstat" && pe.Path==tempPath && errors.Is(pe.Err,os.ErrNotExist) + parentInfo,parentErr:=os.Stat(filepath.Dir(tempPath)) + absent=absent && parentErr==nil && parentInfo.IsDir() + if absent && runErr==nil && segment.Err()==nil { + notice.Code="recording_output_missing";notice.Message="Recorder finished without creating its output file." + } else if !absent || (runErr==nil && segment.Err()==nil) { + if runErr==nil && segment.Err()==nil {notice.Code="recording_publication_failed";notice.Message="Recording could not be validated or published. Check recording storage."} else {notice.PublicationMessage="Recording storage or publication also failed; check recording storage."} + } + } + return &continuousRecordingError{notice:notice,causes:[]error{runErr,segment.Err(),publicationErr}} +} + +type continuousHealthWorker struct {ctx context.Context;notice continuousRecordingNotice} +var continuousHealth = struct {sync.Mutex; workers map[*Camera]*continuousHealthWorker}{workers:make(map[*Camera]*continuousHealthWorker)} + +func beginContinuousHealth(ctx context.Context,cam *Camera) *continuousHealthWorker { + worker:=&continuousHealthWorker{ctx:ctx} + continuousHealth.Lock();continuousHealth.workers[cam]=worker;continuousHealth.Unlock() + return worker +} +func endContinuousHealth(cam *Camera,worker *continuousHealthWorker) { + continuousHealth.Lock();defer continuousHealth.Unlock() + if continuousHealth.workers[cam]==worker {delete(continuousHealth.workers,cam)} +} +func updateContinuousHealth(cam *Camera,worker *continuousHealthWorker,err error) { + continuousHealth.Lock();defer continuousHealth.Unlock() + if continuousHealth.workers[cam]!=worker || worker.ctx.Err()!=nil {return} + if err==nil {worker.notice=continuousRecordingNotice{};return} + var failure *continuousRecordingError + if errors.As(err,&failure) {worker.notice=failure.notice} else {worker.notice=continuousRecordingNotice{Code:"recorder_failed",Message:"Recording failed. Check the camera, stream server, or recording storage."}} + worker.notice.Retrying=true +} +func snapshotContinuousHealth(cam *Camera) continuousRecordingNotice { + continuousHealth.Lock();defer continuousHealth.Unlock() + worker:=continuousHealth.workers[cam] + if worker==nil || worker.ctx.Err()!=nil {return continuousRecordingNotice{}} + return worker.notice +} + +func runContinuousRecorder(ctx context.Context,cam *Camera,attempt func(context.Context,*Camera,int) error,wait func(context.Context) bool) { + worker:=beginContinuousHealth(ctx,cam);defer endContinuousHealth(cam,worker) + for ctx.Err()==nil { + err:=attempt(ctx,cam,600) + if ctx.Err()!=nil {return} + updateContinuousHealth(cam,worker,err) + if err!=nil { + // Log the safe snapshot, never an arbitrary subprocess or storage error. + notice:=snapshotContinuousHealth(cam) + if notice.Code!="" {log.Printf("Recording error for camera %s: %s",cam.ID,(&continuousRecordingError{notice:notice}).Error())} + if !wait(ctx) {return} + } + } +} diff --git a/continuous_recording_health_test.go b/continuous_recording_health_test.go new file mode 100644 index 0000000..453cbf8 --- /dev/null +++ b/continuous_recording_health_test.go @@ -0,0 +1,130 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "html/template" + "log" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" +) + +const upstreamFixture = "[rtsp @ fixture] method DESCRIBE failed: 404 (Not Found)\nrtsp://dummy:SECRET@example.invalid/private?token=HIDDEN\n" + +func fakeContinuousTools(t *testing.T,body string) { + t.Helper();dir:=t.TempDir() + for name,source:=range map[string]string{"ffmpeg":"#!/bin/sh\n"+body,"ffprobe":"#!/bin/sh\nprintf 'h264\\n1\\n'\n"} { + if err:=os.WriteFile(filepath.Join(dir,name),[]byte(source),0700);err!=nil {t.Fatal(err)} + } + t.Setenv("PATH",dir+string(os.PathListSeparator)+os.Getenv("PATH")) +} +func noticeFrom(t *testing.T,err error) continuousRecordingNotice { + t.Helper();var failure *continuousRecordingError + if !errors.As(err,&failure) {t.Fatalf("expected safe recording failure, got %v",err)} + return failure.notice +} +func noDiagnosticSecrets(t *testing.T,text string) { + t.Helper() + for _,bad:=range []string{"SECRET","HIDDEN","example.invalid","rtsp://dummy","lstat",".mp4.tmp"} {if strings.Contains(text,bad) {t.Errorf("diagnostic leaked %q: %s",bad,text)}} +} + +func TestContinuousStderrTailBounded(t *testing.T) { + var tail recordingStderrTail + tail.Write(bytes.Repeat([]byte("x"),recordingStderrLimit*3)) + tail.Write([]byte(upstreamFixture)) + if len(tail.data)!=recordingStderrLimit || !bytes.HasSuffix(tail.data,[]byte(upstreamFixture)) {t.Fatal("incorrect tail")} +} + +func TestContinuousUpstreamFailureAndCameraResponse(t *testing.T) { + fakeContinuousTools(t,"printf '%s\\n' '[rtsp @ fixture] method DESCRIBE failed: 404 (Not Found)' 'rtsp://dummy:SECRET@example.invalid/private?token=HIDDEN' >&2\nexit 8\n") + cam:=&Camera{ID:"affected",Name:"HALL2",RecordDir:t.TempDir(),RecordingMode:ModeContinuous,IsRecording:true} + ctx,cancel:=context.WithCancel(context.Background());defer cancel() + worker:=beginContinuousHealth(ctx,cam);defer endContinuousHealth(cam,worker) + err:=recordSegment(ctx,cam,1) + notice:=noticeFrom(t,err) + if notice.Code!="upstream_rtsp_unavailable" || notice.Detail!="DESCRIBE 404" || notice.PublicationMessage!="" {t.Fatalf("%+v",notice)} + noDiagnosticSecrets(t,err.Error()) + updateContinuousHealth(cam,worker,err);updateContinuousHealth(cam,worker,err) + other:=&Camera{ID:"other",Name:"other",RecordDir:t.TempDir(),RecordingMode:ModeContinuous,IsRecording:true} + if snapshotContinuousHealth(other).Code!="" {t.Fatal("unaffected camera has error")} + oldNVR,oldTemplates:=nvr,templates + defer func(){nvr=oldNVR;templates=oldTemplates}() + nvr=&NVR{cameras:map[string]*Camera{cam.ID:cam,other.ID:other},cameraOrder:[]string{cam.ID,other.ID}} + var parseErr error + templates,parseErr=template.ParseFiles("templates/cameras.html") + if parseErr!=nil {t.Fatal(parseErr)} + w:=httptest.NewRecorder();camerasHandler(w,httptest.NewRequest("GET","https://nvr.test/api/cameras",nil)) + if w.Code!=200 || strings.Count(w.Body.String(),`class="camera-recording-error"`)!=1 || !strings.Contains(w.Body.String(),"DESCRIBE 404") {t.Fatalf("%d %s",w.Code,w.Body.String())} + noDiagnosticSecrets(t,w.Body.String()) + raw,_:=json.Marshal(cam) + if strings.Contains(string(raw),"upstream_rtsp") {t.Fatal("transient health persisted")} + var logs bytes.Buffer;oldWriter:=log.Writer();log.SetOutput(&logs);defer log.SetOutput(oldWriter) + runContinuousRecorder(ctx,cam,func(context.Context,*Camera,int) error{return err},func(context.Context) bool{return false}) + if !strings.Contains(logs.String(),"DESCRIBE 404") {t.Fatal("safe diagnosis not logged")} + noDiagnosticSecrets(t,logs.String()) +} + +func TestContinuousPublicationOutcomes(t *testing.T) { + for _,tc:=range []struct{name,body,code string;partial bool}{ + {"success missing output","exit 0\n","recording_output_missing",false}, + {"generic failure","echo 'rtsp://dummy:SECRET@example.invalid/private' >&2\nexit 8\n","recorder_failed",false}, + {"valid partial salvage","for output do :; done\nprintf 'partial bytes' > \"$output\"\nexit 8\n","recorder_failed",true}, + {"normal publication","for output do :; done\nprintf 'normal bytes' > \"$output\"\nexit 0\n","",false}, + } {t.Run(tc.name,func(t *testing.T){ + fakeContinuousTools(t,tc.body) + cam:=&Camera{ID:"test",Name:"test",RecordDir:t.TempDir()} + err:=recordSegment(context.Background(),cam,1) + if tc.code=="" {if err!=nil {t.Fatal(err)}} else if got:=noticeFrom(t,err);got.Code!=tc.code {t.Fatalf("%+v",got)} + if err!=nil {noDiagnosticSecrets(t,err.Error())} + files,_:=filepath.Glob(filepath.Join(cam.RecordDir,"*_partial.mp4")) + if tc.partial && len(files)!=1 {t.Fatalf("partial not preserved: %v",files)} + if tc.code=="" {files,_=filepath.Glob(filepath.Join(cam.RecordDir,"*.mp4"));if len(files)!=1 {t.Fatal("normal output not published")}} + })} +} + +func TestContinuousStorageFailuresRemainVisible(t *testing.T) { + parent:=context.Background();dir:=t.TempDir();temp:=filepath.Join(dir,"output.mp4.tmp") + for _,pubErr:=range []error{ + &os.PathError{Op:"lstat",Path:filepath.Join(dir,"destination.mp4"),Err:os.ErrNotExist}, + &os.LinkError{Op:"rename",Old:temp,New:filepath.Join(dir,"destination.mp4"),Err:os.ErrNotExist}, + &os.PathError{Op:"open",Path:temp,Err:os.ErrPermission}, + } { + err:=continuousAttemptResult(parent,parent,errors.New("process failure"),pubErr,temp,[]byte(upstreamFixture)) + if noticeFrom(t,err).PublicationMessage=="" {t.Fatal("publication error hidden")} + if !errors.Is(err,pubErr) {t.Fatal("lost underlying publication error")} + noDiagnosticSecrets(t,err.Error()) + } + missing:=filepath.Join(dir,"absent-parent","output.mp4.tmp") + err:=continuousAttemptResult(parent,parent,errors.New("process failure"),&os.PathError{Op:"lstat",Path:missing,Err:os.ErrNotExist},missing,[]byte(upstreamFixture)) + if noticeFrom(t,err).PublicationMessage=="" {t.Fatal("missing storage directory hidden")} + err=continuousAttemptResult(parent,parent,nil,os.ErrPermission,temp,nil) + if noticeFrom(t,err).Code!="recording_publication_failed" {t.Fatal("publication-only failure misclassified")} +} + +func TestContinuousWorkerRecoveryCancellationAndReplacement(t *testing.T) { + cam:=&Camera{ID:"test"};ctx,cancel:=context.WithCancel(context.Background());defer cancel() + old:=beginContinuousHealth(ctx,cam);defer endContinuousHealth(cam,old) + failure:=&continuousRecordingError{notice:continuousRecordingNotice{Code:"upstream_rtsp_unavailable",Message:"safe"}} + updateContinuousHealth(cam,old,failure) + if !snapshotContinuousHealth(cam).Retrying {t.Fatal("failure not retrying")} + updateContinuousHealth(cam,old,nil) + if snapshotContinuousHealth(cam).Code!="" {t.Fatal("normal publication did not clear")} + newer:=beginContinuousHealth(ctx,cam);defer endContinuousHealth(cam,newer) + updateContinuousHealth(cam,newer,failure) + updateContinuousHealth(cam,old,nil);endContinuousHealth(cam,old) + if snapshotContinuousHealth(cam).Code=="" {t.Fatal("old worker cleared replacement")} + replacement:=&Camera{ID:cam.ID} + if snapshotContinuousHealth(replacement).Code!="" {t.Fatal("replacement inherited old status")} + var wg sync.WaitGroup + for i:=0;i<4;i++ {wg.Add(1);go func(){defer wg.Done();for j:=0;j<100;j++ {updateContinuousHealth(cam,newer,failure);snapshotContinuousHealth(cam)}}()} + wg.Wait();cancel() + if snapshotContinuousHealth(cam).Code!="" {t.Fatal("canceled worker still retrying")} + err:=continuousAttemptResult(ctx,ctx,failure,os.ErrNotExist,"unused",nil) + if !errors.Is(err,context.Canceled) {t.Fatal("intentional cancellation became failure")} +} diff --git a/handlers.go b/handlers.go index a914ff7..556200b 100644 --- a/handlers.go +++ b/handlers.go @@ -90,6 +90,7 @@ func camerasHandler(w http.ResponseWriter,r *http.Request) { RetentionDisplay int ModeDisplay string DiskUsage string + RecordingHealth continuousRecordingNotice } nvr.mu.RLock() cameras:=make([]CameraView,0,len(nvr.cameraOrder)) @@ -98,7 +99,9 @@ func camerasHandler(w http.ResponseWriter,r *http.Request) { if cam==nil {continue} days:=cam.RetentionDays;if days==0 {days=7} mode:="Continuous";if cam.RecordingMode==ModeEventBased {mode="Event Based"} else if cam.RecordingMode==ModeNoRecording {mode="No Recording"} - cameras=append(cameras,CameraView{ID:cam.ID,Name:cam.Name,RTSPUrl:cam.RTSPUrl,RecordDir:cam.RecordDir,IsRecording:cam.IsRecording,RecordingMode:cam.RecordingMode,RetentionDisplay:days,ModeDisplay:mode}) + health:=continuousRecordingNotice{} + if cam.IsRecording && cam.RecordingMode==ModeContinuous {health=snapshotContinuousHealth(cam)} + cameras=append(cameras,CameraView{ID:cam.ID,Name:cam.Name,RTSPUrl:cam.RTSPUrl,RecordDir:cam.RecordDir,IsRecording:cam.IsRecording,RecordingMode:cam.RecordingMode,RetentionDisplay:days,ModeDisplay:mode,RecordingHealth:health}) } nvr.mu.RUnlock() for i:=range cameras {cameras[i].DiskUsage=calculateDiskUsage(cameras[i].RecordDir)} diff --git a/recorder.go b/recorder.go index f8f1ba0..00015bc 100644 --- a/recorder.go +++ b/recorder.go @@ -43,9 +43,7 @@ func newBufferManager(ctx context.Context,cam *Camera) *BufferManager { func recordCamera(ctx context.Context,cam *Camera) { defer cam.recordWG.Done() - for ctx.Err()==nil { - if err:=recordSegment(ctx,cam,600);err!=nil && ctx.Err()==nil {log.Printf("Recording error for camera %s: %v",cam.ID,err);if !waitRecordingRetry(ctx) {return}} - } + runContinuousRecorder(ctx,cam,recordSegment,waitRecordingRetry) } func waitRecordingRetry(ctx context.Context) bool { @@ -234,6 +232,7 @@ func (bm *BufferManager) concatenateEventRecording(job *bufferEvent,eventFile st job.keepStaging=true if err:=job.recoveryCheckpoint("assembling","");err!=nil {return ""} execution.requireCheckpoint(job,"assembling") + job.keepStaging=true if err:=job.writeEventConcat(bm.ctx,eventFile);err!=nil {return ""} name:=fmt.Sprintf("%s_%s_%s.mp4",job.preStart.UTC().Format("20060102_150405"),sanitizeFilename(job.event.Label),sanitizeFilename(job.event.ID)) final:=filepath.Join(bm.camera.RecordDir,name);temp:=final+".tmp" @@ -270,16 +269,14 @@ func recordSegment(ctx context.Context,cam *Camera,durationSeconds int) error { filePath:=filepath.Join(cam.RecordDir,filename);tempPath:=filePath+".tmp" segmentCtx,cancel:=context.WithTimeout(ctx,time.Duration(durationSeconds)*time.Second+5*time.Second);defer cancel() cmd:=ffmpegCommand(segmentCtx,"-hide_banner","-loglevel","error","-nostats","-rtsp_transport","tcp","-fflags","+genpts+igndts+ignidx+discardcorrupt","-i",cam.RTSPUrl,"-t",strconv.Itoa(durationSeconds),"-c","copy","-movflags","faststart","-f","mp4","-y",tempPath) + stderr:=&recordingStderrTail{} + cmd.Stderr=stderr runErr:=cmd.Run() if runErr!=nil || segmentCtx.Err()!=nil {filePath=partialRecordingName(filePath)} finalizeCtx,finalizeCancel:=context.WithTimeout(context.WithoutCancel(ctx),recordingFinalizeGrace);defer finalizeCancel() - if err:=publishRecording(finalizeCtx,tempPath,filePath);err!=nil { - removeEmptyRecording(tempPath) - return errors.Join(runErr,segmentCtx.Err(),err) - } - if ctx.Err()!=nil {return ctx.Err()} - if segmentCtx.Err()!=nil {return segmentCtx.Err()} - return runErr + publicationErr:=publishRecording(finalizeCtx,tempPath,filePath) + if publicationErr!=nil {removeEmptyRecording(tempPath)} + return continuousAttemptResult(ctx,segmentCtx,runErr,publicationErr,tempPath,stderr.data) } func sanitizeFilename(name string) string { diff --git a/templates/cameras.html b/templates/cameras.html index e1242e6..c8e595e 100644 --- a/templates/cameras.html +++ b/templates/cameras.html @@ -9,6 +9,9 @@ Disk Usage: {{ .DiskUsage }} Status: {{ if .IsRecording }}Recording enabled{{ else }}Recording stopped{{ end }} Recording state reflects intent, not camera connectivity or successful footage capture. + {{ if .RecordingHealth.Code }} + {{ .RecordingHealth.Message }}{{ if .RecordingHealth.Detail }} ({{ .RecordingHealth.Detail }}){{ end }}{{ if .RecordingHealth.PublicationMessage }} {{ .RecordingHealth.PublicationMessage }}{{ end }}{{ if .RecordingHealth.Retrying }} Retrying…{{ end }} + {{ end }} Recording Mode: {{ .ModeDisplay }}