Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 22 additions & 8 deletions continuous_recording_health.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,11 +47,25 @@ func (e *continuousRecordingError) Unwrap() []error {return e.causes}

var describeFailure = regexp.MustCompile(`(?m)method DESCRIBE failed: ([1-5][0-9]{2})(?:[ (]|$)`)

// continuousAttemptResult classifies using the contexts' current state. Callers
// that validate or publish after capture must use continuousAttemptOutcome with
// the capture error observed when the capture process returned.
func continuousAttemptResult(parent,segment context.Context,runErr,publicationErr error,tempPath string,stderr []byte) error {
if parent.Err()!=nil {return parent.Err()}
if runErr==nil && segment.Err()==nil && publicationErr==nil {return nil}
return continuousAttemptOutcome(parent.Err(),segment.Err(),runErr,publicationErr,tempPath,stderr)
}

// captureErr is the capture context error observed immediately after FFmpeg
// returned. A deadline that expires later, during validation or publication,
// must not turn a completed capture into a capture timeout.
func continuousAttemptOutcome(parentErr,captureErr,runErr,publicationErr error,tempPath string,stderr []byte) error {
if parentErr!=nil {return parentErr}
if runErr==nil && captureErr==nil && publicationErr==nil {return nil}
notice:=continuousRecordingNotice{Code:"recorder_failed",Message:"Recording failed. Check the camera or stream server."}
if runErr!=nil {
if errors.Is(captureErr,context.DeadlineExceeded) {
// The capture deadline killed or outlasted FFmpeg, whether or not it
// reported a process error.
notice.Code="recording_timeout";notice.Message="Recording exceeded its capture deadline."
} else if runErr!=nil {
if match:=describeFailure.FindSubmatch(stderr);len(match)>1 {
notice.Code="upstream_rtsp_unavailable"
notice.Message="Couldn't open the upstream RTSP stream. Check the camera or stream server. If this stream comes from go2rtc, check its configuration and camera source."
Expand All @@ -60,7 +74,7 @@ func continuousAttemptResult(parent,segment context.Context,runErr,publicationEr
var exit *exec.ExitError
if errors.As(runErr,&exit) && exit.ExitCode()>=0 {notice.Detail=fmt.Sprintf("FFmpeg exit %d",exit.ExitCode())}
}
} else if segment.Err()!=nil {
} else if captureErr!=nil {
notice.Code="recording_timeout";notice.Message="Recording exceeded its capture deadline."
}
if publicationErr!=nil {
Expand All @@ -72,13 +86,13 @@ func continuousAttemptResult(parent,segment context.Context,runErr,publicationEr
absent:=errors.As(publicationErr,&pe) && pe.Op=="lstat" && pe.Path==tempPath && errors.Is(pe.Err,os.ErrNotExist)
parentInfo,parentErr:=os.Stat(filepath.Dir(tempPath))
absent=absent && parentErr==nil && parentInfo.IsDir()
if absent && runErr==nil && segment.Err()==nil {
if absent && runErr==nil && captureErr==nil {
notice.Code="recording_output_missing";notice.Message="Recorder finished without creating its output file."
} else if !absent || (runErr==nil && segment.Err()==nil) {
if runErr==nil && segment.Err()==nil {notice.Code="recording_publication_failed";notice.Message="Recording could not be validated or published. Check recording storage."} else {notice.PublicationMessage="Recording storage or publication also failed; check recording storage."}
} else if !absent || (runErr==nil && captureErr==nil) {
if runErr==nil && captureErr==nil {notice.Code="recording_publication_failed";notice.Message="Recording could not be validated or published. Check recording storage."} else {notice.PublicationMessage="Recording storage or publication also failed; check recording storage."}
}
}
return &continuousRecordingError{notice:notice,causes:[]error{runErr,segment.Err(),publicationErr}}
return &continuousRecordingError{notice:notice,causes:[]error{runErr,captureErr,publicationErr}}
}

type continuousHealthWorker struct {ctx context.Context;notice continuousRecordingNotice}
Expand Down
26 changes: 26 additions & 0 deletions continuous_recording_health_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,3 +128,29 @@ func TestContinuousWorkerRecoveryCancellationAndReplacement(t *testing.T) {
err:=continuousAttemptResult(ctx,ctx,failure,os.ErrNotExist,"unused",nil)
if !errors.Is(err,context.Canceled) {t.Fatal("intentional cancellation became failure")}
}

func TestContinuousTimeoutOnlyReportsCaptureDeadline(t *testing.T) {
temp:=filepath.Join(t.TempDir(),"output.mp4.tmp")
if err:=continuousAttemptOutcome(nil,nil,nil,nil,temp,nil);err!=nil {t.Fatalf("completed capture and publication reported %v",err)}
for _,runErr:=range []error{nil,errors.New("signal: killed")} {
err:=continuousAttemptOutcome(nil,context.DeadlineExceeded,runErr,nil,temp,[]byte(upstreamFixture))
if got:=noticeFrom(t,err);got.Code!="recording_timeout" {t.Fatalf("deadline during capture misclassified: %+v",got)}
if !errors.Is(err,context.DeadlineExceeded) {t.Fatal("deadline cause lost")}
noDiagnosticSecrets(t,err.Error())
}
err:=continuousAttemptOutcome(nil,nil,nil,os.ErrPermission,temp,nil)
if got:=noticeFrom(t,err);got.Code!="recording_publication_failed" {t.Fatalf("publication failure after completed capture misclassified: %+v",got)}
err=continuousAttemptOutcome(context.Canceled,context.DeadlineExceeded,errors.New("signal: killed"),nil,temp,nil)
if !errors.Is(err,context.Canceled) {t.Fatal("intentional cancellation became failure")}
}

func TestRecordSegmentFreezesCaptureOutcomeBeforePublication(t *testing.T) {
raw,err:=os.ReadFile("recorder.go");if err!=nil {t.Fatal(err)}
source:=string(raw)
start,end:=strings.Index(source,"func recordSegment("),strings.Index(source,"func sanitizeFilename(")
if start<0 || end<=start {t.Fatal("recordSegment not found")}
body:=source[start:end]
if strings.Count(body,"segmentCtx.Err()")!=1 {t.Fatal("capture context must be observed exactly once, when FFmpeg returns")}
if strings.Index(body,"captureErr:=segmentCtx.Err()")<0 || strings.Index(body,"captureErr:=segmentCtx.Err()")>strings.Index(body,"publishRecording(") {t.Fatal("capture outcome must be fixed before publication")}
if !strings.Contains(body,"continuousAttemptOutcome(ctx.Err(),captureErr,") || strings.Contains(body,"continuousAttemptResult(") {t.Fatal("recordSegment must classify with the frozen capture outcome")}
}
7 changes: 5 additions & 2 deletions recorder.go
Original file line number Diff line number Diff line change
Expand Up @@ -272,11 +272,14 @@ func recordSegment(ctx context.Context,cam *Camera,durationSeconds int) error {
stderr:=&recordingStderrTail{}
cmd.Stderr=stderr
runErr:=cmd.Run()
if runErr!=nil || segmentCtx.Err()!=nil {filePath=partialRecordingName(filePath)}
// Freeze the capture outcome as FFmpeg returns. Validation and publication run
// under their own finalize deadline and must not rewrite a completed capture.
captureErr:=segmentCtx.Err()
if runErr!=nil || captureErr!=nil {filePath=partialRecordingName(filePath)}
finalizeCtx,finalizeCancel:=context.WithTimeout(context.WithoutCancel(ctx),recordingFinalizeGrace);defer finalizeCancel()
publicationErr:=publishRecording(finalizeCtx,tempPath,filePath)
if publicationErr!=nil {removeEmptyRecording(tempPath)}
return continuousAttemptResult(ctx,segmentCtx,runErr,publicationErr,tempPath,stderr.data)
return continuousAttemptOutcome(ctx.Err(),captureErr,runErr,publicationErr,tempPath,stderr.data)
}

func sanitizeFilename(name string) string {
Expand Down
116 changes: 70 additions & 46 deletions recording_intent_status_test.go
Original file line number Diff line number Diff line change
@@ -1,52 +1,76 @@
package main

import (
"bytes"
"html/template"
"strings"
"testing"
"bytes"
"html/template"
"strings"
"testing"
)

func TestCameraTemplateDescribesRecordingIntentNotHealth(t *testing.T) {
tmpl, err := template.ParseFiles("templates/cameras.html")
if err != nil {
t.Fatal(err)
}
for _, tc := range []struct {
name string
enabled bool
wantStatus string
wantAction string
absentAction string
}{
{"enabled", true, "Recording enabled", "Stop Recording", "Start Recording"},
{"stopped", false, "Recording stopped", "Start Recording", "Stop Recording"},
} {
t.Run(tc.name, func(t *testing.T) {
view := map[string]interface{}{
"ID": "test", "Name": "Test camera", "RTSPUrl": "rtsp://example.invalid/live",
"RecordDir": "/recordings/test", "DiskUsage": "0 B", "IsRecording": tc.enabled,
"RecordingMode": ModeEventBased, "ModeDisplay": "Event Based", "RetentionDisplay": 7,
}
var out bytes.Buffer
if err := tmpl.ExecuteTemplate(&out, "cameras.html", []map[string]interface{}{view}); err != nil {
t.Fatal(err)
}
body := out.String()
for _, want := range []string{
">" + tc.wantStatus + "</span>",
">" + tc.wantAction + "</button>",
"Recording state reflects intent, not camera connectivity or successful footage capture.",
} {
if !strings.Contains(body, want) {
t.Fatalf("rendered template missing %q", want)
}
}
for _, absent := range []string{">Recording</span>", ">Idle</span>", ">" + tc.absentAction + "</button>"} {
if strings.Contains(body, absent) {
t.Fatalf("rendered template retained unsupported status or action %q", absent)
}
}
})
}
func TestCameraTemplateRecordingStatus(t *testing.T) {
tmpl,err:=template.ParseFiles("templates/cameras.html")
if err!=nil {t.Fatal(err)}
failure:=continuousRecordingNotice{Code:"upstream_rtsp_unavailable",Message:"Could not open upstream stream",Detail:"DESCRIBE 404",PublicationMessage:"Check recording storage",Retrying:true}
notRetrying:=failure;notRetrying.Retrying=false
for _,tc:=range []struct {
name string
enabled bool
mode RecordingMode
health continuousRecordingNotice
status,class string
failed bool
}{
{"continuous enabled",true,ModeContinuous,continuousRecordingNotice{},"Recording enabled","status-recording",false},
{"continuous failed retrying",true,ModeContinuous,failure,"Recording failed — retrying","status-failed recording",true},
{"continuous failed without retry claim",true,ModeContinuous,notRetrying,"Recording failed","status-failed recording",true},
{"stopped",false,ModeContinuous,continuousRecordingNotice{},"Recording stopped","status-idle",false},
{"stopped ignores stale failure",false,ModeContinuous,failure,"Recording stopped","status-idle",false},
{"event enabled unchanged",true,ModeEventBased,continuousRecordingNotice{},"Recording enabled","status-recording",false},
{"event ignores continuous failure",true,ModeEventBased,failure,"Recording enabled","status-recording",false},
{"no recording",false,ModeNoRecording,failure,"Recording stopped","status-idle",false},
} {t.Run(tc.name,func(t *testing.T){
view:=map[string]interface{}{
"ID":"test","Name":"Test camera","RTSPUrl":"rtsp://example.invalid/live",
"RecordDir":"/recordings/test","DiskUsage":"0 B","IsRecording":tc.enabled,
"RecordingMode":tc.mode,"ModeDisplay":string(tc.mode),"RetentionDisplay":7,"RecordingHealth":tc.health,
}
var out bytes.Buffer
if err:=tmpl.ExecuteTemplate(&out,"cameras.html",[]map[string]interface{}{view});err!=nil {t.Fatal(err)}
body:=out.String()
action,absentAction:="Start Recording","Stop Recording"
if tc.enabled {action,absentAction=absentAction,action}
for _,want:=range []string{`class="`+tc.class+`">`+tc.status+`</span>`,">"+action+"</button>"} {
if !strings.Contains(body,want) {t.Fatalf("missing %q in %s",want,body)}
}
for _,absent:=range []string{"Recording state reflects intent", "camera-status-explanation",">Recording</span>",">Idle</span>",">"+absentAction+"</button>"} {
if strings.Contains(body,absent) {t.Fatalf("unexpected %q",absent)}
}
if tc.failed {
for _,want:=range []string{"Recording setting: Enabled",tc.health.Message,tc.health.Detail,tc.health.PublicationMessage} {
if !strings.Contains(body,want) {t.Fatalf("missing failure context %q",want)}
}
if strings.Contains(body,">Recording enabled</span>") {t.Fatal("failure masked by enabled primary status")}
} else {
for _,absent:=range []string{"Recording failed","camera-recording-error","Recording setting: Enabled"} {
if strings.Contains(body,absent) {t.Fatalf("unexpected failure context %q",absent)}
}
}
})}
}

func TestCameraTemplateRecordingRecoveryAndEscaping(t *testing.T) {
tmpl,err:=template.ParseFiles("templates/cameras.html");if err!=nil {t.Fatal(err)}
view:=map[string]interface{}{
"ID":"test","Name":"Test camera","RTSPUrl":"","RecordDir":"/recordings/test","DiskUsage":"0 B",
"IsRecording":true,"RecordingMode":ModeContinuous,"ModeDisplay":"Continuous","RetentionDisplay":7,
"RecordingHealth":continuousRecordingNotice{Code:"failed",Message:`<script>alert("message")</script>`,Detail:`<img src=x onerror=alert(1)>`,PublicationMessage:`<svg onload=alert(1)>`,Retrying:true},
}
render:=func()string {t.Helper();var out bytes.Buffer;if err:=tmpl.ExecuteTemplate(&out,"cameras.html",[]map[string]interface{}{view});err!=nil {t.Fatal(err)};return out.String()}
failed:=render()
for _,raw:=range []string{"<script>","<img ","<svg "} {if strings.Contains(failed,raw) {t.Fatalf("unescaped diagnostic: %s",failed)}}
if !strings.Contains(failed,"&lt;script&gt;") || !strings.Contains(failed,"Recording failed — retrying") {t.Fatal("escaped failure context missing")}
view["RecordingHealth"]=continuousRecordingNotice{}
recovered:=render()
if !strings.Contains(recovered,`class="status-recording">Recording enabled</span>`) || strings.Contains(recovered,"Recording failed") || strings.Contains(recovered,"camera-recording-error") {t.Fatal("cleared health did not restore enabled status")}
if !strings.Contains(recovered,">Stop Recording</button>") {t.Fatal("recovery changed recording control")}
}
9 changes: 5 additions & 4 deletions templates/cameras.html
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
{{ range . }}
{{ $failed := and .IsRecording (eq .RecordingMode "continuous") .RecordingHealth.Code }}
<div class="camera-box" data-camera-id="{{ .ID }}">
<div class="drag-handle" draggable="true">⋮⋮</div>
<div class="camera-details">
Expand All @@ -7,10 +8,10 @@
<span class="camera-rtsp">RTSP: {{ .RTSPUrl }}</span>
<span class="camera-folder">Folder: {{ .RecordDir }}</span>
<span class="camera-disk-usage">Disk Usage: {{ .DiskUsage }}</span>
<span class="camera-status">Status: <span class="{{ if .IsRecording }}status-recording{{ else }}status-idle{{ end }}">{{ if .IsRecording }}Recording enabled{{ else }}Recording stopped{{ end }}</span></span>
<span class="camera-status-explanation">Recording state reflects intent, not camera connectivity or successful footage capture.</span>
{{ if .RecordingHealth.Code }}
<span class="camera-recording-error" role="status" data-error-code="{{ .RecordingHealth.Code }}">{{ .RecordingHealth.Message }}{{ if .RecordingHealth.Detail }} ({{ .RecordingHealth.Detail }}){{ end }}{{ if .RecordingHealth.PublicationMessage }} {{ .RecordingHealth.PublicationMessage }}{{ end }}{{ if .RecordingHealth.Retrying }} Retrying…{{ end }}</span>
<span class="camera-status">Status: <span class="{{ if $failed }}status-failed recording{{ else if .IsRecording }}status-recording{{ else }}status-idle{{ end }}">{{ if $failed }}Recording failed{{ if .RecordingHealth.Retrying }} — retrying{{ end }}{{ else if .IsRecording }}Recording enabled{{ else }}Recording stopped{{ end }}</span></span>
{{ if $failed }}
<span class="camera-recording-setting">Recording setting: Enabled</span>
<span class="camera-recording-error" role="status" data-error-code="{{ .RecordingHealth.Code }}">{{ .RecordingHealth.Message }}{{ if .RecordingHealth.Detail }} ({{ .RecordingHealth.Detail }}){{ end }}{{ if .RecordingHealth.PublicationMessage }} {{ .RecordingHealth.PublicationMessage }}{{ end }}</span>
{{ end }}
<span class="recording-mode">Recording Mode: <span>{{ .ModeDisplay }}</span></span>
</div>
Expand Down
Loading