From 38b2030fd97a745d3f036c69e0094bb9015363d4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 12 Sep 2026 21:27:19 +0000 Subject: [PATCH 01/43] Android camera: a setup screen you can see, and hand-held robustness The first real run picked a persistent news graphic rather than the bug and there was no way to know. Camera setup is now a screen: the live picture with the found TV in green and the logo box in yellow, a magnified crop of that box, a live match score (PRESENT/absent), Watch 45 s for the automatic finder, and drag-to-draw to override it. Save & start writes the template and starts the service; AdHush pauses while the screen is open. Hand-held at eight feet: the screen is re-found every frame and each edge refined at full resolution (the coarse box was only good to 8 px, most of a small logo), capture is 1280 x 720, frames come out upright, a blur guard makes a smeared frame inert rather than "absent", and absence must last 2.5 s. LogoFinder exposes the screen, a stability map and a template for any box; LogoTemplate carries the screen's edge level for the guard. Tests cover rotation, a wandering screen, a blurred frame and a hand-drawn box. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01BcfLmuUiHYY3PbRUskf3t4 --- CHANGELOG.md | 13 ++ platforms/android/README.md | 23 +- .../android/app/src/main/AndroidManifest.xml | 1 + .../kotlin/io/adhush/android/AdHushService.kt | 5 +- .../io/adhush/android/CameraSetupActivity.kt | 199 ++++++++++++++++++ .../kotlin/io/adhush/android/CameraSource.kt | 16 +- .../kotlin/io/adhush/android/MainActivity.kt | 2 +- .../app/src/main/res/layout/activity_main.xml | 2 +- .../src/main/kotlin/io/adhush/core/Vision.kt | 143 ++++++++++--- .../test/kotlin/io/adhush/core/VisionTest.kt | 67 ++++++ 10 files changed, 422 insertions(+), 49 deletions(-) create mode 100644 platforms/android/app/src/main/kotlin/io/adhush/android/CameraSetupActivity.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c087fc..4588588 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,19 @@ All notable changes to this project are documented here. Format follows Keep a Changelog; versioning follows SemVer. +## [Unreleased] +### Changed +- Android camera: a **setup screen** replaces the blind 45-second button — + the live picture with the screen and logo boxes drawn on it, a magnified + crop of the chosen box, a live match score, and drag-to-draw to override + the automatic choice. The first real run found the finder picking a + persistent news graphic rather than the bug, unverifiable without seeing + it. +- Hand-held phones: the screen is re-found in every frame with edges refined + to the pixel (the coarse box was only good to 8 px), capture is 1280 × 720, + a blur guard makes a smeared frame inert rather than "absent", and absence + must last 2.5 s. Frames come out upright. + ## [0.10.0] - 2026-09-12 ### Added - Android **camera logo watching** (ADR 0011): the back camera feeds two diff --git a/platforms/android/README.md b/platforms/android/README.md index 78d484a..e0d6511 100644 --- a/platforms/android/README.md +++ b/platforms/android/README.md @@ -99,15 +99,20 @@ At the top of the settings, pick how *this* phone reaches *its* TV: ## Watching for the logo with the camera -Tick **Use the back camera** and, with a show on and the whole TV in the -camera's view, press **Set up camera (45 s)**. The app finds the screen, -watches which edges never move — the network bug — and saves that corner as -the thing to look for. When it is done the notification says where it found -the logo; if it says none was found, check that a show (not an ad) was on and -the screen was fully in view, then press again. From then on the bug -disappearing ducks the set within about two seconds on its own, and the bug -reappearing restores it at once. Camera set-up is per channel: press it again -after switching channels. Details: ADR 0011. +Press **Camera setup**. AdHush pauses and the screen shows what the back +camera sees, with the TV outlined in green. Hold the phone upright with the +whole TV in view (eight feet away is fine) and, with a show on, press +**Watch 45 s**. When it finishes, a yellow box marks what it thinks is the +logo and the strip underneath shows that box magnified: it should look like +the channel's bug. If it grabbed the wrong thing — on a news channel the +ticker frame or a banner is a common mistake — drag a box around the real +logo with your finger. The status line shows a live match score: high while +the logo is on screen, low during a commercial. When it reads PRESENT during +a show, press **Save & start**. Set-up is per channel. + +Hand-held is expected: the TV is re-found in every frame, a smeared frame +(the phone swung) counts as "can't see" rather than "logo gone", and the +logo has to be missing for 2.5 s before the set is ducked. ## Teaching it the commercials diff --git a/platforms/android/app/src/main/AndroidManifest.xml b/platforms/android/app/src/main/AndroidManifest.xml index fc631e8..152e44f 100644 --- a/platforms/android/app/src/main/AndroidManifest.xml +++ b/platforms/android/app/src/main/AndroidManifest.xml @@ -24,6 +24,7 @@ + lastStatus = s; main.post { update(describe(s)) } } engine = eng @@ -148,7 +149,7 @@ class AdHushService : Service(), LifecycleOwner { try { m.start() } catch (e: Exception) { update("mic failed: ${e.message}"); stopSelf(); return } mic = m if (cameraWanted) { - val cam = CameraSource(this, this) { gray -> + val cam = CameraSource(this, this) { gray -> // 2 fps, upright val ts = mic?.mediaTime ?: 0.0 eng.onFrame(gray, ts) finder?.let { f -> diff --git a/platforms/android/app/src/main/kotlin/io/adhush/android/CameraSetupActivity.kt b/platforms/android/app/src/main/kotlin/io/adhush/android/CameraSetupActivity.kt new file mode 100644 index 0000000..bb360c4 --- /dev/null +++ b/platforms/android/app/src/main/kotlin/io/adhush/android/CameraSetupActivity.kt @@ -0,0 +1,199 @@ +package io.adhush.android + +import android.Manifest +import android.content.Intent +import android.content.pm.PackageManager +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.graphics.RectF +import android.os.Bundle +import android.view.Gravity +import android.view.MotionEvent +import android.view.View +import android.widget.Button +import android.widget.ImageView +import android.widget.LinearLayout +import android.widget.TextView +import androidx.appcompat.app.AppCompatActivity +import androidx.core.app.ActivityCompat +import androidx.core.content.ContextCompat +import io.adhush.core.Box +import io.adhush.core.Gray +import io.adhush.core.HANDHELD_LOGO_CONFIG +import io.adhush.core.LogoAbsenceDetector +import io.adhush.core.LogoFinder +import io.adhush.core.LogoTemplate +import io.adhush.core.Roi +import io.adhush.core.Vision +import java.io.File +import java.util.Locale +import kotlin.math.max +import kotlin.math.min + +/** + * See what the camera sees, and choose the logo by eye. The live picture shows + * the found screen (green) and the logo box (yellow); a magnified crop of the + * box sits below it with a live match score. "Watch 45 s" runs the automatic + * finder; dragging a box on the picture overrides its choice; "Save & start" + * writes the template and starts the service. AdHush is stopped while this + * screen is open, because two things cannot hold the camera at once. + */ +class CameraSetupActivity : AppCompatActivity() { + private lateinit var picture: LogoOverlayView + private lateinit var status: TextView + private lateinit var crop: ImageView + private lateinit var saveBtn: Button + private var camera: CameraSource? = null + private var finder = LogoFinder() + @Volatile private var watching = false + private var watchStartedAt = 0L + @Volatile private var roi: Roi? = null + @Volatile private var template: LogoTemplate? = null + @Volatile private var detector: LogoAbsenceDetector? = null + private var ts = 0.0 + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + ContextCompat.startForegroundService(this, Intent(this, AdHushService::class.java).setAction(AdHushService.ACTION_STOP)) + val root = LinearLayout(this).apply { orientation = LinearLayout.VERTICAL; setPadding(24, 24, 24, 24) } + status = TextView(this).apply { textSize = 15f; text = "starting the camera …" } + picture = LogoOverlayView(this) + crop = ImageView(this).apply { scaleType = ImageView.ScaleType.FIT_CENTER; setBackgroundColor(Color.BLACK) } + val watch = Button(this).apply { text = "Watch 45 s (find the logo)" } + saveBtn = Button(this).apply { text = "Save & start"; isEnabled = false } + val cancel = Button(this).apply { text = "Cancel" } + val row = LinearLayout(this).apply { orientation = LinearLayout.HORIZONTAL } + for (b in listOf(watch, saveBtn, cancel)) row.addView(b, LinearLayout.LayoutParams(0, LinearLayout.LayoutParams.WRAP_CONTENT, 1f)) + root.addView(status) + root.addView(picture, LinearLayout.LayoutParams(LinearLayout.LayoutParams.MATCH_PARENT, 0, 1f)) + root.addView(TextView(this).apply { text = "the logo box, magnified — it should look like the channel's bug"; textSize = 12f; gravity = Gravity.CENTER }) + root.addView(crop, LinearLayout.LayoutParams(LinearLayout.LayoutParams.MATCH_PARENT, 260)) + root.addView(row) + setContentView(root) + + watch.setOnClickListener { + finder = LogoFinder(); watching = true; watchStartedAt = System.currentTimeMillis() + status.text = "watching — keep a show on, whole TV in view, hold still" + } + saveBtn.setOnClickListener { + val t = template ?: return@setOnClickListener + runCatching { t.save(File(filesDir, AdHushService.LOGO_FILE)) } + Settings(this).camera = true + ContextCompat.startForegroundService(this, Intent(this, AdHushService::class.java)) + finish() + } + cancel.setOnClickListener { finish() } + picture.onBox = { boxInFrame -> chooseBox(boxInFrame) } + + if (ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED) + ActivityCompat.requestPermissions(this, arrayOf(Manifest.permission.CAMERA), 7) + else startCamera() + } + + override fun onRequestPermissionsResult(requestCode: Int, permissions: Array, grantResults: IntArray) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults) + if (requestCode == 7 && grantResults.all { it == PackageManager.PERMISSION_GRANTED }) startCamera() else status.text = "camera permission is needed" + } + + private fun startCamera() { + val cam = CameraSource(this, this, intervalMs = 250) { gray -> onFrame(gray) } + camera = cam + cam.start { msg -> runOnUiThread { status.text = msg } } + } + + /** The user dragged a box on the picture: turn it into a normalised screen region and re-verify live. */ + private fun chooseBox(boxInFrame: Box) { + val screen = detector?.lastScreen ?: finder.lastScreen ?: return + val x = ((boxInFrame.x0 - screen.x0).toDouble() / screen.w).coerceIn(0.0, 0.99) + val y = ((boxInFrame.y0 - screen.y0).toDouble() / screen.h).coerceIn(0.0, 0.99) + val w = (boxInFrame.w.toDouble() / screen.w).coerceIn(0.02, 1.0 - x) + val h = (boxInFrame.h.toDouble() / screen.h).coerceIn(0.02, 1.0 - y) + roi = Roi(x, y, w, h) + adoptTemplate(finder.templateFor(Roi(x, y, w, h)), "your box") + } + + private fun adoptTemplate(t: LogoTemplate?, how: String) { + template = t + if (t == null) { runOnUiThread { status.text = "press Watch first, then draw or accept a box"; saveBtn.isEnabled = false }; return } + val d = LogoAbsenceDetector(HANDHELD_LOGO_CONFIG, t); d.warmup(); detector = d + runOnUiThread { status.text = "$how: ${t.roi.corner}, stability ${"%.2f".format(Locale.US, t.stability)} — watch the score, then Save"; saveBtn.isEnabled = true } + } + + private fun onFrame(gray: Gray) { + if (watching) { + finder.feed(gray) + val elapsed = (System.currentTimeMillis() - watchStartedAt) / 1000 + if (elapsed >= 45) { + watching = false + val chosen = roi + adoptTemplate(if (chosen != null) finder.templateFor(chosen) else finder.result()?.also { roi = it.roi }, if (chosen != null) "your box" else "found") + } else runOnUiThread { status.text = "watching $elapsed / 45 s · screen seen ${finder.frames} / ${finder.frames + finder.screenMisses}" } + } + val det = detector + var line = "" + if (det != null) { + det.observeFrame(gray, ts); ts += 0.25 + line = if (det.active) "logo match ${"%.2f".format(Locale.US, det.lastScore)} ${if (det.programPresent) "PRESENT" else "absent"}" else "camera: ${det.inertReason}" + } + val screen = det?.lastScreen ?: finder.lastScreen ?: Vision.findScreen(gray) + val roiBox = det?.lastRoiInFrame ?: roi?.let { r -> screen?.let { b -> Box(b.x0 + (r.x * b.w).toInt(), b.y0 + (r.y * b.h).toInt(), b.x0 + ((r.x + r.w) * b.w).toInt(), b.y0 + ((r.y + r.h) * b.h).toInt()) } } + val cropBmp = roiBox?.let { toBitmap(gray.crop(Box(max(0, it.x0), max(0, it.y0), min(gray.w, it.x1), min(gray.h, it.y1)))) } + runOnUiThread { + picture.update(toBitmap(gray), screen, roiBox) + if (line.isNotEmpty() && !watching) status.text = line + (template?.let { " · box ${it.roi.corner}" } ?: "") + if (cropBmp != null) crop.setImageBitmap(cropBmp) + } + } + + override fun onDestroy() { camera?.stop(); camera = null; super.onDestroy() } + + private fun toBitmap(g: Gray): Bitmap { + val px = IntArray(g.w * g.h) { val v = g.px[it].toInt().coerceIn(0, 255); (0xFF shl 24) or (v shl 16) or (v shl 8) or v } + return Bitmap.createBitmap(px, g.w, g.h, Bitmap.Config.ARGB_8888) + } +} + +/** The camera picture with the screen (green) and logo (yellow) boxes; drag to draw your own box. */ +class LogoOverlayView(context: android.content.Context) : View(context) { + private var bitmap: Bitmap? = null + private var screen: Box? = null + private var roi: Box? = null + private var drag: RectF? = null + private var scale = 1f; private var offX = 0f; private var offY = 0f + var onBox: ((Box) -> Unit)? = null + private val green = Paint().apply { color = Color.GREEN; style = Paint.Style.STROKE; strokeWidth = 4f } + private val yellow = Paint().apply { color = Color.YELLOW; style = Paint.Style.STROKE; strokeWidth = 4f } + private val white = Paint().apply { color = Color.WHITE; style = Paint.Style.STROKE; strokeWidth = 3f } + + fun update(b: Bitmap, s: Box?, r: Box?) { bitmap = b; screen = s; roi = r; invalidate() } + + override fun onDraw(canvas: Canvas) { + val b = bitmap ?: return + scale = min(width.toFloat() / b.width, height.toFloat() / b.height) + offX = (width - b.width * scale) / 2; offY = (height - b.height * scale) / 2 + canvas.drawBitmap(b, null, RectF(offX, offY, offX + b.width * scale, offY + b.height * scale), null) + screen?.let { canvas.drawRect(rect(it), green) } + roi?.let { canvas.drawRect(rect(it), yellow) } + drag?.let { canvas.drawRect(it, white) } + } + + private fun rect(bx: Box) = RectF(offX + bx.x0 * scale, offY + bx.y0 * scale, offX + bx.x1 * scale, offY + bx.y1 * scale) + + override fun onTouchEvent(e: MotionEvent): Boolean { + when (e.action) { + MotionEvent.ACTION_DOWN -> { drag = RectF(e.x, e.y, e.x, e.y); return true } + MotionEvent.ACTION_MOVE -> { drag?.let { it.right = e.x; it.bottom = e.y; invalidate() }; return true } + MotionEvent.ACTION_UP -> { + val d = drag ?: return true + drag = null + val x0 = ((min(d.left, d.right) - offX) / scale).toInt(); val x1 = ((max(d.left, d.right) - offX) / scale).toInt() + val y0 = ((min(d.top, d.bottom) - offY) / scale).toInt(); val y1 = ((max(d.top, d.bottom) - offY) / scale).toInt() + if (x1 - x0 >= 6 && y1 - y0 >= 6) onBox?.invoke(Box(x0, y0, x1, y1)) + invalidate(); return true + } + } + return super.onTouchEvent(e) + } +} diff --git a/platforms/android/app/src/main/kotlin/io/adhush/android/CameraSource.kt b/platforms/android/app/src/main/kotlin/io/adhush/android/CameraSource.kt index f456ed1..efb0323 100644 --- a/platforms/android/app/src/main/kotlin/io/adhush/android/CameraSource.kt +++ b/platforms/android/app/src/main/kotlin/io/adhush/android/CameraSource.kt @@ -12,12 +12,14 @@ import io.adhush.core.Gray import java.util.concurrent.Executors /** - * The back camera as a trickle of luma frames — two a second at 640 × 480 is - * plenty to see whether a bug is on the screen, and a small fraction of what - * recording video costs. Frames are stamped by the caller with media time so - * the logo detector and the audio detectors share one clock. + * The back camera as a trickle of luma frames — two a second at 1280 × 720 is + * plenty to see whether a bug is on a screen eight feet away, and a small + * fraction of what recording video costs. Frames come out upright (rotated + * by the sensor's rotationDegrees) so the setup screen and the service see + * the same picture. Frames are stamped by the caller with media time so the + * logo detector and the audio detectors share one clock. */ -class CameraSource(private val context: Context, private val owner: LifecycleOwner, private val onFrame: (Gray) -> Unit) { +class CameraSource(private val context: Context, private val owner: LifecycleOwner, private val intervalMs: Long = INTERVAL_MS, private val onFrame: (Gray) -> Unit) { private val executor = Executors.newSingleThreadExecutor() private var provider: ProcessCameraProvider? = null @Volatile private var lastAt = 0L @@ -31,13 +33,13 @@ class CameraSource(private val context: Context, private val owner: LifecycleOwn val p = future.get(); provider = p @Suppress("DEPRECATION") val analysis = ImageAnalysis.Builder() - .setTargetResolution(Size(640, 480)) + .setTargetResolution(Size(1280, 720)) // an 8 ft living room needs the pixels .setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST) .build() analysis.setAnalyzer(executor) { image -> try { val now = System.currentTimeMillis() - if (now - lastAt >= INTERVAL_MS) { lastAt = now; frames++; onFrame(toGray(image)) } + if (now - lastAt >= intervalMs) { lastAt = now; frames++; onFrame(toGray(image).rotate(image.imageInfo.rotationDegrees)) } } finally { image.close() } } p.unbindAll() diff --git a/platforms/android/app/src/main/kotlin/io/adhush/android/MainActivity.kt b/platforms/android/app/src/main/kotlin/io/adhush/android/MainActivity.kt index 9872f6d..44cf7bf 100644 --- a/platforms/android/app/src/main/kotlin/io/adhush/android/MainActivity.kt +++ b/platforms/android/app/src/main/kotlin/io/adhush/android/MainActivity.kt @@ -50,7 +50,7 @@ class MainActivity : AppCompatActivity() { findViewById