Media directory monitoring: notifications + adaptive reconciliation, no disable on timeout

buildPy2exe.py

@@ -750,7 +750,7 @@ def run(self):
     options={
         'py2exe': {
             'dist_dir': OUT_DIR,
-            'packages': '{}, cffi, OpenSSL, certifi'.format(QT_PACKAGES),
+            'packages': '{}, cffi, OpenSSL, certifi, watchdog'.format(QT_PACKAGES),
             'includes': 'twisted, sys, encodings, datetime, os, time, math, urllib, ast, unicodedata, _ssl, win32pipe, win32file, sqlite3',
             'excludes': 'venv, doctest, pdb, unittest, win32clipboard, win32pdh, win32security, win32trace, win32ui, winxpgui, win32process, tcl, tkinter',
             'dll_excludes': 'msvcr71.dll, MSVCP90.dll, POWRPROF.dll',

requirements.txt

@@ -1,6 +1,7 @@
 certifi>=2018.11.29
 pem>=21.2.0
 twisted[tls]>=16.4.0
+watchdog>=2.1.0,<4.0.0
 appnope>=0.1.0; sys_platform == 'darwin'
 pypiwin32>=223; sys_platform == 'win32'
 zope.interface>=4.4.0; sys_platform == 'win32'

syncplay/constants.py

@@ -116,8 +116,13 @@ def getValueForOS(constantDict):
 # Options for the File Switch feature:
 FOLDER_SEARCH_FIRST_FILE_TIMEOUT = 25.0  # Secs - How long to wait to find the first file in folder search (to take account of HDD spin up)
 FOLDER_SEARCH_TIMEOUT = 20.0  # Secs - How long to wait until searches in folder to update cache are aborted (after first file is found)
-FOLDER_SEARCH_WARNING_THRESHOLD = 2.0 # Secs - how long until a warning saying how many files have been scanned
-FOLDER_SEARCH_DOUBLE_CHECK_INTERVAL = 30.0  # Secs - Frequency of updating cache
+FOLDER_SEARCH_WARNING_THRESHOLD = 2.0 # Secs - additional grace (on top of FOLDER_SEARCH_WARNING_BASE_DELAY) before the slow-scan warning may appear
+FOLDER_SEARCH_WARNING_BASE_DELAY = 3.0 # Secs - base grace before a slow scan may warn; effective point is this + folderSearchWarningThreshold
+FOLDER_SEARCH_DOUBLE_CHECK_INTERVAL = 30.0  # Secs - Urgent polling cadence used while a needed current/next local file is unresolved
+FOLDER_SEARCH_RECONCILIATION_INTERVAL = 300.0  # Secs - Slow safety reconciliation cadence while needed files are resolvable (internal, not user-configurable)
+FOLDER_SEARCH_EVENT_COALESCE_INTERVAL = 1.0  # Secs - Coalescing window for the downstream fileSwitchFoundFiles() notification after direct cache events
+FOLDER_SEARCH_DEGRADED_RECOVERY_SCANS = 3  # Consecutive successful full reconciliations required to clear the degraded operational state
+FOLDER_SEARCH_NETWORK_WATCH_LIMIT = 32  # Process-wide budget for Windows network-directory watches (recursive roots + supplemental direct watches)
 
 # Changable values for watched features (you usually don't need to change these)
 WATCHED_CHECKQUEUE_INTERVAL = 1.0 # Secs

syncplay/filemonitor.py

@@ -0,0 +1,264 @@
+# coding:utf8
+
+"""
+Client-side filesystem-watch helper for media directory monitoring.
+
+FileMonitor is deliberately playlist-blind: it knows about paths and native
+filesystem watch characteristics only. It maintains the most useful native
+watchdog watches within a budget and reports structured filesystem events back
+to its owner (FileSwitchManager) on the Twisted reactor thread. It never touches
+mediaFilesCache, playlist state, episode parsing or file-switching decisions.
+
+watchdog is an optional dependency: if it is unavailable, or a watch cannot be
+established, FileMonitor simply provides no events and Syncplay falls back to its
+own reconciliation scanning.
+"""
+
+import os
+import sys
+from collections import namedtuple
+
+from twisted.internet import reactor
+
+from syncplay import constants
+
+try:
+    from watchdog.observers import Observer as _Observer
+    from watchdog.events import FileSystemEventHandler as _FileSystemEventHandler
+    watchdogAvailable = True
+except ImportError:
+    watchdogAvailable = False
+    _FileSystemEventHandler = object
+
+# Simple immutable value handed to the owner's callback. It carries filesystem
+# facts only - never playlist or file-matching information.
+FileMonitorEvent = namedtuple("FileMonitorEvent", ["root", "eventType", "sourcePath", "destinationPath", "isDirectory"])
+
+
+def isWindowsNetworkPath(path):
+    # A Windows UNC path, or a mapped drive the OS reports as remote. Everywhere
+    # else this is False and a single recursive native watch is trusted at depth.
+    if not path or sys.platform != "win32":
+        return False
+    if path.startswith("\\\\") or path.startswith("//"):
+        return True
+    try:
+        import ctypes
+        drive = os.path.splitdrive(os.path.abspath(path))[0]
+        if not drive:
+            return False
+        DRIVE_REMOTE = 4
+        return ctypes.windll.kernel32.GetDriveTypeW(drive + "\\") == DRIVE_REMOTE
+    except Exception:
+        return False
+
+
+def _normPath(path):
+    return os.path.normcase(os.path.normpath(path))
+
+
+def _pathIsWithin(path, root):
+    # Component-aware containment so that, for example, "P:\\TV2" is not treated
+    # as being inside "P:\\TV".
+    normedPath = _normPath(path)
+    normedRoot = _normPath(root)
+    if normedPath == normedRoot:
+        return True
+    return normedPath.startswith(normedRoot + os.sep)
+
+
+if watchdogAvailable:
+    class _WatchdogAdapter(_FileSystemEventHandler):
+        def __init__(self, monitor):
+            _FileSystemEventHandler.__init__(self)
+            self._monitor = monitor
+
+        def on_any_event(self, event):
+            self._monitor._onWatchdogEvent(event)
+
+
+class FileMonitor(object):
+    def __init__(self, eventCallback, debugCallback=None):
+        self._eventCallback = eventCallback
+        self._debugCallback = debugCallback
+        self._observer = None
+        self._adapter = None
+        self._mediaDirectories = []
+        self._rootWatches = {}      # normed root -> (watch, path, isNetwork)
+        self._priorityWatches = {}  # normed dir  -> (watch, path)
+
+    def isAvailable(self):
+        return watchdogAvailable
+
+    def _debug(self, message):
+        if self._debugCallback:
+            try:
+                self._debugCallback(message)
+            except Exception:
+                pass
+
+    def _ensureObserver(self):
+        if self._observer is None:
+            observer = _Observer()
+            self._adapter = _WatchdogAdapter(self)
+            observer.start()
+            self._observer = observer
+        return self._observer
+
+    # -- public API -------------------------------------------------------
+
+    def setMediaDirectories(self, mediaDirectories):
+        if not watchdogAvailable:
+            return
+        newRoots = [directory for directory in (mediaDirectories or []) if directory]
+        self._mediaDirectories = list(newRoots)
+        newNormedRoots = set(_normPath(root) for root in newRoots)
+
+        for normedRoot in list(self._rootWatches.keys()):
+            if normedRoot not in newNormedRoots:
+                self._removeRootWatch(normedRoot)
+
+        for normedDir in list(self._priorityWatches.keys()):
+            watchedPath = self._priorityWatches[normedDir][1]
+            if not any(_pathIsWithin(watchedPath, root) for root in newRoots):
+                self._removePriorityWatch(normedDir)
+
+        for root in newRoots:
+            normedRoot = _normPath(root)
+            if normedRoot in self._rootWatches:
+                continue
+            isNetwork = isWindowsNetworkPath(root)
+            if isNetwork and self._networkWatchCount() >= constants.FOLDER_SEARCH_NETWORK_WATCH_LIMIT:
+                self._debug("Network watch budget reached; media root not natively watched: {}".format(root))
+                continue
+            self._scheduleRoot(root, isNetwork)
+
+    def setPriorityDirectories(self, directories):
+        # Supplemental non-recursive watches are only useful for Windows network
+        # roots; a local recursive root watch already reports changes at depth.
+        if not watchdogAvailable:
+            return
+        wanted = []
+        seen = set()
+        for directory in (directories or []):
+            if not directory:
+                continue
+            normedDir = _normPath(directory)
+            if normedDir in seen or normedDir in self._rootWatches:
+                continue
+            if self._findNetworkRoot(directory) is None:
+                continue
+            seen.add(normedDir)
+            wanted.append((normedDir, directory))
+
+        wantedNormed = set(normedDir for normedDir, _ in wanted)
+        for normedDir in list(self._priorityWatches.keys()):
+            if normedDir not in wantedNormed:
+                self._removePriorityWatch(normedDir)
+
+        for normedDir, directory in wanted:
+            if normedDir in self._priorityWatches:
+                continue
+            if self._networkWatchCount() >= constants.FOLDER_SEARCH_NETWORK_WATCH_LIMIT:
+                self._debug("Network watch budget reached; priority directory not watched: {}".format(directory))
+                break
+            self._schedulePriority(directory)
+
+    def stop(self):
+        observer = self._observer
+        self._observer = None
+        self._adapter = None
+        self._rootWatches = {}
+        self._priorityWatches = {}
+        if observer is not None:
+            try:
+                observer.unschedule_all()
+                observer.stop()
+                observer.join(5.0)
+            except Exception:
+                pass
+
+    # -- watch bookkeeping ------------------------------------------------
+
+    def _networkWatchCount(self):
+        count = sum(1 for entry in self._rootWatches.values() if entry[2])
+        count += len(self._priorityWatches)
+        return count
+
+    def _scheduleRoot(self, root, isNetwork):
+        try:
+            observer = self._ensureObserver()
+            watch = observer.schedule(self._adapter, root, recursive=True)
+            self._rootWatches[_normPath(root)] = (watch, root, isNetwork)
+        except Exception as e:
+            self._debug("Could not watch media root {}: {}: {}".format(root, type(e).__name__, e))
+
+    def _schedulePriority(self, directory):
+        try:
+            observer = self._ensureObserver()
+            watch = observer.schedule(self._adapter, directory, recursive=False)
+            self._priorityWatches[_normPath(directory)] = (watch, directory)
+        except Exception as e:
+            self._debug("Could not watch priority directory {}: {}: {}".format(directory, type(e).__name__, e))
+
+    def _removeRootWatch(self, normedRoot):
+        entry = self._rootWatches.pop(normedRoot, None)
+        if entry is not None and self._observer is not None:
+            try:
+                self._observer.unschedule(entry[0])
+            except Exception:
+                pass
+
+    def _removePriorityWatch(self, normedDir):
+        entry = self._priorityWatches.pop(normedDir, None)
+        if entry is not None and self._observer is not None:
+            try:
+                self._observer.unschedule(entry[0])
+            except Exception:
+                pass
+
+    # -- event handling ---------------------------------------------------
+
+    def _findRoot(self, path):
+        if not path:
+            return None
+        best = None
+        bestLength = -1
+        for root in self._mediaDirectories:
+            if _pathIsWithin(path, root):
+                length = len(_normPath(root))
+                if length > bestLength:
+                    best = root
+                    bestLength = length
+        return best
+
+    def _findNetworkRoot(self, path):
+        root = self._findRoot(path)
+        if root is not None and isWindowsNetworkPath(root):
+            return root
+        return None
+
+    def _onWatchdogEvent(self, event):
+        # Runs on the watchdog worker thread. Copy the useful fields and hand off
+        # to the reactor thread; do not touch shared Syncplay state here.
+        try:
+            sourcePath = getattr(event, "src_path", None)
+            destinationPath = getattr(event, "dest_path", None) or None
+            root = self._findRoot(sourcePath)
+            if root is None and destinationPath:
+                root = self._findRoot(destinationPath)
+            if root is None:
+                return
+            payload = FileMonitorEvent(
+                root,
+                getattr(event, "event_type", None),
+                sourcePath,
+                destinationPath,
+                bool(getattr(event, "is_directory", False)),
+            )
+        except Exception:
+            return
+        try:
+            reactor.callFromThread(self._eventCallback, payload)
+        except Exception:
+            pass

syncplay/messages_en.py

@@ -179,7 +179,7 @@
 
     "switch-file-not-found-error": "Could not switch to file '{0}'. Syncplay looks in specified media directories.",  # File not found
     "folder-search-timeout-error": "The search for media in media directories was aborted as it took too long to search through '{}' after having processed the first {:,} files. This will occur if you select a folder with too many subfolders in your list of media folders to search through or if there are too many files to process. For automatic file switching to work again please select File->Set Media Directories in the menu bar and remove this directory or replace it with an appropriate subfolder. If the folder is actually fine then you can re-enable it by selecting File->Set Media Directories and pressing 'OK'.",  # Folder, Files processed. Note: {:,} is {} but with added commas seprators.
-    "folder-search-timeout-warning": "Warning: It has taken {} seconds to scan {:,} files in the folder '{}'. This will occur if you select a folder with too many subfolders in your list of media folders to search through or if there are too many files to process.",  # Folder, Files processed. Note: {:,} is {} but with added commas seprators.
+    "folder-search-timeout-warning": "Note: Syncplay is taking a while to scan your media directories (about {} seconds and {:,} files so far, currently in '{}'). This can happen with very large folders or slow/network drives. Syncplay will keep trying automatically in the background, so no action is needed.",  # Seconds, Files processed, Folder. Note: {:,} is {} but with added commas separators.
     "folder-search-first-file-timeout-error": "The search for media in '{}' was aborted as it took too long to access the directory. This could happen if it is a network drive or if you configure your drive to spin down after a period of inactivity. For automatic file switching to work again please go to File->Set Media Directories and either remove the directory or resolve the issue (e.g. by changing power saving settings).",  # Folder
     "added-file-not-in-media-directory-error": "You loaded a file in '{}' which is not a known media directory. You can add this as a media directory by selecting File->Set Media Directories in the menu bar.",  # Folder
     "no-media-directories-error": "No media directories have been set. For shared playlist and file switching features to work properly please select File->Set Media Directories and specify where Syncplay should look to find media files.",

syncplay/watched.py

@@ -716,6 +716,49 @@ def _normaliseEpisodeNumber(self, episodeNumber):
         return episodeNumber
 
 
+def getRelatedEpisodeDirectories(filenames, mediaFilesCache):
+    """Directories in mediaFilesCache holding the same series/season as any of the
+    target filenames, using the existing episode parser. Ranking hint only: it does
+    no filesystem access, must not affect playlist behaviour, and a false positive
+    is harmless. When a match lives in a Watched subfolder, the parent (the likely
+    location for a newly-arriving unwatched episode) is returned instead.
+    """
+    if not filenames or not mediaFilesCache:
+        return []
+    parser = EpisodeFilenameParser()
+    targetKeys = set()
+    targetContext = parser.getContext(filenames)
+    for filename in filenames:
+        info = parser.parse(filename, context=targetContext)
+        if info and info.get("seriesKey") is not None:
+            targetKeys.add((info.get("seriesKey"), info.get("season")))
+    if not targetKeys:
+        return []
+
+    related = []
+    seen = set()
+    watchedName = constants.WATCHED_SUBFOLDER.lower() if constants.WATCHED_SUBFOLDER else None
+    for directory, files in mediaFilesCache.items():
+        if not files:
+            continue
+        directoryContext = parser.getContext(files)
+        matched = False
+        for filename in files:
+            info = parser.parse(filename, context=directoryContext)
+            if info and (info.get("seriesKey"), info.get("season")) in targetKeys:
+                matched = True
+                break
+        if not matched:
+            continue
+        resultDirectory = directory
+        if watchedName and os.path.basename(os.path.normpath(directory)).lower() == watchedName:
+            resultDirectory = os.path.dirname(os.path.normpath(directory))
+        key = os.path.normcase(os.path.normpath(resultDirectory))
+        if key not in seen:
+            seen.add(key)
+            related.append(resultDirectory)
+    return related
+
 
 class WatchedManager(object):
     """