From 6eb2690191d44829ca69b4cc78fee4f25fa1654c Mon Sep 17 00:00:00 2001 From: Peter-J-Freeman Date: Fri, 31 Jul 2026 10:25:20 +0100 Subject: [PATCH 1/2] Add LRU caching concept for SeqFetcher and HDP Implement in-process LRU cache decorators for the SeqFetcher and HGVS data provider (HDP) to reduce repeated lookups during validation. The SeqFetcher cache stores repeated fetch_seq() requests. The HDP cache stores results from get_tx_identity_info(), get_tx_for_gene(), get_pro_ac_for_tx_ac(), get_tx_exons(), get_gene_info() and get_tx_mapping_options(). get_tx_for_region() was intentionally excluded because these genomic coordinate queries are unlikely to repeat during validation. Both wrappers use the decorator pattern and transparently delegate all uncached methods to the underlying implementations. Benchmarking shows the SeqFetcher cache primarily improves runtime stability, while the addition of the HDP cache provides a substantial reduction in overall validation time. Refs #876 --- VariantValidator/modules/vvMixinInit.py | 234 +++++++++++++++++++++++- 1 file changed, 231 insertions(+), 3 deletions(-) diff --git a/VariantValidator/modules/vvMixinInit.py b/VariantValidator/modules/vvMixinInit.py index 36802662..dce61181 100644 --- a/VariantValidator/modules/vvMixinInit.py +++ b/VariantValidator/modules/vvMixinInit.py @@ -2,6 +2,7 @@ import logging import os +from functools import lru_cache from configparser import ConfigParser import vvhgvs @@ -41,6 +42,216 @@ class InitialisationError(Exception): pass +class CachedSeqFetcher: + """ + LRU cache wrapper for SeqFetcher. + + This class implements the Decorator pattern around SeqFetcher. + Only the most frequently repeated sequence lookup is cached using + functools.lru_cache(); all other methods and attributes are + transparently delegated to the wrapped SeqFetcher. + + The cache exists solely within the current Python process and is + discarded when the process exits. + """ + + def __init__(self, sf): + """ + Wrap an existing SeqFetcher. + + Parameters + ---------- + sf + An instantiated SeqFetcher object. + """ + self._sf = sf + + def __getattr__(self, name): + """ + Delegate all uncached methods and attributes to the wrapped + SeqFetcher. + + Python only calls __getattr__ when an attribute is not found on + CachedSeqFetcher itself. Consequently, only the methods + explicitly implemented below are intercepted and cached; + everything else behaves exactly as if the original SeqFetcher + were being used directly. + """ + return getattr(self._sf, name) + + @lru_cache(maxsize=32768) + def fetch_seq(self, ac, start_i=None, end_i=None): + """ + Retrieve a sequence or sequence slice. + + The cache key is formed from the accession, start coordinate + and end coordinate. Repeated requests for the same sequence + slice are therefore served directly from memory rather than + performing another SeqRepo lookup. + """ + return self._sf.fetch_seq(ac, start_i, end_i) + + def clear_caches(self): + """ + Clear every LRU cache maintained by this wrapper. + + This is primarily intended for: + * benchmarking cold versus warm cache performance; + * releasing cached sequence slices; + * forcing fresh sequence lookups. + """ + self.fetch_seq.cache_clear() + + def cache_info(self): + """ + Return cache statistics for every cached method. + + Each entry contains the standard functools CacheInfo tuple: + hits + misses + maxsize + currsize + + This method is intended for benchmarking and determining + whether the cache provides sufficient benefit to justify its + memory usage. + """ + return { + "fetch_seq": self.fetch_seq.cache_info(), + } + + +class CachedHDP: + """ + LRU cache wrapper for the HGVS data provider. + + This class implements the Decorator pattern around the HGVS UTA data + provider. Only the most frequently repeated database lookups are + cached using functools.lru_cache(); all other methods and attributes + are transparently delegated to the wrapped provider. + + The cache exists solely within the current Python process and is + discarded when the process exits. + """ + + def __init__(self, hdp): + """ + Wrap an existing HGVS data provider. + + Parameters + ---------- + hdp + An instantiated HGVS data provider returned by + vvhgvs.dataproviders.uta.connect(). + """ + self._hdp = hdp + + def __getattr__(self, name): + """ + Delegate all uncached methods and attributes to the wrapped + HGVS data provider. + + Python only calls __getattr__ when an attribute is not found on + CachedHDP itself. Consequently, only the methods explicitly + implemented below are intercepted and cached; everything else + behaves exactly as if the original HGVS data provider were being + used directly. + """ + return getattr(self._hdp, name) + + @lru_cache(maxsize=8192) + def get_tx_identity_info(self, tx_ac): + """ + Retrieve transcript identity information for a transcript + accession. + + This is the most frequently repeated HDP lookup performed by + VariantValidator and therefore has the largest cache. + """ + return self._hdp.get_tx_identity_info(tx_ac) + + @lru_cache(maxsize=4096) + def get_tx_for_gene(self, gene): + """ + Retrieve all mapped transcripts associated with an HGNC gene + symbol. + """ + return self._hdp.get_tx_for_gene(gene) + + @lru_cache(maxsize=4096) + def get_pro_ac_for_tx_ac(self, tx_ac): + """ + Retrieve the associated protein accession for a transcript + accession. + """ + return self._hdp.get_pro_ac_for_tx_ac(tx_ac) + + @lru_cache(maxsize=4096) + def get_tx_exons(self, tx_ac, alt_ac, alt_aln_method): + """ + Retrieve transcript exon structures for a transcript/genome + alignment. + """ + return self._hdp.get_tx_exons(tx_ac, alt_ac, alt_aln_method) + + @lru_cache(maxsize=2048) + def get_gene_info(self, gene): + """ + Retrieve HGNC gene information for a gene symbol. + """ + return self._hdp.get_gene_info(gene) + + @lru_cache(maxsize=4096) + def get_tx_mapping_options(self, tx_ac, gap_warn=False): + """ + Retrieve all genomic mapping options for a transcript. + + The gap_warn argument forms part of the cache key, meaning + cached results for gap_warn=True and gap_warn=False remain + independent. + """ + return self._hdp.get_tx_mapping_options(tx_ac, gap_warn) + + def clear_caches(self): + """ + Clear every LRU cache maintained by this wrapper. + + This is primarily intended for: + * benchmarking cold versus warm cache performance; + * releasing cached objects; + * forcing fresh database lookups after updating the + underlying transcript database. + """ + self.get_tx_identity_info.cache_clear() + self.get_tx_for_gene.cache_clear() + self.get_pro_ac_for_tx_ac.cache_clear() + self.get_tx_exons.cache_clear() + self.get_gene_info.cache_clear() + self.get_tx_mapping_options.cache_clear() + + def cache_info(self): + """ + Return cache statistics for every cached method. + + Each entry contains the standard functools CacheInfo tuple: + hits + misses + maxsize + currsize + + This method is intended for benchmarking and determining + whether each cache provides sufficient benefit to justify its + memory usage. + """ + return { + "get_tx_identity_info": self.get_tx_identity_info.cache_info(), + "get_tx_for_gene": self.get_tx_for_gene.cache_info(), + "get_pro_ac_for_tx_ac": self.get_pro_ac_for_tx_ac.cache_info(), + "get_tx_exons": self.get_tx_exons.cache_info(), + "get_gene_info": self.get_gene_info.cache_info(), + "get_tx_mapping_options": self.get_tx_mapping_options.cache_info(), + } + class Mixin: """ Initialise the persistent VariantValidator infrastructure. @@ -236,9 +447,17 @@ def __init__(self): # HGVS data provider # -------------------------------------------------------------- - self.hdp = vvhgvs.dataproviders.uta.connect( - pooling=True, - ) + # Create the HGVS data provider. + self.hdp = vvhgvs.dataproviders.uta.connect(pooling=True) + + # Wrap the HGVS data provider with the LRU cache layer. + # + # CachedHDP intercepts only the high-frequency database lookups and + # caches their results. All other methods and attributes are delegated + # transparently to the original HGVS data provider via __getattr__(). + # + # To disable all HDP caching, simply comment out the line below. + self.hdp = CachedHDP(self.hdp) self.utaSchema = str( self.hdp.data_version() @@ -283,6 +502,15 @@ def __init__(self): self.check_same_thread, ) + # Wrap the SeqFetcher with the LRU cache layer. + # + # CachedSeqFetcher intercepts sequence fetches and caches the returned + # sequence slices. All other methods and attributes are transparently + # delegated to the original SeqFetcher via __getattr__(). + # + # To disable all SeqFetcher caching, simply comment out the line below. + self.sf = CachedSeqFetcher(self.sf) + # -------------------------------------------------------------- # Persistent alignment-specific normalizers # -------------------------------------------------------------- From f56556b16b8b2cc195828437e900978e9030e3e5 Mon Sep 17 00:00:00 2001 From: Peter-J-Freeman Date: Sat, 1 Aug 2026 01:09:20 +0100 Subject: [PATCH 2/2] Set HGVS cache to benchmarked optimum Configure the benchmark branch with the current optimal cache configuration identified during performance testing. Changes: - Increase the HGVS internal LRU cache size to 1000 entries. - Leave the SeqFetcher cache enabled. - Disable the Local HDP cache wrapper. - Retain the Local HDP cache implementation in the codebase for future benchmarking and evaluation. Benchmarking indicates that an HGVS LRU cache size of 1000 provides the best balance between execution speed and expected memory usage for the current VariantValidator workload. Increasing the cache beyond 1000 yielded only marginal additional performance improvements. --- VariantValidator/modules/vvMixinInit.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/VariantValidator/modules/vvMixinInit.py b/VariantValidator/modules/vvMixinInit.py index dce61181..f2f37bff 100644 --- a/VariantValidator/modules/vvMixinInit.py +++ b/VariantValidator/modules/vvMixinInit.py @@ -287,6 +287,14 @@ def __init__(self): HGVS_SEQREPO_DIR.split('/')[-1] """ + # -------------------------------------------------------------- + # HGVS global configuration + # -------------------------------------------------------------- + + vvhgvs.global_config.uta.pool_max = 25 + vvhgvs.global_config.formatting.max_ref_length = 1000000 + vvhgvs.global_config.lru_cache.maxsize = 1000 + # -------------------------------------------------------------- # Configuration # -------------------------------------------------------------- @@ -436,13 +444,6 @@ def __init__(self): self.reverse_merge_normalizer = None self.no_norm_evm = None - # -------------------------------------------------------------- - # HGVS global configuration - # -------------------------------------------------------------- - - vvhgvs.global_config.uta.pool_max = 25 - vvhgvs.global_config.formatting.max_ref_length = 1000000 - # -------------------------------------------------------------- # HGVS data provider # -------------------------------------------------------------- @@ -457,7 +458,7 @@ def __init__(self): # transparently to the original HGVS data provider via __getattr__(). # # To disable all HDP caching, simply comment out the line below. - self.hdp = CachedHDP(self.hdp) + # self.hdp = CachedHDP(self.hdp) self.utaSchema = str( self.hdp.data_version()