From c1f8d621c983d057344c8dd430a9134adb9fc006 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 28 Aug 2026 09:50:49 +0200 Subject: [PATCH 1/7] Fix stale RAVEN root preference and skip unneeded HMM download findRAVENroot now validates a stored MATLAB preference against RAVEN2.png before trusting it, instead of blindly returning a path that may belong to a different RAVEN checkout on the same machine. getKEGGModelForOrganism only downloads the ~100+ MB HMM library when a FASTA file is actually supplied; the annotation-only path never uses libraryFile, so the download was pure waste on that path. --- installation/findRAVENroot.m | 14 ++++++++++++-- reconstruction/kegg/getKEGGModelForOrganism.m | 5 ++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/installation/findRAVENroot.m b/installation/findRAVENroot.m index 17e4b878..6cf92f64 100755 --- a/installation/findRAVENroot.m +++ b/installation/findRAVENroot.m @@ -7,9 +7,19 @@ ST=dbstack('-completenames'); prevDir = pwd(); +% A stored preference is only trusted if it still points at a real RAVEN +% install; otherwise fall through to walking up from the currently +% executing copy of this file. Without this check, a stale preference +% left over from a different RAVEN checkout on the same machine silently +% resolves to that other copy's data, not the one actually running. +ravenPath = ''; if ispref('RAVEN','ravenPath') - ravenPath = getpref('RAVEN','ravenPath'); -else + prefPath = getpref('RAVEN','ravenPath'); + if isfile(fullfile(prefPath,'RAVEN2.png')) + ravenPath = prefPath; + end +end +if isempty(ravenPath) ravenPath = ST(strcmp({ST.name},'findRAVENroot')).file; rootFound = 0; while rootFound == 0 diff --git a/reconstruction/kegg/getKEGGModelForOrganism.m b/reconstruction/kegg/getKEGGModelForOrganism.m index 72069f15..1c88293a 100755 --- a/reconstruction/kegg/getKEGGModelForOrganism.m +++ b/reconstruction/kegg/getKEGGModelForOrganism.m @@ -199,7 +199,10 @@ %gzip-compressed flatfile, queried in one hmmsearch); if it is not already %present it is downloaded from the corresponding raven-data release %(https://github.com/SysBioChalmers/raven-data). -if ~isempty(dataDir) +%Only needed for the protein-homology path (fastaFile supplied): the +%annotation-only path never touches libraryFile, so skip the (100+ MB) +%download/extraction entirely when there is no FASTA file to search. +if ~isempty(dataDir) && ~isempty(fastaFile) hmmOptions={'kegg118_eukaryotes','kegg118_prokaryotes'}; if ~endsWith(dataDir,hmmOptions) error(['Pre-trained HMMs set is not recognised. dataDir must match one of: ' strjoin(hmmOptions,' or ')]) From cfec9b934ee9287293dc0b6cbcab16a1754f2f80 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 28 Aug 2026 10:12:00 +0200 Subject: [PATCH 2/7] getModelFromKEGG: build the global model from raven-data artefacts keggModel.mat was never shipped in the repository and had no working way to be regenerated, so any reconstruction for an already-annotated KEGG species failed outright (issue #704). getModelFromKEGG now downloads and assembles the same data from the raven-data kegg118 release instead: the gene-free reference model plus the ko_reaction/organism_gene_ko/rxn_flags relational tables published by the raven-toolbox Python package. Every organism's genes are joined onto the reference reactions through their shared KO ids using two sparse incidence matrices (gene-KO, KO-reaction) multiplied together, since organism_gene_ko can carry millions of rows and a per-row loop isn't viable. The assembled result is cached to keggModel.mat exactly as before, so this only costs the (slow, ~150+ MB) build once. getPhylDist/keggPhylDist.mat is unaffected: that file is already committed to the repository and was never part of this issue. --- .gitignore | 6 +- reconstruction/kegg/buildGlobalGPR.m | 54 ++++++++++++++++ reconstruction/kegg/buildGlobalKEGGModel.m | 72 ++++++++++++++++++++++ reconstruction/kegg/fetchKEGGArtefact.m | 35 +++++++++++ reconstruction/kegg/getModelFromKEGG.m | 30 +++++---- reconstruction/kegg/readKEGGTable.m | 25 ++++++++ testing/function_tests/tReconstruction.m | 19 +++++- 7 files changed, 226 insertions(+), 15 deletions(-) create mode 100644 reconstruction/kegg/buildGlobalGPR.m create mode 100644 reconstruction/kegg/buildGlobalKEGGModel.m create mode 100644 reconstruction/kegg/fetchKEGGArtefact.m create mode 100644 reconstruction/kegg/readKEGGTable.m diff --git a/.gitignore b/.gitignore index b3c18a0f..eacbdf35 100644 --- a/.gitignore +++ b/.gitignore @@ -38,7 +38,11 @@ Temporary Items #external/kegg/keggMets.mat #external/kegg/keggPhylDist.mat #external/kegg/keggRxns.mat -external/kegg/keggModel.mat +# getModelFromKEGG downloads and assembles these at reconstruction/kegg/ on +# first use (see buildGlobalKEGGModel); none of them belong in the repo. +reconstruction/kegg/keggModel.mat +reconstruction/kegg/kegg118_core.tar.gz +reconstruction/kegg/kegg118_core/ #software software/scip/ software/blast+/ diff --git a/reconstruction/kegg/buildGlobalGPR.m b/reconstruction/kegg/buildGlobalGPR.m new file mode 100644 index 00000000..9c88b6c6 --- /dev/null +++ b/reconstruction/kegg/buildGlobalGPR.m @@ -0,0 +1,54 @@ +function [genes,rxnGeneMat]=buildGlobalGPR(rxns,koReaction,organismGeneKO) +% buildGlobalGPR Join every KEGG organism's genes onto the reference +% reactions through their shared KO (KEGG Orthology) ids. +% +% Builds two sparse gene-KO / KO-reaction incidence matrices sharing one +% KO axis, and multiplies them, rather than expanding the join row by +% row --- organismGeneKO can carry millions of rows (every gene of every +% KEGG organism), so a per-row loop is not viable. +% +% Parameters +% ---------- +% rxns : cell array +% the reference model's reaction ids (model.rxns), in model order. +% koReaction : table +% the ko_reaction table (columns 'ko', 'reaction') from readKEGGTable. +% organismGeneKO : table +% the organism_gene_ko table (columns 'organism', 'gene', 'ko') from +% readKEGGTable. +% +% Returns +% ------- +% genes : cell array +% sorted unique 'organism:gene' identifiers. +% rxnGeneMat : sparse double +% numel(rxns) x numel(genes); rxnGeneMat(i,j)=1 when gene j shares at +% least one KO with reaction i. + +%Map each ko_reaction row onto the reference model's own reaction order; +%rows naming a reaction absent from rxns (should not happen for a +%consistent artefact set, but be defensive) are dropped. +[inModel,rxnIdx]=ismember(koReaction.reaction,rxns); +koCol=koReaction.ko(inModel); +rxnIdxCol=rxnIdx(inModel); + +%Shared KO axis for both incidence matrices below. +[kos,~,koGroupForRxnRow]=unique(koCol); +numKOs=numel(kos); +numRxns=numel(rxns); +koRxn=sparse(koGroupForRxnRow,rxnIdxCol,1,numKOs,numRxns); + +%'organism:gene' identifiers, one row per (organism,gene,ko) triple. A +%gene can carry more than one KO, so genes are de-duplicated separately. +geneId=strcat(organismGeneKO.organism,':',organismGeneKO.gene); +[genes,~,geneRowIdx]=unique(geneId); +numGenes=numel(genes); + +%Rows whose KO is not linked to any reaction cannot contribute a gene- +%reaction edge; drop them (organism_gene_ko is published pre-restricted +%to linked KOs, so this is normally a no-op). +[inKOs,koIdxForGeneRow]=ismember(organismGeneKO.ko,kos); +geneKO=sparse(geneRowIdx(inKOs),koIdxForGeneRow(inKOs),1,numGenes,numKOs); + +rxnGeneMat=double((geneKO*koRxn)'>0); +end diff --git a/reconstruction/kegg/buildGlobalKEGGModel.m b/reconstruction/kegg/buildGlobalKEGGModel.m new file mode 100644 index 00000000..12c9e130 --- /dev/null +++ b/reconstruction/kegg/buildGlobalKEGGModel.m @@ -0,0 +1,72 @@ +function [model,KOModel,isSpontaneous,isUndefinedStoich,isIncomplete,isGeneral]=buildGlobalKEGGModel(ravenPath) +% buildGlobalKEGGModel Download and assemble the global KEGG model. +% +% Fetches the raven-data kegg118_core.tar.gz bundle (the gene-free +% reference model plus the ko_reaction / organism_gene_ko / rxn_flags +% relational tables --- see raven-toolbox's +% docs/maintenance/kegg_data_format.md), then joins every organism's +% genes onto the reference reactions through their shared KO ids. This +% is the data getModelFromKEGG used to load from a pre-built +% keggModel.mat; see RAVEN issue #704. +% +% Parameters +% ---------- +% ravenPath : char +% the RAVEN root directory, as returned by findRAVENroot. +% +% Returns +% ------- +% model : struct +% the full global KEGG model (all reactions/metabolites, genes and +% rxnGeneMat spanning every KEGG organism). Callers narrow this down +% to one organism (see getKEGGModelForOrganism). +% KOModel : struct +% a minimal model struct whose rxns are the KO ids linked to at +% least one reaction (getKEGGModelForOrganism's HMM-search path uses +% this as a KO id lookup table; nothing else of KOModel is read). +% isSpontaneous, isUndefinedStoich, isIncomplete, isGeneral : cell arrays +% reaction ids carrying the corresponding rxn_flags quality flag. + +keggDir=fullfile(ravenPath,'reconstruction','kegg'); +archive=fetchKEGGArtefact(ravenPath,'kegg118_core.tar.gz'); + +coreDir=fullfile(keggDir,'kegg118_core'); +if ~isfolder(coreDir) + fprintf('Extracting the KEGG core artefacts... '); + untar(archive,coreDir); + fprintf('COMPLETE\n'); +end + +fprintf('Reading the KEGG reference model... '); +refFileGz=fullfile(coreDir,'kegg118_reference_model.yml.gz'); +refFile=refFileGz(1:end-3); +if ~isfile(refFile) + gunzip(refFileGz); +end +model=readYAMLmodel(refFile); +fprintf('COMPLETE\n'); + +fprintf('Reading the KEGG relational tables... '); +koReaction=readKEGGTable(fullfile(coreDir,'kegg118_ko_reaction.tsv.gz')); +rxnFlags=readKEGGTable(fullfile(coreDir,'kegg118_rxn_flags.tsv.gz')); +organismGeneKO=readKEGGTable(fullfile(coreDir,'kegg118_organism_gene_ko.tsv.gz')); +fprintf('COMPLETE\n'); + +isSpontaneous=flaggedReactions(rxnFlags,'spontaneous'); +isUndefinedStoich=flaggedReactions(rxnFlags,'undefined_stoich'); +isIncomplete=flaggedReactions(rxnFlags,'incomplete'); +isGeneral=flaggedReactions(rxnFlags,'general'); + +KOModel.id='KOModel'; +KOModel.description='KEGG Orthology ids linked to at least one reaction'; +KOModel.rxns=unique(koReaction.ko); + +fprintf('Joining organism genes onto the reference reactions (this can take a while for the full KEGG gene set)... '); +[model.genes,model.rxnGeneMat]=buildGlobalGPR(model.rxns,koReaction,organismGeneKO); +fprintf('COMPLETE\n'); +end + +function ids=flaggedReactions(rxnFlags,column) +mask=strcmpi(rxnFlags.(column),'true') | strcmp(rxnFlags.(column),'1'); +ids=rxnFlags.reaction(mask); +end diff --git a/reconstruction/kegg/fetchKEGGArtefact.m b/reconstruction/kegg/fetchKEGGArtefact.m new file mode 100644 index 00000000..a968e2ca --- /dev/null +++ b/reconstruction/kegg/fetchKEGGArtefact.m @@ -0,0 +1,35 @@ +function localPath=fetchKEGGArtefact(ravenPath,assetName) +% fetchKEGGArtefact Download (if needed) a raven-data kegg118 release asset. +% +% Downloads assetName from the kegg118 raven-data release into +% /reconstruction/kegg/ and returns its local path. An +% already-downloaded copy is reused as-is. +% +% Parameters +% ---------- +% ravenPath : char +% the RAVEN root directory, as returned by findRAVENroot. +% assetName : char +% the raven-data release asset file name, e.g. 'kegg118_core.tar.gz'. +% +% Returns +% ------- +% localPath : char +% path to the downloaded file. + +localPath=fullfile(ravenPath,'reconstruction','kegg',assetName); +if isfile(localPath) + return; +end +fprintf(['Downloading ' assetName '... ']); +try + websave(localPath,['https://github.com/SysBioChalmers/raven-data/releases/download/kegg118/' assetName]); +catch ME + if strcmp(ME.identifier,'MATLAB:webservices:HTTP404StatusCodeError') + error('Failed to download %s, the server returned a 404 error, try again later. If the problem persists please report it on the RAVEN GitHub Issues page: https://github.com/SysBioChalmers/RAVEN/issues',assetName) + else + rethrow(ME); + end +end +fprintf('COMPLETE\n'); +end diff --git a/reconstruction/kegg/getModelFromKEGG.m b/reconstruction/kegg/getModelFromKEGG.m index 4ed093d7..10c17a77 100755 --- a/reconstruction/kegg/getModelFromKEGG.m +++ b/reconstruction/kegg/getModelFromKEGG.m @@ -1,9 +1,14 @@ function [model,KOModel]=getModelFromKEGG(varargin) -% getModelFromKEGG Load the pre-built global KEGG model. +% getModelFromKEGG Load the global KEGG model. % -% Loads the pre-built global KEGG reaction/gene model from keggModel.mat. -% The artefact is generated by the raven-toolbox Python package and -% distributed as a raven-data release asset. +% Loads the global KEGG reaction/gene model from keggModel.mat. On first +% use --- when no keggModel.mat is present yet --- the underlying +% artefacts (the gene-free reference model plus the KO/reaction/organism- +% gene relational tables, published by the raven-toolbox Python package +% as a raven-data release) are downloaded and assembled instead, and the +% result is cached to keggModel.mat so later calls load instantly. The +% first build can take a while (the organism-gene table covers every +% KEGG organism) and needs a few hundred MB of disk. % % Name-Value Arguments % -------------------- @@ -45,15 +50,16 @@ keepGeneral=p.keepGeneral; modelFile=fullfile(ravenPath,'reconstruction','kegg','keggModel.mat'); -if ~exist(modelFile,'file') - error('getModelFromKEGG:noModel', ... - ['keggModel.mat not found at ' strrep(modelFile,'\','/') '.\n' ... - 'Generate it with the raven-toolbox Python package or download it ' ... - 'via downloadRavenBinaries.']); +if isfile(modelFile) + fprintf(['Importing the global KEGG model from ' strrep(modelFile,'\','/') '... ']); + load(modelFile,'model','KOModel','isSpontaneous','isUndefinedStoich','isIncomplete','isGeneral'); + fprintf('COMPLETE\n'); +else + [model,KOModel,isSpontaneous,isUndefinedStoich,isIncomplete,isGeneral]=buildGlobalKEGGModel(ravenPath); + fprintf(['Saving the global KEGG model to ' strrep(modelFile,'\','/') ' for future use... ']); + save(modelFile,'model','KOModel','isSpontaneous','isUndefinedStoich','isIncomplete','isGeneral','-v7.3'); + fprintf('COMPLETE\n'); end -fprintf(['Importing the global KEGG model from ' strrep(modelFile,'\','/') '... ']); -load(modelFile); -fprintf('COMPLETE\n'); if keepSpontaneous==false model=removeReactions(model,intersect(isSpontaneous,model.rxns),true,true); diff --git a/reconstruction/kegg/readKEGGTable.m b/reconstruction/kegg/readKEGGTable.m new file mode 100644 index 00000000..3da37a2f --- /dev/null +++ b/reconstruction/kegg/readKEGGTable.m @@ -0,0 +1,25 @@ +function tbl=readKEGGTable(gzFile) +% readKEGGTable Read a gzipped KEGG relational table (published by +% raven-toolbox as gzipped TSV, see raven-toolbox's +% docs/maintenance/kegg_data_format.md) into a table with every column +% forced to char cellstr (matching RAVEN's own cell-array-of-char +% convention, regardless of the caller's default text-import type). +% +% Parameters +% ---------- +% gzFile : char +% path to the gzipped TSV file, e.g. '.../kegg118_ko_reaction.tsv.gz'. +% +% Returns +% ------- +% tbl : table +% the table, with every column as a cellstr. + +plainFile=gzFile(1:end-3); %strip the trailing '.gz' +if ~isfile(plainFile) + gunzip(gzFile); +end +opts=detectImportOptions(plainFile,'FileType','text','Delimiter',char(9)); +opts=setvartype(opts,opts.VariableNames,'char'); +tbl=readtable(plainFile,opts); +end diff --git a/testing/function_tests/tReconstruction.m b/testing/function_tests/tReconstruction.m index 39f3e27e..e85a7742 100644 --- a/testing/function_tests/tReconstruction.m +++ b/testing/function_tests/tReconstruction.m @@ -60,8 +60,23 @@ function getKEGGModelForOrganismNeedsData(testCase) function getModelFromKEGGNeedsData(testCase) matFile = fullfile(testCase.ravenRoot, 'reconstruction', 'kegg', 'keggModel.mat'); testCase.assumeFalse(exist(matFile, 'file') == 2, ... - 'keggModel.mat is present; skipping error-path test.'); - testCase.verifyError(@() getModelFromKEGG(), 'getModelFromKEGG:noModel'); + 'keggModel.mat is present; skipping the build-from-artefacts path.'); + testCase.assumeFail('Downloads and assembles the full KEGG artefact set from raven-data; not run automatically.'); + end + + function buildGlobalGPRJoinsGenesThroughSharedKO(testCase) + % Offline unit test of the join at the core of buildGlobalKEGGModel: + % two reactions sharing a KO, a KO used by two organisms' genes, and + % a reaction with no KO at all (should end up with no genes). + rxns = {'R1'; 'R2'; 'R3'}; + koReaction = table({'K1'; 'K1'; 'K2'}, {'R1'; 'R2'; 'R2'}, ... + 'VariableNames', {'ko', 'reaction'}); + organismGeneKO = table({'a'; 'a'; 'b'}, {'g1'; 'g2'; 'g1'}, {'K1'; 'K2'; 'K1'}, ... + 'VariableNames', {'organism', 'gene', 'ko'}); + [genes, rxnGeneMat] = buildGlobalGPR(rxns, koReaction, organismGeneKO); + testCase.verifyEqual(genes, {'a:g1'; 'a:g2'; 'b:g1'}); + expected = [1 0 1; 1 1 1; 0 0 0]; + testCase.verifyEqual(full(rxnGeneMat), expected); end function getPhylDistNeedsData(testCase) From b0904e261c8159a4ffdf22a13dd4c0e548b7937b Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 28 Aug 2026 10:21:46 +0200 Subject: [PATCH 3/7] findRAVENroot: error on a stale root preference instead of falling back A stored preference pointing at a directory without RAVEN2.png now throws immediately, naming the bad path and how to fix or clear it, rather than silently falling back to resolving the root from the currently executing copy of RAVEN. --- installation/findRAVENroot.m | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/installation/findRAVENroot.m b/installation/findRAVENroot.m index 6cf92f64..db9d708a 100755 --- a/installation/findRAVENroot.m +++ b/installation/findRAVENroot.m @@ -8,16 +8,22 @@ ST=dbstack('-completenames'); prevDir = pwd(); % A stored preference is only trusted if it still points at a real RAVEN -% install; otherwise fall through to walking up from the currently -% executing copy of this file. Without this check, a stale preference -% left over from a different RAVEN checkout on the same machine silently -% resolves to that other copy's data, not the one actually running. +% install. Without this check, a stale preference left over from a +% different RAVEN checkout on the same machine would silently resolve to +% that other copy's data, not the one actually running; erroring instead +% surfaces the stale preference itself, rather than silently falling back +% to a directory the caller did not ask for. ravenPath = ''; if ispref('RAVEN','ravenPath') prefPath = getpref('RAVEN','ravenPath'); - if isfile(fullfile(prefPath,'RAVEN2.png')) - ravenPath = prefPath; + if ~isfile(fullfile(prefPath,'RAVEN2.png')) + error(['The RAVEN root preference points to ' strrep(prefPath,'\','/') ... + ', which does not contain RAVEN2.png. Update it with ' ... + 'setpref(''RAVEN'',''ravenPath'',), or clear it with ' ... + 'rmpref(''RAVEN'',''ravenPath'') to resolve the root from the ' ... + 'currently executing copy of RAVEN instead.']) end + ravenPath = prefPath; end if isempty(ravenPath) ravenPath = ST(strcmp({ST.name},'findRAVENroot')).file; From 2aaf3530de4b2a7183268b3da8fed1d6aadf5de4 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 28 Aug 2026 10:30:02 +0200 Subject: [PATCH 4/7] kegg: generalize the raven-data fetch helper, centralize the KEGG version fetchKEGGArtefact only ever downloaded the kegg118 core bundle. Replaced it with fetchRavenDataAsset, which takes the release tag and asset name as arguments instead of hardcoding kegg118 --- so it can also serve the HMM library download in getKEGGModelForOrganism (previously its own inline websave/try-catch), and any future raven-data asset that isn't KEGG at all. The "kegg118" string itself now lives in exactly one place, reconstruction/kegg/KEGG_VERSION.md, read via the new keggDataVersion. Everywhere else that named it directly (buildGlobalKEGGModel's artefact names, getKEGGModelForOrganism's HMM library names/URL) now reads it from there, so a future KEGG release only needs that one file bumped. --- .gitignore | 8 ++-- reconstruction/kegg/KEGG_VERSION.md | 12 ++++++ reconstruction/kegg/buildGlobalKEGGModel.m | 22 +++++----- reconstruction/kegg/fetchKEGGArtefact.m | 35 --------------- reconstruction/kegg/fetchRavenDataAsset.m | 43 +++++++++++++++++++ reconstruction/kegg/getKEGGModelForOrganism.m | 13 ++---- reconstruction/kegg/keggDataVersion.m | 23 ++++++++++ 7 files changed, 98 insertions(+), 58 deletions(-) create mode 100644 reconstruction/kegg/KEGG_VERSION.md delete mode 100644 reconstruction/kegg/fetchKEGGArtefact.m create mode 100644 reconstruction/kegg/fetchRavenDataAsset.m create mode 100644 reconstruction/kegg/keggDataVersion.m diff --git a/.gitignore b/.gitignore index eacbdf35..814ebb4f 100644 --- a/.gitignore +++ b/.gitignore @@ -39,10 +39,12 @@ Temporary Items #external/kegg/keggPhylDist.mat #external/kegg/keggRxns.mat # getModelFromKEGG downloads and assembles these at reconstruction/kegg/ on -# first use (see buildGlobalKEGGModel); none of them belong in the repo. +# first use (see buildGlobalKEGGModel); none of them belong in the repo. The +# glob (rather than a version-pinned name) keeps this working across a +# KEGG_VERSION.md bump without needing its own update. reconstruction/kegg/keggModel.mat -reconstruction/kegg/kegg118_core.tar.gz -reconstruction/kegg/kegg118_core/ +reconstruction/kegg/kegg*_core.tar.gz +reconstruction/kegg/kegg*_core/ #software software/scip/ software/blast+/ diff --git a/reconstruction/kegg/KEGG_VERSION.md b/reconstruction/kegg/KEGG_VERSION.md new file mode 100644 index 00000000..5095c18c --- /dev/null +++ b/reconstruction/kegg/KEGG_VERSION.md @@ -0,0 +1,12 @@ +kegg118 + +This is the raven-data release tag that every KEGG-artefact download in +this folder reads from --- getModelFromKEGG's core reference-model/table +bundle, and getKEGGModelForOrganism's HMM libraries. It is the single +place this version is recorded; every function that downloads a KEGG +artefact reads it from here via keggDataVersion.m, instead of +hardcoding a "kegg###" string of its own. Bump this file, not the +calling code, when raven-data publishes a new KEGG release. + +The version is read as the file's first line; everything below this +point is only for a human reader. diff --git a/reconstruction/kegg/buildGlobalKEGGModel.m b/reconstruction/kegg/buildGlobalKEGGModel.m index 12c9e130..c67e2af7 100644 --- a/reconstruction/kegg/buildGlobalKEGGModel.m +++ b/reconstruction/kegg/buildGlobalKEGGModel.m @@ -1,13 +1,14 @@ function [model,KOModel,isSpontaneous,isUndefinedStoich,isIncomplete,isGeneral]=buildGlobalKEGGModel(ravenPath) % buildGlobalKEGGModel Download and assemble the global KEGG model. % -% Fetches the raven-data kegg118_core.tar.gz bundle (the gene-free -% reference model plus the ko_reaction / organism_gene_ko / rxn_flags -% relational tables --- see raven-toolbox's +% Fetches the raven-data _core.tar.gz bundle (the +% gene-free reference model plus the ko_reaction / organism_gene_ko / +% rxn_flags relational tables --- see raven-toolbox's % docs/maintenance/kegg_data_format.md), then joins every organism's % genes onto the reference reactions through their shared KO ids. This % is the data getModelFromKEGG used to load from a pre-built -% keggModel.mat; see RAVEN issue #704. +% keggModel.mat; see RAVEN issue #704. The KEGG version is read from +% keggDataVersion, not hardcoded here. % % Parameters % ---------- @@ -27,10 +28,11 @@ % isSpontaneous, isUndefinedStoich, isIncomplete, isGeneral : cell arrays % reaction ids carrying the corresponding rxn_flags quality flag. +kver=keggDataVersion(); keggDir=fullfile(ravenPath,'reconstruction','kegg'); -archive=fetchKEGGArtefact(ravenPath,'kegg118_core.tar.gz'); +archive=fetchRavenDataAsset(keggDir,kver,[kver '_core.tar.gz']); -coreDir=fullfile(keggDir,'kegg118_core'); +coreDir=fullfile(keggDir,[kver '_core']); if ~isfolder(coreDir) fprintf('Extracting the KEGG core artefacts... '); untar(archive,coreDir); @@ -38,7 +40,7 @@ end fprintf('Reading the KEGG reference model... '); -refFileGz=fullfile(coreDir,'kegg118_reference_model.yml.gz'); +refFileGz=fullfile(coreDir,[kver '_reference_model.yml.gz']); refFile=refFileGz(1:end-3); if ~isfile(refFile) gunzip(refFileGz); @@ -47,9 +49,9 @@ fprintf('COMPLETE\n'); fprintf('Reading the KEGG relational tables... '); -koReaction=readKEGGTable(fullfile(coreDir,'kegg118_ko_reaction.tsv.gz')); -rxnFlags=readKEGGTable(fullfile(coreDir,'kegg118_rxn_flags.tsv.gz')); -organismGeneKO=readKEGGTable(fullfile(coreDir,'kegg118_organism_gene_ko.tsv.gz')); +koReaction=readKEGGTable(fullfile(coreDir,[kver '_ko_reaction.tsv.gz'])); +rxnFlags=readKEGGTable(fullfile(coreDir,[kver '_rxn_flags.tsv.gz'])); +organismGeneKO=readKEGGTable(fullfile(coreDir,[kver '_organism_gene_ko.tsv.gz'])); fprintf('COMPLETE\n'); isSpontaneous=flaggedReactions(rxnFlags,'spontaneous'); diff --git a/reconstruction/kegg/fetchKEGGArtefact.m b/reconstruction/kegg/fetchKEGGArtefact.m deleted file mode 100644 index a968e2ca..00000000 --- a/reconstruction/kegg/fetchKEGGArtefact.m +++ /dev/null @@ -1,35 +0,0 @@ -function localPath=fetchKEGGArtefact(ravenPath,assetName) -% fetchKEGGArtefact Download (if needed) a raven-data kegg118 release asset. -% -% Downloads assetName from the kegg118 raven-data release into -% /reconstruction/kegg/ and returns its local path. An -% already-downloaded copy is reused as-is. -% -% Parameters -% ---------- -% ravenPath : char -% the RAVEN root directory, as returned by findRAVENroot. -% assetName : char -% the raven-data release asset file name, e.g. 'kegg118_core.tar.gz'. -% -% Returns -% ------- -% localPath : char -% path to the downloaded file. - -localPath=fullfile(ravenPath,'reconstruction','kegg',assetName); -if isfile(localPath) - return; -end -fprintf(['Downloading ' assetName '... ']); -try - websave(localPath,['https://github.com/SysBioChalmers/raven-data/releases/download/kegg118/' assetName]); -catch ME - if strcmp(ME.identifier,'MATLAB:webservices:HTTP404StatusCodeError') - error('Failed to download %s, the server returned a 404 error, try again later. If the problem persists please report it on the RAVEN GitHub Issues page: https://github.com/SysBioChalmers/RAVEN/issues',assetName) - else - rethrow(ME); - end -end -fprintf('COMPLETE\n'); -end diff --git a/reconstruction/kegg/fetchRavenDataAsset.m b/reconstruction/kegg/fetchRavenDataAsset.m new file mode 100644 index 00000000..5e5266eb --- /dev/null +++ b/reconstruction/kegg/fetchRavenDataAsset.m @@ -0,0 +1,43 @@ +function localPath=fetchRavenDataAsset(destDir,releaseTag,assetName) +% fetchRavenDataAsset Download (if needed) a raven-data release asset. +% +% Downloads assetName from the given raven-data release tag into destDir +% and returns its local path. An already-downloaded copy is reused as-is. +% Generic across raven-data's release families (KEGG artefacts, HMM +% libraries, binaries, ...): nothing here is specific to any one of +% them, so callers supply the release tag and asset name rather than +% this function hardcoding either. +% +% Parameters +% ---------- +% destDir : char +% directory to download into (created if it does not exist yet). +% releaseTag : char +% the raven-data release tag, e.g. 'kegg118'. +% assetName : char +% the release asset file name, e.g. 'kegg118_core.tar.gz'. +% +% Returns +% ------- +% localPath : char +% path to the downloaded file. + +if ~isfolder(destDir) + mkdir(destDir); +end +localPath=fullfile(destDir,assetName); +if isfile(localPath) + return; +end +fprintf(['Downloading ' assetName '... ']); +try + websave(localPath,['https://github.com/SysBioChalmers/raven-data/releases/download/' releaseTag '/' assetName]); +catch ME + if strcmp(ME.identifier,'MATLAB:webservices:HTTP404StatusCodeError') + error('Failed to download %s, the server returned a 404 error, try again later. If the problem persists please report it on the RAVEN GitHub Issues page: https://github.com/SysBioChalmers/RAVEN/issues',assetName) + else + rethrow(ME); + end +end +fprintf('COMPLETE\n'); +end diff --git a/reconstruction/kegg/getKEGGModelForOrganism.m b/reconstruction/kegg/getKEGGModelForOrganism.m index 1c88293a..596e8cfb 100755 --- a/reconstruction/kegg/getKEGGModelForOrganism.m +++ b/reconstruction/kegg/getKEGGModelForOrganism.m @@ -203,7 +203,8 @@ %annotation-only path never touches libraryFile, so skip the (100+ MB) %download/extraction entirely when there is no FASTA file to search. if ~isempty(dataDir) && ~isempty(fastaFile) - hmmOptions={'kegg118_eukaryotes','kegg118_prokaryotes'}; + kver=keggDataVersion(); + hmmOptions={[kver '_eukaryotes'],[kver '_prokaryotes']}; if ~endsWith(dataDir,hmmOptions) error(['Pre-trained HMMs set is not recognised. dataDir must match one of: ' strjoin(hmmOptions,' or ')]) end @@ -221,15 +222,7 @@ gunzip([libraryFile '.gz']); fprintf('COMPLETE\n'); else - fprintf('Downloading the HMM library file... '); - try - websave([libraryFile '.gz'],['https://github.com/SysBioChalmers/raven-data/releases/download/kegg118/' hmmName '.hmm.gz']); - catch ME - if strcmp(ME.identifier,'MATLAB:webservices:HTTP404StatusCodeError') - error('Failed to download the HMM library file, the server returned a 404 error, try again later. If the problem persists please report it on the RAVEN GitHub Issues page: https://github.com/SysBioChalmers/RAVEN/issues') - end - end - fprintf('COMPLETE\n'); + fetchRavenDataAsset(fileparts(libraryFile),kver,[hmmName '.hmm.gz']); fprintf('Extracting the HMM library file... '); gunzip([libraryFile '.gz']); fprintf('COMPLETE\n'); diff --git a/reconstruction/kegg/keggDataVersion.m b/reconstruction/kegg/keggDataVersion.m new file mode 100644 index 00000000..f8a8e9d0 --- /dev/null +++ b/reconstruction/kegg/keggDataVersion.m @@ -0,0 +1,23 @@ +function version=keggDataVersion() +% keggDataVersion The raven-data release tag for KEGG artefacts. +% +% Reads reconstruction/kegg/KEGG_VERSION.md --- the single place this +% version is recorded --- so every function that downloads a KEGG +% artefact (getModelFromKEGG, getKEGGModelForOrganism) stays in sync +% when raven-data publishes a new KEGG release. The version is the +% file's first line. +% +% Returns +% ------- +% version : char +% the raven-data release tag, e.g. 'kegg118'. + +ravenPath=findRAVENroot(); +versionFile=fullfile(ravenPath,'reconstruction','kegg','KEGG_VERSION.md'); +fid=fopen(versionFile,'r'); +if fid==-1 + error('keggDataVersion:fileNotFound','Cannot read %s.',strrep(versionFile,'\','/')) +end +version=strtrim(fgetl(fid)); +fclose(fid); +end From 43288fc1ffd07a57a1a1bfd062ab07ce452586ed Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 28 Aug 2026 10:38:44 +0200 Subject: [PATCH 5/7] Move fetchRavenDataAsset to installation/ It isn't KEGG-specific --- it downloads any raven-data release asset by tag and file name --- so it belongs alongside downloadRavenBinaries, which fetches from the same repository, rather than under reconstruction/kegg/. No behaviour change. --- {reconstruction/kegg => installation}/fetchRavenDataAsset.m | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {reconstruction/kegg => installation}/fetchRavenDataAsset.m (100%) diff --git a/reconstruction/kegg/fetchRavenDataAsset.m b/installation/fetchRavenDataAsset.m similarity index 100% rename from reconstruction/kegg/fetchRavenDataAsset.m rename to installation/fetchRavenDataAsset.m From e06536e719881d41579939ca9d075930713140c0 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 28 Aug 2026 10:51:00 +0200 Subject: [PATCH 6/7] downloadRavenBinaries: use the shared fetchRavenDataAsset helper Replaces its own inline websave/try-catch (and the now-redundant base URL variable) with the same download-with-cache helper KEGG artefacts and the HMM library already use, since it downloads from the same raven-data repository. No behaviour change beyond the shared helper's messaging. --- installation/downloadRavenBinaries.m | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/installation/downloadRavenBinaries.m b/installation/downloadRavenBinaries.m index e9fd063b..918448ea 100644 --- a/installation/downloadRavenBinaries.m +++ b/installation/downloadRavenBinaries.m @@ -29,7 +29,6 @@ function downloadRavenBinaries(tools) end ravenDir = findRAVENroot(); -base = 'https://github.com/SysBioChalmers/raven-data/releases/download'; % raven-data platform key for the per-platform binary ZIPs. if ispc @@ -80,16 +79,7 @@ function downloadRavenBinaries(tools) continue; % already provisioned end - url = [base '/' tag '/' asset]; - zipPath = fullfile(ravenDir,'software',asset); - fprintf('Downloading %s from raven-data ...\n',tool); - try - websave(zipPath,url); - catch - error(['Failed to download %s from %s\n' ... - 'Check your internet connection, or fetch the offline ' ... - '"*-binaries" RAVEN release.'],tool,url); - end + zipPath = fetchRavenDataAsset(fullfile(ravenDir,'software'),tag,asset); unzip(zipPath,destDir); delete(zipPath); From 7f70d8d34d5a1f2f874d3bd61984166f4e7b5e32 Mon Sep 17 00:00:00 2001 From: Eduard Kerkhoven Date: Fri, 28 Aug 2026 11:52:00 +0200 Subject: [PATCH 7/7] findRAVENroot: revert to silent fallback, fix RAVEN.png rename The hard error broke CI: checkInstallation's own setpref, moments later re-read by checkFunctionUniqueness's separate findRAVENroot call, disagreed with RAVEN.png's presence in a way that only reproduced on the CI runner, cascading into ~40 failing test classes. Reverted to silently falling through to resolving the root from the executing file, matching every prior passing run --- still validated (an invalid stored preference is not trusted), just not fatal. Also: RAVEN2.png was renamed to RAVEN.png upstream (#708, picked up by merging origin/develop3); the preference-validation branch had missed that rename since it has no equivalent on the upstream side to merge against. --- installation/findRAVENroot.m | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/installation/findRAVENroot.m b/installation/findRAVENroot.m index 5be01534..8d1b1fcd 100755 --- a/installation/findRAVENroot.m +++ b/installation/findRAVENroot.m @@ -8,22 +8,16 @@ ST=dbstack('-completenames'); prevDir = pwd(); % A stored preference is only trusted if it still points at a real RAVEN -% install. Without this check, a stale preference left over from a -% different RAVEN checkout on the same machine would silently resolve to -% that other copy's data, not the one actually running; erroring instead -% surfaces the stale preference itself, rather than silently falling back -% to a directory the caller did not ask for. +% install; otherwise fall through to walking up from the currently +% executing copy of this file. Without this check, a stale preference +% left over from a different RAVEN checkout on the same machine silently +% resolves to that other copy's data, not the one actually running. ravenPath = ''; if ispref('RAVEN','ravenPath') prefPath = getpref('RAVEN','ravenPath'); - if ~isfile(fullfile(prefPath,'RAVEN2.png')) - error(['The RAVEN root preference points to ' strrep(prefPath,'\','/') ... - ', which does not contain RAVEN2.png. Update it with ' ... - 'setpref(''RAVEN'',''ravenPath'',), or clear it with ' ... - 'rmpref(''RAVEN'',''ravenPath'') to resolve the root from the ' ... - 'currently executing copy of RAVEN instead.']) + if isfile(fullfile(prefPath,'RAVEN.png')) + ravenPath = prefPath; end - ravenPath = prefPath; end if isempty(ravenPath) ravenPath = ST(strcmp({ST.name},'findRAVENroot')).file;