diff --git a/CMakeLists.txt b/CMakeLists.txt index 01199cade..0e2931986 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2389,6 +2389,19 @@ if (ENGINE_BUILD_TESTS) NAME server_model_installer_test COMMAND server_model_installer_test ) + + add_executable(package_manager_modelscope_test + tests/unittests/test_package_manager_modelscope.cpp + ) + target_link_libraries(package_manager_modelscope_test PRIVATE + Threads::Threads audiocpp_package_manager) + target_compile_definitions(package_manager_modelscope_test PRIVATE + AUDIOCPP_NATIVE_MANAGER_FIXTURE="${CMAKE_CURRENT_SOURCE_DIR}/tests/fixtures/native_model_manager_server.py") + + add_test( + NAME package_manager_modelscope_test + COMMAND package_manager_modelscope_test + ) endif() if (ENGINE_BUILD_MODEL_TESTS) diff --git a/docs/maintainers/model_specs.md b/docs/maintainers/model_specs.md index 87a30f0fd..a0cd766d5 100644 --- a/docs/maintainers/model_specs.md +++ b/docs/maintainers/model_specs.md @@ -181,6 +181,15 @@ overrides. } ``` +`kind: "modelscope_snapshot"` downloads the same way from a ModelScope +(modelscope.cn) repo. It takes the same fields (`repo` required, `revision` +optional); the only differences are that the default revision is `master` +(ModelScope's default branch) and the `gated` flag does not apply. The native +package manager resolves the endpoint through `AUDIOCPP_MS_BASE_URL` +(default `https://www.modelscope.cn`), mirroring `AUDIOCPP_HF_BASE_URL` for +Hugging Face. ModelScope requests authenticate with `AUDIOCPP_MS_TOKEN` only; +the Hugging Face token is never sent to a ModelScope endpoint. + Dependencies describe extra model-level resources required by runtime features. Use `kind: "model"` for another model family, and `kind: "bundled_model"` for an in-repo bundled model asset. Do not use dependencies for sidecars or tensor diff --git a/docs/model_manager.md b/docs/model_manager.md index 71e291f67..bd42d713a 100644 --- a/docs/model_manager.md +++ b/docs/model_manager.md @@ -122,6 +122,44 @@ For support status and tested precision coverage, see the [GGUF guide](gguf.md). For measured 16-bit vs Q8 speed and peak-VRAM results, see the [Q8 performance report](reports/gguf_q8_performance.md). +## Download Sources + +Both the native package manager and `tools/model_manager_v2.py` download from +Hugging Face (`kind: "huggingface_snapshot"`) or ModelScope +(`kind: "modelscope_snapshot"`); see +[maintainers/model_specs.md](maintainers/model_specs.md) for the spec fields. +The native endpoints can be overridden for mirrors or tests with +`AUDIOCPP_HF_BASE_URL` (default `https://huggingface.co`) and +`AUDIOCPP_MS_BASE_URL` (default `https://www.modelscope.cn`). The Python tool +uses the standard `HF_ENDPOINT` for Hugging Face and the same +`AUDIOCPP_MS_BASE_URL` for ModelScope. + +Authentication is provider-scoped: Hugging Face requests may carry +`HF_TOKEN` / `HUGGING_FACE_HUB_TOKEN`, while ModelScope requests only carry +`AUDIOCPP_MS_TOKEN` (optional, for access-restricted ModelScope repos). The +Hugging Face token is never sent to a ModelScope endpoint, and vice versa. + +### Python Source Override + +The Python v2 manager can redirect any package to ModelScope on demand, +without editing `model_specs/*.json`: + +```bash +python3 tools/model_manager_v2.py install qwen3_tts --source modelscope --source-repo HereIsMark/audio.cpp-gguf +``` + +`--source modelscope` is accepted by `install` and `sizes`. `--source-repo` +names the ModelScope repo (`namespace/name`); when omitted, the spec's own +repo name is reused on ModelScope. Passing `--source-repo` without +`--source modelscope` is rejected. Revision translation: a spec revision that +is unset or `main` becomes ModelScope's default branch `master`; any other +explicit revision passes through unchanged. + +Note that manifest etags are source-specific (Hugging Face etag vs ModelScope +sha256), so cross-source freshness checks can spuriously report that an +installed package has an update. Query with the same `--source` that was used +to install. + ## Dependencies The native manager needs no Python runtime. The default bundled-TLS build needs diff --git a/src/framework/model_spec/schema.cpp b/src/framework/model_spec/schema.cpp index fbc49aa9e..fbc159cd2 100644 --- a/src/framework/model_spec/schema.cpp +++ b/src/framework/model_spec/schema.cpp @@ -104,7 +104,7 @@ const std::unordered_set & precisions() { const std::unordered_set & download_kinds() { static const std::unordered_set values = { - "huggingface_snapshot", "local_snapshot", "converter", "unsupported", + "huggingface_snapshot", "modelscope_snapshot", "local_snapshot", "converter", "unsupported", }; return values; } @@ -408,7 +408,7 @@ void validate_runtime(const json::Value & value, std::string_view path) { validate_string_array(require_spec_field(value, "tags", path), &runtime_tags(), std::string(path) + ".tags", "runtime tag"); } -void validate_hf_snapshot_download(const json::Value & value, std::string_view path) { +void validate_snapshot_download(const json::Value & value, std::string_view path) { require_spec_object(value, path); (void) require_spec_string(require_spec_field(value, "repo", path), std::string(path) + ".repo"); if (const auto * revision = value.find("revision")) { @@ -431,8 +431,8 @@ void validate_download(const json::Value & value, std::string_view path) { require_spec_object(value, path); const auto kind = require_spec_string(require_spec_field(value, "kind", path), std::string(path) + ".kind"); validate_enum(kind, download_kinds(), std::string(path) + ".kind", "download kind"); - if (kind == "huggingface_snapshot") { - validate_hf_snapshot_download(value, path); + if (kind == "huggingface_snapshot" || kind == "modelscope_snapshot") { + validate_snapshot_download(value, path); } else if (kind == "local_snapshot") { (void) require_spec_string(require_spec_field(value, "path", path), std::string(path) + ".path"); if (const auto * array = value.find("include")) { diff --git a/src/framework/package_manager/manager.cpp b/src/framework/package_manager/manager.cpp index 87ad42202..f7549ddb9 100644 --- a/src/framework/package_manager/manager.cpp +++ b/src/framework/package_manager/manager.cpp @@ -45,7 +45,7 @@ struct Package { std::string strip_prefix; std::string download_kind; std::string repo; - std::string revision = "main"; + std::string revision; bool gated = false; }; @@ -94,6 +94,10 @@ std::string huggingface_token() { return token; } +std::string modelscope_token() { + return getenv_text("AUDIOCPP_MS_TOKEN"); +} + bool unreserved(unsigned char ch) { return std::isalnum(ch) != 0 || ch == '-' || ch == '_' || ch == '.' || ch == '~'; } @@ -121,6 +125,23 @@ std::string hf_url(const Package & package, const std::string & remote_path) { url_encode(package.revision, false) + "/" + url_encode(remote_path, true); } +std::string modelscope_base_url() { + auto base = getenv_text("AUDIOCPP_MS_BASE_URL"); + if (base.empty()) base = "https://www.modelscope.cn"; + while (!base.empty() && base.back() == '/') base.pop_back(); + return base; +} + +std::string ms_url(const Package & package, const std::string & remote_path) { + return modelscope_base_url() + "/models/" + package.repo + "/resolve/" + + url_encode(package.revision, false) + "/" + url_encode(remote_path, true); +} + +std::string ms_files_url(const Package & package) { + return modelscope_base_url() + "/api/v1/models/" + package.repo + + "/repo/files?Revision=" + url_encode(package.revision, false) + "&Recursive=true"; +} + std::filesystem::path validate_relative_path(const std::filesystem::path & path, const char * label) { if (path.empty() || path == "." || path.is_absolute()) { throw std::runtime_error(std::string(label) + " must be a non-empty relative path: " + path.string()); @@ -192,13 +213,13 @@ std::vector parse_specs(const std::vectorfind("download")) { default_kind = json::optional_string(*download, "kind", ""); default_repo = json::optional_string(*download, "repo", ""); - default_revision = json::optional_string(*download, "revision", "main"); + default_revision = json::optional_string(*download, "revision", ""); default_gated = json::optional_bool(*download, "gated", false); } } @@ -225,9 +246,15 @@ std::vector parse_specs(const std::vector & cancelled, - const std::function & progress) { + const std::function & progress, + RequestAuth auth) { const auto parsed = parse_http_url(url); httplib::Client client(http_origin(parsed)); - client.set_follow_location(true); - client.set_connection_timeout(60, 0); - client.set_read_timeout(head ? 60 : 300, 0); - client.set_write_timeout(60, 0); -#ifdef CPPHTTPLIB_OPENSSL_SUPPORT - client.enable_server_certificate_verification(true); -#endif + configure_client(client, head ? 60 : 300); - httplib::Headers request_headers{{"User-Agent", "audio.cpp native model manager/1.0"}}; - const auto token = huggingface_token(); - if (!token.empty()) { - request_headers.emplace("Authorization", "Bearer " + token); - } + const auto headers = request_headers(auth); uint64_t downloaded = 0; bool write_failed = false; @@ -369,8 +417,8 @@ HttpResult http_request( }; httplib::Result response = head - ? client.Head(parsed.path, request_headers) - : client.Get(parsed.path, request_headers, receiver, keep_downloading); + ? client.Head(parsed.path, headers) + : client.Get(parsed.path, headers, receiver, keep_downloading); if (!response) { if (cancelled && cancelled->load()) throw Cancelled(); if (write_failed) throw std::runtime_error("could not write downloaded model file"); @@ -385,8 +433,104 @@ HttpResult http_request( return result; } -RemoteFileInfo remote_info(const Package & package, const std::string & remote) { - const auto response = http_request(hf_url(package, remote), true, nullptr, {}, {}); +HttpResult http_get_body(const std::string & url, std::string & body, RequestAuth auth) { + const auto parsed = parse_http_url(url); + httplib::Client client(http_origin(parsed)); + configure_client(client, 60); + auto response = client.Get(parsed.path, request_headers(auth)); + if (!response) { + throw std::runtime_error("model host request failed: " + httplib::to_string(response.error())); + } + HttpResult result; + result.status = response->status; + for (const auto & [name, value] : response->headers) { + result.headers[lower(name)] = value; + } + body = std::move(response->body); + return result; +} + +using RemoteFileMap = std::map; + +// ModelScope resolve HEAD responses carry no Content-Length or ETag, so +// per-file size and checksum come from the repo file-list API instead. The +// listing is fetched once per repo+revision and shared through this cache. +using ListingCache = std::map; + +RemoteFileMap fetch_modelscope_listing(const Package & package) { + std::string body; + const auto response = http_get_body(ms_files_url(package), body, RequestAuth::modelscope); + if (response.status < 200 || response.status >= 300) { + throw std::runtime_error("ModelScope repo listing is not accessible: " + package.repo + + " (HTTP " + std::to_string(response.status) + ")"); + } + const auto root = json::parse(body); + if (json::optional_i64(root, "Code", 0) != 200) { + throw std::runtime_error("ModelScope repo listing failed for " + package.repo + ": " + + json::optional_string(root, "Message", "unknown error")); + } + const auto * data = root.find("Data"); + const auto * files = data != nullptr ? data->find("Files") : nullptr; + if (files == nullptr || !files->is_array()) { + throw std::runtime_error("ModelScope repo listing has no file list: " + package.repo); + } + RemoteFileMap listing; + for (const auto & item : files->as_array()) { + if (json::optional_string(item, "Type", "") != "blob") continue; + const auto path = json::optional_string(item, "Path", ""); + if (path.empty()) continue; + RemoteFileInfo info; + info.etag = json::optional_string(item, "Sha256", ""); + const auto size = json::optional_i64(item, "Size", -1); + if (size >= 0) info.size = static_cast(size); + listing.emplace(path, std::move(info)); + } + return listing; +} + +RemoteFileInfo modelscope_remote_info( + const Package & package, + const std::string & remote, + ListingCache & cache) { + const auto key = package.repo + "\n" + package.revision; + auto found = cache.find(key); + if (found == cache.end()) { + RemoteFileMap listing; + try { + listing = fetch_modelscope_listing(package); + } catch (...) { + listing.clear(); + } + found = cache.emplace(key, std::move(listing)).first; + } + const auto file = found->second.find(remote); + if (file != found->second.end()) return file->second; + if (!found->second.empty()) { + throw std::runtime_error("remote file is not accessible: " + package.repo + "/" + remote + + " (not in the ModelScope repo listing)"); + } + // The listing API is unreachable; fall back to a HEAD on the resolve URL, + // which reports the file checksum as X-Linked-Etag but no size. + const auto response = http_request(ms_url(package, remote), true, nullptr, {}, {}, RequestAuth::modelscope); + if (response.status < 200 || response.status >= 300) { + throw std::runtime_error("remote file is not accessible: " + package.repo + "/" + remote + + " (HTTP " + std::to_string(response.status) + ")"); + } + RemoteFileInfo result; + const auto etag = response.headers.find("x-linked-etag"); + if (etag != response.headers.end()) result.etag = trim_quotes(etag->second); + return result; +} + +RemoteFileInfo remote_info(const Package & package, const std::string & remote, ListingCache * ms_cache) { + if (package.download_kind == "modelscope_snapshot") { + if (ms_cache == nullptr) { + ListingCache local; + return modelscope_remote_info(package, remote, local); + } + return modelscope_remote_info(package, remote, *ms_cache); + } + const auto response = http_request(hf_url(package, remote), true, nullptr, {}, {}, RequestAuth::huggingface); if (response.status == 401 || response.status == 403) { if (package.gated) return {}; } @@ -415,11 +559,16 @@ void download_file( std::filesystem::create_directories(destination.parent_path()); std::ofstream output(destination, std::ios::binary | std::ios::trunc); if (!output) throw std::runtime_error("could not create " + destination.string()); - const auto response = http_request(hf_url(package, remote), false, &output, cancelled, progress); + const auto url = package.download_kind == "modelscope_snapshot" + ? ms_url(package, remote) + : hf_url(package, remote); + const auto response = http_request(url, false, &output, cancelled, progress, package_auth(package)); output.close(); if (response.status == 401 || response.status == 403) { throw std::runtime_error(package.repo + "/" + remote + - " requires accepted Hugging Face access and a valid HF token"); + (package.download_kind == "modelscope_snapshot" + ? " requires ModelScope access to this repo" + : " requires accepted Hugging Face access and a valid HF token")); } if (response.status < 200 || response.status >= 300) { throw std::runtime_error("failed to download " + package.repo + "/" + remote + @@ -513,11 +662,12 @@ std::string PackageManager::install( } std::map remote_files; + ListingCache ms_cache; uint64_t total = 0; bool known_total = true; for (const auto & [remote, output] : downloads) { (void) output; - auto info = remote_info(package, remote); + auto info = remote_info(package, remote, &ms_cache); if (!info.size) known_total = false; else total += *info.size; remote_files.emplace(remote, std::move(info)); } @@ -568,7 +718,7 @@ std::string PackageManager::install( // Include reused sidecars in the per-package manifest as well. for (const auto & [remote, output] : plan) { (void) output; - if (remote_files.count(remote) == 0) remote_files.emplace(remote, remote_info(package, remote)); + if (remote_files.count(remote) == 0) remote_files.emplace(remote, remote_info(package, remote, &ms_cache)); } json::Value::Object manifest; manifest["schema_version"] = number_value(1); @@ -659,6 +809,7 @@ std::string PackageManager::inventory(bool query_remote, const std::string & pac workers.reserve(worker_count); for (size_t worker = 0; worker < worker_count; ++worker) { workers.push_back(std::async(std::launch::async, [this, query_remote, &selected, &rows, &next] { + ListingCache ms_cache; for (;;) { const auto index = next.fetch_add(1); if (index >= selected.size()) return; @@ -683,7 +834,7 @@ std::string PackageManager::inventory(bool query_remote, const std::string & pac std::map remote; std::string revision; for (const auto & file : package.files) { - auto info = remote_info(package, file); + auto info = remote_info(package, file, &ms_cache); if (!info.size) size_known = false; else total += *info.size; if (revision.empty()) revision = info.revision; remote.emplace(file, std::move(info)); diff --git a/tests/fixtures/native_model_manager_server.py b/tests/fixtures/native_model_manager_server.py index be343f6ba..01e077d29 100644 --- a/tests/fixtures/native_model_manager_server.py +++ b/tests/fixtures/native_model_manager_server.py @@ -2,8 +2,11 @@ """Local HTTP fixture for server_model_installer_test; never used at runtime.""" import argparse +import hashlib +import json import threading import time +import urllib.parse from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer FILES = { @@ -13,6 +16,28 @@ "/org/repo/resolve/main/Slow/model.gguf": b"s" * (16 * 1024 * 1024), } +# ModelScope-style repo: default branch is master, and the file-list API is +# the source of truth for per-file size and sha256. +MS_REPO = "ms/repo" +MS_REVISION = "master" +MS_FILES = { + "Demo/model-q8.gguf": b"q8-model-payload", + "Demo/shared.json": b'{"shared":true}\n', +} + + +def ms_listing_payload(): + files = [ + { + "Path": path, + "Size": len(payload), + "Sha256": hashlib.sha256(payload).hexdigest(), + "Type": "blob", + } + for path, payload in sorted(MS_FILES.items()) + ] + return json.dumps({"Code": 200, "Success": True, "Data": {"Files": files}}).encode() + class Handler(BaseHTTPRequestHandler): def do_HEAD(self): @@ -22,6 +47,17 @@ def do_GET(self): self.respond(True) def respond(self, body): + parsed = urllib.parse.urlsplit(self.path) + if parsed.path.startswith("/api/v1/models/"): + if not self.check_ms_auth(): + return + self.respond_ms_listing(parsed, body) + return + if parsed.path.startswith("/models/"): + if not self.check_ms_auth(): + return + self.respond_ms_resolve(parsed.path, body) + return payload = FILES.get(self.path) if payload is None: self.send_error(404) @@ -31,6 +67,58 @@ def respond(self, body): self.send_header("ETag", '"fixture-etag-' + str(len(payload)) + '"') self.send_header("X-Repo-Commit", "fixture-commit") self.end_headers() + self.write_payload(payload, body) + + def check_ms_auth(self): + # ModelScope requests must never carry the Hugging Face credential; + # when --ms-token is given they must carry exactly that token. + auth = self.headers.get("Authorization", "") + hf_token = self.server.hf_token + ms_token = self.server.ms_token + if hf_token and auth == "Bearer " + hf_token: + self.send_error(403, "HF token leaked into a ModelScope request") + self.count_request() + return False + if ms_token and auth != "Bearer " + ms_token: + self.send_error(401, "ModelScope request missing its own token") + self.count_request() + return False + return True + + def respond_ms_listing(self, parsed, body): + prefix = "/api/v1/models/" + MS_REPO + "/repo/files" + query = urllib.parse.parse_qs(parsed.query) + revision = query.get("Revision", [""])[0] + if parsed.path != prefix or revision != MS_REVISION: + self.send_error(404) + return + payload = ms_listing_payload() + self.send_response(200) + self.send_header("Content-Length", str(len(payload))) + self.send_header("Content-Type", "application/json") + self.end_headers() + self.write_payload(payload, body) + + def respond_ms_resolve(self, path, body): + prefix = "/models/" + MS_REPO + "/resolve/" + MS_REVISION + "/" + if not path.startswith(prefix): + self.send_error(404) + return + payload = MS_FILES.get(path[len(prefix):]) + if payload is None: + self.send_error(404) + return + etag = hashlib.sha256(payload).hexdigest() + self.send_response(200) + # Like real ModelScope resolve responses: no Content-Length and no + # ETag on HEAD, only X-Linked-Etag. + self.send_header("X-Linked-Etag", etag) + if body: + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.write_payload(payload, body) + + def write_payload(self, payload, body): if body: try: for offset in range(0, len(payload), 64 * 1024): @@ -40,6 +128,9 @@ def respond(self, body): time.sleep(0.003) except (BrokenPipeError, ConnectionResetError): pass + self.count_request() + + def count_request(self): with self.server.remaining_lock: self.server.remaining -= 1 if self.server.remaining <= 0: @@ -51,10 +142,15 @@ def log_message(self, *_args): parser = argparse.ArgumentParser() parser.add_argument("--requests", type=int, default=14) +parser.add_argument("--port", type=int, default=18991) +parser.add_argument("--hf-token", default="") +parser.add_argument("--ms-token", default="") args = parser.parse_args() -server = ThreadingHTTPServer(("127.0.0.1", 18991), Handler) +server = ThreadingHTTPServer(("127.0.0.1", args.port), Handler) server.remaining = args.requests server.remaining_lock = threading.Lock() +server.hf_token = args.hf_token +server.ms_token = args.ms_token timeout = threading.Timer(30, server.shutdown) timeout.daemon = True timeout.start() diff --git a/tests/unittests/test_model_spec_system.cpp b/tests/unittests/test_model_spec_system.cpp index 6c9230e23..eb9c431d6 100644 --- a/tests/unittests/test_model_spec_system.cpp +++ b/tests/unittests/test_model_spec_system.cpp @@ -227,6 +227,32 @@ void expect_rejects(const std::string & label, const std::string & spec_text, co engine::test::require(rejected, label + " should reject with: " + needle); } +std::string spec_with_download(const std::string & download) { + auto text = schema_v1_spec_text("[]"); + const std::string anchor = "\"download\": {\"kind\": \"unsupported\", \"reason\": \"test fixture\"}"; + const auto at = text.find(anchor); + engine::test::require(at != std::string::npos, "download fixture anchor exists"); + text.replace(at, anchor.size(), "\"download\": " + download); + return text; +} + +void test_download_kinds_schema() { + // ModelScope snapshot downloads validate like Hugging Face snapshots: + // repo is required, revision is optional. + engine::model_spec::validate_spec( + json::parse(spec_with_download( + R"JSON({"kind": "modelscope_snapshot", "repo": "audio-cpp/toy-model"})JSON")), + "modelscope_snapshot_repo_only"); + engine::model_spec::validate_spec( + json::parse(spec_with_download( + R"JSON({"kind": "modelscope_snapshot", "repo": "audio-cpp/toy-model", "revision": "master"})JSON")), + "modelscope_snapshot_with_revision"); + expect_rejects( + "modelscope_snapshot_missing_repo", + spec_with_download(R"JSON({"kind": "modelscope_snapshot"})JSON"), + "missing required field 'repo'"); +} + void test_legacy_dependencies_schema() { // Valid legacy specs accept required model dependencies and conditional bundled dependencies. const auto spec = json::parse(schema_v1_spec_text(R"JSON([ @@ -1284,6 +1310,7 @@ void test_loading_and_resource_bundle() { int main() { try { test_legacy_dependencies_schema(); + test_download_kinds_schema(); test_typed_schema_renamed_dependencies(); test_dependency_option_mapping_from_production_spec(); test_options_schema(); diff --git a/tests/unittests/test_package_manager_modelscope.cpp b/tests/unittests/test_package_manager_modelscope.cpp new file mode 100644 index 000000000..81d4f8c70 --- /dev/null +++ b/tests/unittests/test_package_manager_modelscope.cpp @@ -0,0 +1,147 @@ +#include "engine/framework/package_manager/manager.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void require(bool condition, const std::string & message) { + if (!condition) throw std::runtime_error(message); +} + +std::filesystem::path make_root() { + const auto suffix = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()).count(); + const auto root = std::filesystem::temp_directory_path() / + ("audiocpp-modelscope-package-manager-test-" + std::to_string(suffix)); + std::filesystem::create_directories(root); + return root; +} + +void write(const std::filesystem::path & path, const std::string & value) { + std::filesystem::create_directories(path.parent_path()); + std::ofstream output(path, std::ios::binary | std::ios::trunc); + output << value; +} + +std::string read(const std::filesystem::path & path) { + std::ifstream input(path, std::ios::binary); + std::ostringstream buffer; + buffer << input.rdbuf(); + return buffer.str(); +} + +void set_base_url(const std::string & value) { +#ifdef _WIN32 + _putenv_s("AUDIOCPP_MS_BASE_URL", value.c_str()); +#else + setenv("AUDIOCPP_MS_BASE_URL", value.c_str(), 1); +#endif +} + +void set_env(const char * name, const std::string & value) { +#ifdef _WIN32 + _putenv_s(name, value.c_str()); +#else + setenv(name, value.c_str(), 1); +#endif +} + +void test_modelscope_package_lifecycle() { + const auto root = make_root(); + try { + // No explicit revision: modelscope_snapshot must default to master. + write(root / "model_specs" / "demo_ms.json", R"JSON({ + "family":"demo_ms","display_name":"Demo MS","description":"","category":"tts", + "status":"supported","tasks":["tts"],"modes":["offline"],"languages":[], + "capabilities":{},"runtime":{}, + "package_defaults":{"download":{"kind":"modelscope_snapshot","repo":"ms/repo"}}, + "packages":[ + {"id":"demo_ms_q8","display_name":"Demo MS Q8","default":true,"format":"gguf","precision":"q8_0", + "target_directory":"DemoMS","files":["Demo/model-q8.gguf","Demo/shared.json"],"strip_prefix":"Demo"} + ],"sources":[] + })JSON"); + set_base_url("http://127.0.0.1:18992"); + // The fixture rejects any ModelScope request carrying this HF token, + // and requires every ModelScope request to carry AUDIOCPP_MS_TOKEN. + set_env("HF_TOKEN", "hf-secret-fixture-token"); + set_env("AUDIOCPP_MS_TOKEN", "ms-secret-fixture-token"); + auto fixture = std::async(std::launch::async, [] { +#ifdef _WIN32 + return std::system("python \"" AUDIOCPP_NATIVE_MANAGER_FIXTURE "\" --port 18992 --requests 4" + " --hf-token hf-secret-fixture-token --ms-token ms-secret-fixture-token"); +#else + return std::system("python3 \"" AUDIOCPP_NATIVE_MANAGER_FIXTURE "\" --port 18992 --requests 4" + " --hf-token hf-secret-fixture-token --ms-token ms-secret-fixture-token"); +#endif + }); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + + engine::package_manager::PackageManager manager(root, root / "models"); + uint64_t last_total = 0; + const auto installed = manager.install("demo_ms_q8", false, nullptr, + [&](const engine::package_manager::PackageProgress & progress) { + last_total = progress.total_bytes; + }); + require(installed.find("Installed demo_ms_q8") != std::string::npos, + "modelscope package installs through the fixture"); + require(last_total == 16 + 16, "listing sizes drive the download total"); + require(std::filesystem::is_regular_file(root / "models" / "DemoMS" / "model-q8.gguf"), + "model payload is installed"); + require(std::filesystem::is_regular_file(root / "models" / "DemoMS" / "shared.json"), + "sidecar is installed"); + + const auto manifest_path = + root / "models" / "DemoMS" / ".audiocpp-package-demo_ms_q8.json"; + require(std::filesystem::is_regular_file(manifest_path), + "native install writes a version manifest"); + const auto manifest = read(manifest_path); + require(manifest.find("\"requested_revision\":\"master\"") != std::string::npos, + "modelscope packages default to the master revision"); + require(manifest.find("\"resolved_revision\":\"master\"") != std::string::npos, + "resolved revision falls back to the requested revision"); + require(manifest.find("\"etag\":\"1a53a6a47a59980589f5c699aa3da20ee9502d67224708a2bf5113852e1e1fa6\"") + != std::string::npos, + "manifest records the ModelScope sha256 as the etag"); + + const auto again = manager.install("demo_ms_q8", false, nullptr, nullptr); + require(again.find("Already installed demo_ms_q8") != std::string::npos, + "re-install without overwrite is a no-op"); + + const auto inventory = manager.inventory(true); + require(inventory.find("\"id\":\"demo_ms_q8\"") != std::string::npos, + "inventory includes the modelscope package"); + require(inventory.find("\"version_state\":\"up_to_date\"") != std::string::npos, + "sha256 etag version check reports up to date"); + require(inventory.find("\"size_bytes\":32") != std::string::npos, + "inventory reports the listing size total"); + + require(fixture.get() == 0, "local HTTP fixture completed normally"); + } catch (...) { + std::error_code error; + std::filesystem::remove_all(root, error); + throw; + } + std::error_code error; + std::filesystem::remove_all(root, error); +} + +} // namespace + +int main() { + try { test_modelscope_package_lifecycle(); } + catch (const std::exception & error) { + std::cerr << error.what() << '\n'; + return 1; + } + std::cout << "package_manager_modelscope_test passed\n"; + return 0; +} diff --git a/tools/model_manager_v2.py b/tools/model_manager_v2.py index d6bde8d1a..5b5811a56 100644 --- a/tools/model_manager_v2.py +++ b/tools/model_manager_v2.py @@ -8,8 +8,9 @@ import shutil import sys import tempfile +import threading import time -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from typing import Any from urllib.error import HTTPError @@ -41,6 +42,7 @@ class PackageRecord: strip_prefix: str download: dict[str, Any] default: bool + source_overridden: bool = False @dataclass(frozen=True) @@ -73,9 +75,28 @@ def hf_endpoint() -> str: return endpoint or "https://huggingface.co" -def http_headers() -> dict[str, str]: +def ms_endpoint() -> str: + """Base URL for ModelScope requests. + + Honors AUDIOCPP_MS_BASE_URL, mirroring the native C++ package manager. + Falls back to https://www.modelscope.cn. Empty values and trailing slashes + are tolerated. + """ + endpoint = os.environ.get("AUDIOCPP_MS_BASE_URL", "").strip().rstrip("/") + return endpoint or "https://www.modelscope.cn" + + +def modelscope_token() -> str | None: + token = os.environ.get("AUDIOCPP_MS_TOKEN", "").strip() + return token or None + + +def http_headers(source: str = "huggingface") -> dict[str, str]: + # Auth is provider-scoped: HF requests may carry the HF token, ModelScope + # requests carry only AUDIOCPP_MS_TOKEN, so one provider's credential is + # never sent to the other provider's host. headers = {"User-Agent": "audio.cpp model_manager_v2.py"} - token = huggingface_token() + token = modelscope_token() if source == "modelscope" else huggingface_token() if token: headers["Authorization"] = f"Bearer {token}" return headers @@ -123,6 +144,47 @@ def merged_download(spec: dict[str, Any], package: dict[str, Any]) -> dict[str, return download +def package_kind(package: PackageRecord) -> str: + return str(package.download.get("kind", "")) + + +def package_source(package: PackageRecord) -> str: + """Download provider for a package: 'modelscope' or 'huggingface'.""" + return "modelscope" if package_kind(package) == "modelscope_snapshot" else "huggingface" + + +def package_revision(package: PackageRecord) -> str: + """Kind-aware revision default: master on ModelScope, main on Hugging Face.""" + revision = str(package.download.get("revision", "")).strip() + if revision: + return revision + return "master" if package_kind(package) == "modelscope_snapshot" else "main" + + +def apply_source_override(record: PackageRecord, source_repo: str | None) -> PackageRecord: + """Redirect a package's download to ModelScope (--source modelscope). + + The repo comes from --source-repo when given, otherwise the spec's own + repo is reused on ModelScope. An unset or 'main' spec revision becomes + ModelScope's default branch ('master', via the kind-aware default); any + other explicit revision passes through unchanged. + """ + download = dict(record.download) + download["kind"] = "modelscope_snapshot" + if source_repo: + download["repo"] = source_repo + revision = str(download.get("revision", "")).strip() + if not revision or revision == "main": + download.pop("revision", None) + return replace(record, download=download, source_overridden=True) + + +def source_override_hint(package: PackageRecord) -> str: + if package.source_overridden: + return "; the spec repo may not exist on ModelScope, name it with --source-repo " + return "" + + def flatten_packages(specs: list[dict[str, Any]]) -> list[PackageRecord]: records: list[PackageRecord] = [] for spec in specs: @@ -177,9 +239,99 @@ def hf_url(repo: str, revision: str, remote_path: str) -> str: return f"{hf_endpoint()}/{repo}/resolve/{quote(revision, safe='')}/{quote_repo_path(remote_path)}" +def ms_url(repo: str, revision: str, remote_path: str) -> str: + return f"{ms_endpoint()}/models/{repo}/resolve/{quote(revision, safe='')}/{quote_repo_path(remote_path)}" + + +def ms_files_url(repo: str, revision: str) -> str: + return f"{ms_endpoint()}/api/v1/models/{repo}/repo/files?Revision={quote(revision, safe='')}&Recursive=true" + + +def download_url(package: PackageRecord, remote_path: str) -> str: + repo = package.download["repo"] + revision = package_revision(package) + if package_kind(package) == "modelscope_snapshot": + return ms_url(repo, revision, remote_path) + return hf_url(repo, revision, remote_path) + + +# ModelScope resolve HEAD responses carry no Content-Length or ETag, so +# per-file size and checksum come from the repo file-list API instead. The +# listing is fetched once per repo+revision and shared through this cache; an +# empty entry means the listing was unavailable and HEAD fallback applies. +_MS_LISTING_CACHE: dict[tuple[str, str, str], dict[str, RemoteFileInfo]] = {} +_MS_LISTING_LOCK = threading.Lock() + + +def ms_repo_listing(repo: str, revision: str) -> dict[str, RemoteFileInfo]: + key = (ms_endpoint(), repo, revision) + with _MS_LISTING_LOCK: + cached = _MS_LISTING_CACHE.get(key) + if cached is not None: + return cached + listing: dict[str, RemoteFileInfo] = {} + try: + request = Request(ms_files_url(repo, revision), headers=http_headers("modelscope")) + with urlopen(request, timeout=60) as response: + payload = json.loads(response.read().decode("utf-8")) + if payload.get("Code") != 200: + raise ManagerError(f"ModelScope repo listing failed for {repo}: {payload.get('Message', 'unknown error')}") + files = payload.get("Data", {}).get("Files") + if not isinstance(files, list): + raise ManagerError(f"ModelScope repo listing has no file list: {repo}") + for item in files: + if not isinstance(item, dict) or item.get("Type") != "blob": + continue + path = item.get("Path") + if not path: + continue + size = item.get("Size") + listing[str(path)] = RemoteFileInfo( + size=int(size) if size is not None else None, + revision="", + etag=str(item.get("Sha256") or ""), + ) + except Exception: + listing = {} + with _MS_LISTING_LOCK: + _MS_LISTING_CACHE[key] = listing + return listing + + +def check_ms_remote_file(package: PackageRecord, remote_path: str) -> RemoteFileInfo: + repo = package.download["repo"] + revision = package_revision(package) + listing = ms_repo_listing(repo, revision) + info = listing.get(remote_path) + if info is not None: + return info + if listing: + raise ManagerError( + f"remote file is not accessible: {repo}/{remote_path} (not in the ModelScope repo listing)" + f"{source_override_hint(package)}" + ) + # The listing API is unreachable; fall back to a HEAD on the resolve URL, + # which reports the file checksum as X-Linked-Etag but no size. + request = Request(ms_url(repo, revision, remote_path), headers=http_headers("modelscope"), method="HEAD") + try: + with urlopen(request, timeout=60) as response: + return RemoteFileInfo( + size=None, + revision="", + etag=response.headers.get("X-Linked-Etag", "").strip('"'), + ) + except HTTPError as error: + raise ManagerError( + f"remote file is not accessible: {repo}/{remote_path} ({error.code})" + f"{source_override_hint(package)}" + ) from error + + def check_remote_file(package: PackageRecord, remote_path: str) -> RemoteFileInfo: + if package_kind(package) == "modelscope_snapshot": + return check_ms_remote_file(package, remote_path) repo = package.download["repo"] - revision = package.download.get("revision", "main") + revision = package_revision(package) request = Request(hf_url(repo, revision, remote_path), headers=http_headers(), method="HEAD") try: with urlopen(request, timeout=60) as response: @@ -207,8 +359,7 @@ def download_file( cancel_file: Path | None = None, ) -> None: repo = package.download["repo"] - revision = package.download.get("revision", "main") - request = Request(hf_url(repo, revision, remote_path), headers=http_headers()) + request = Request(download_url(package, remote_path), headers=http_headers(package_source(package))) try: with urlopen(request, timeout=300) as response: expected_header = response.headers.get("Content-Length") @@ -239,18 +390,23 @@ def download_file( raise ManagerError( f"{repo}/{remote_path} requires accepted Hugging Face access and a valid HF token" ) from error - raise ManagerError(f"failed to download {repo}/{remote_path}: HTTP {error.code}") from error + if package_kind(package) == "modelscope_snapshot" and error.code in (401, 403): + raise ManagerError(f"{repo}/{remote_path} requires ModelScope access to this repo") from error + raise ManagerError( + f"failed to download {repo}/{remote_path}: HTTP {error.code}{source_override_hint(package)}" + ) from error -def ensure_hf_package(package: PackageRecord) -> None: +def ensure_snapshot_package(package: PackageRecord) -> None: kind = package.download.get("kind") - if kind != "huggingface_snapshot": + if kind not in ("huggingface_snapshot", "modelscope_snapshot"): raise ManagerError( - f"{package.id} uses download kind '{kind}'. model_manager_v2 only installs huggingface_snapshot packages; " + f"{package.id} uses download kind '{kind}'. model_manager_v2 only installs " + "huggingface_snapshot and modelscope_snapshot packages; " "use tools/model_manager.py for legacy composite or converter installs." ) if not package.download.get("repo"): - raise ManagerError(f"{package.id} has no Hugging Face repo") + raise ManagerError(f"{package.id} has no remote repo for download kind '{kind}'") def package_manifest_path(package: PackageRecord, models_root: Path) -> Path: @@ -281,7 +437,7 @@ def write_package_manifest( "schema_version": 1, "package_id": package.id, "repo": package.download.get("repo", ""), - "requested_revision": package.download.get("revision", "main"), + "requested_revision": package_revision(package), "resolved_revision": resolved_revision, "installed_at_unix": int(time.time()), "files": { @@ -321,7 +477,7 @@ def reusable_package_outputs( def install_package(package: PackageRecord, records: list[PackageRecord], args: argparse.Namespace) -> None: - ensure_hf_package(package) + ensure_snapshot_package(package) target_dir = validate_relative_path(package.target_directory, "target_directory") models_root = Path(args.models_root) final_dir = models_root / target_dir @@ -329,7 +485,7 @@ def install_package(package: PackageRecord, records: list[PackageRecord], args: full_plan = list(plan) print(f"selected {package.id} ({package.family})") - print(f"repo {package.download['repo']}@{package.download.get('revision', 'main')}") + print(f"repo {package.download['repo']}@{package_revision(package)}") print(f"target {final_dir}") for remote, output in plan: if args.check: @@ -421,7 +577,7 @@ def emit_progress(downloaded: int) -> None: shutil.rmtree(staging) resolved_revision = next( (info.revision for info in remote_files.values() if info.revision), - package.download.get("revision", "main"), + package_revision(package), ) write_package_manifest(package, models_root, resolved_revision, remote_files) except Exception: @@ -578,7 +734,7 @@ def package_version_state( def package_size_record(package: PackageRecord, models_root: Path | None = None) -> dict[str, Any]: installed = package_is_installed(package, models_root) try: - ensure_hf_package(package) + ensure_snapshot_package(package) total = 0 unknown = False remote_revision = "" @@ -651,6 +807,21 @@ def command_installed(records: list[PackageRecord], args: argparse.Namespace) -> print(json.dumps(rows, ensure_ascii=False)) +def add_source_arguments(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--source", + choices=["huggingface", "modelscope"], + default="huggingface", + help="download source override; 'modelscope' downloads from ModelScope even when the " + "spec says huggingface_snapshot (an unset or 'main' revision becomes 'master'). " + "Manifest etags are source-specific, so cross-source checks may report updates", + ) + parser.add_argument( + "--source-repo", + help="ModelScope repo (namespace/name) for --source modelscope; defaults to the spec's repo", + ) + + def make_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description="Install audio.cpp model packages from model_specs/*.json.") parser.add_argument("--specs-dir", default=str(DEFAULT_SPECS_DIR), help="directory containing model spec JSON files") @@ -670,6 +841,7 @@ def make_parser() -> argparse.ArgumentParser: sizes_parser.add_argument("--jobs", type=int, default=12, help="parallel metadata checks") sizes_parser.add_argument("--models-root", help="also report packages whose required files are installed") sizes_parser.add_argument("--json", action="store_true", help="retained for command symmetry; output is JSON") + add_source_arguments(sizes_parser) installed_parser = sub.add_parser("installed", help="report locally installed packages without network access") installed_parser.add_argument("--models-root", default="models") @@ -685,7 +857,7 @@ def make_parser() -> argparse.ArgumentParser: clean_parser.add_argument("package", help="package id or family") clean_parser.add_argument("--models-root", default="models") - install_parser = sub.add_parser("install", help="install one Hugging Face snapshot package") + install_parser = sub.add_parser("install", help="install one Hugging Face or ModelScope snapshot package") install_parser.add_argument("package", help="package id or family") install_parser.add_argument("--format") install_parser.add_argument("--precision") @@ -699,14 +871,21 @@ def make_parser() -> argparse.ArgumentParser: action="store_true", help="emit machine-readable AUDIOCPP_PROGRESS lines while downloading", ) + add_source_arguments(install_parser) return parser def main() -> int: parser = make_parser() args = parser.parse_args() + source = getattr(args, "source", "huggingface") + source_repo = getattr(args, "source_repo", None) + if source_repo and source != "modelscope": + parser.error("--source-repo requires --source modelscope") try: records = flatten_packages(load_specs(Path(args.specs_dir))) + if source == "modelscope": + records = [apply_source_override(record, source_repo) for record in records] if args.command == "list": command_list(records, args) elif args.command == "info":