diff --git a/src/pull_module/hf_pull_model_module.cpp b/src/pull_module/hf_pull_model_module.cpp index fd7e5fb0c8..9479a21494 100644 --- a/src/pull_module/hf_pull_model_module.cpp +++ b/src/pull_module/hf_pull_model_module.cpp @@ -50,9 +50,9 @@ static std::string getEnvReturnOrDefaultIfNotSet(const std::string& envName, con const char* envValue = std::getenv(envName.c_str()); if (envValue) { value = std::string(envValue); - SPDLOG_DEBUG("{} environment variable set. Using value: {};", envName, value); + SPDLOG_DEBUG("{} environment variable set.", envName); } else { - SPDLOG_DEBUG("{} environment variable not set. Using default value: {};", envName, defaultValue); + SPDLOG_DEBUG("{} environment variable not set. Using default value.", envName); } return value; } @@ -244,7 +244,7 @@ Status HfPullModelModule::clone() { return std::get(guardOrError); } - downloader = std::make_unique(this->hfSettings.sourceModel, IModelDownloader::getGraphDirectory(this->hfSettings.downloadPath, this->hfSettings.sourceModel), this->GetHfEndpoint(), this->GetHfToken(), this->GetProxy(), this->hfSettings.overwriteModels); + downloader = std::make_unique(this->hfSettings.sourceModel, IModelDownloader::getGraphDirectory(this->hfSettings.downloadPath, this->hfSettings.sourceModel), this->GetHfEndpoint(), this->GetProxy(), this->hfSettings.overwriteModels); } else if (this->hfSettings.downloadType == OPTIMUM_CLI_DOWNLOAD) { downloader = std::make_unique(this->hfSettings.exportSettings, this->hfSettings.task, this->hfSettings.sourceModel, IModelDownloader::getGraphDirectory(this->hfSettings.downloadPath, this->hfSettings.sourceModel), this->hfSettings.overwriteModels); } else if (this->hfSettings.downloadType == GGUF_DOWNLOAD) { @@ -265,7 +265,7 @@ Status HfPullModelModule::clone() { if (std::holds_alternative(this->hfSettings.graphSettings) && std::get(this->hfSettings.graphSettings).draftModelDirName.has_value()) { auto& graphSettings = std::get(this->hfSettings.graphSettings); std::unique_ptr draftModelDownloader; - draftModelDownloader = std::make_unique(graphSettings.draftModelDirName.value(), GraphExport::getDraftModelDirectoryPath(graphDirectory, graphSettings.draftModelDirName.value()), this->GetHfEndpoint(), this->GetHfToken(), this->GetProxy(), this->hfSettings.overwriteModels); + draftModelDownloader = std::make_unique(graphSettings.draftModelDirName.value(), GraphExport::getDraftModelDirectoryPath(graphDirectory, graphSettings.draftModelDirName.value()), this->GetHfEndpoint(), this->GetProxy(), this->hfSettings.overwriteModels); status = draftModelDownloader->downloadModel(); if (!status.ok()) { return status; diff --git a/src/pull_module/libgit2.cpp b/src/pull_module/libgit2.cpp index 75c25c90d9..46f5843b67 100644 --- a/src/pull_module/libgit2.cpp +++ b/src/pull_module/libgit2.cpp @@ -342,48 +342,21 @@ Libgt2InitGuard::~Libgt2InitGuard() { git_libgit2_shutdown(); } -const std::string PROTOCOL_SEPARATOR = "://"; - bool HfDownloader::CheckIfProxySet() { if (this->httpProxy != "") return true; return false; } -std::string HfDownloader::GetRepositoryUrlWithPassword() { - std::string repoPass = ""; - if (this->hfToken != "") { - repoPass += this->hfToken + ":" + this->hfToken + "@"; - } else { - SPDLOG_DEBUG("HF_TOKEN environment variable not set"); - return this->hfEndpoint + this->sourceModel; - } - - std::string outputWithPass = ""; - size_t match = this->hfEndpoint.find(PROTOCOL_SEPARATOR); - if (match != std::string::npos) { - // https://huggingface.co - // protocol[match]//address - std::string protocol = this->hfEndpoint.substr(0, match); - std::string address = this->hfEndpoint.substr(match + PROTOCOL_SEPARATOR.size()); - outputWithPass = protocol + PROTOCOL_SEPARATOR + repoPass + address + this->sourceModel; - } else { - outputWithPass = repoPass + this->hfEndpoint + this->sourceModel; - } - - return outputWithPass; -} - std::string HfDownloader::GetRepoUrl() { std::string repoUrl = ""; repoUrl += this->hfEndpoint + this->sourceModel; return repoUrl; } -HfDownloader::HfDownloader(const std::string& inSourceModel, const std::string& inDownloadPath, const std::string& inHfEndpoint, const std::string& inHfToken, const std::string& inHttpProxy, bool inOverwrite) : +HfDownloader::HfDownloader(const std::string& inSourceModel, const std::string& inDownloadPath, const std::string& inHfEndpoint, const std::string& inHttpProxy, bool inOverwrite) : IModelDownloader(inSourceModel, inDownloadPath, inOverwrite), hfEndpoint(inHfEndpoint), - hfToken(inHfToken), httpProxy(inHttpProxy) {} Status HfDownloader::RemoveReadonlyFileAttributeFromDir(const std::string& directoryPath) { @@ -1433,14 +1406,14 @@ void configureCloneOptions(git_clone_options& cloneOptions, bool useProxy, const * Executes git clone for a model repository and handles cancellation/error mapping. * * @param downloadPath Destination repository path on local filesystem. - * @param passRepoUrl Source repository URL (possibly with embedded credentials). + * @param repoUrl Source repository URL without embedded credentials. * @param cloneOptions Prepared libgit2 clone options. * @return StatusCode::OK on success, cancellation or clone failure status otherwise. * @note Connects to remote git endpoint and writes repository data to local filesystem. */ -Status executeClone(const std::string& downloadPath, const std::string& passRepoUrl, git_clone_options& cloneOptions) { +Status executeClone(const std::string& downloadPath, const std::string& repoUrl, git_clone_options& cloneOptions) { git_repository* clonedRepo = nullptr; - const char* url = passRepoUrl.c_str(); + const char* url = repoUrl.c_str(); const char* path = downloadPath.c_str(); SPDLOG_TRACE("Starting git clone to: {}", path); if (!libgit2::createLfsWipMarker(downloadPath)) { @@ -1498,7 +1471,6 @@ Status finalizeAfterClone(const std::string& downloadPath, Status handleFreshClone(const std::string& downloadPath, const std::string& repoUrl, - const std::string& passRepoUrl, bool useProxy, const std::string& proxyUrl, const std::function& checkRepositoryStatusFn, @@ -1513,7 +1485,7 @@ Status handleFreshClone(const std::string& downloadPath, SPDLOG_DEBUG("Downloading from url: {}", repoUrl.c_str()); - auto status = executeClone(downloadPath, passRepoUrl, cloneOptions); + auto status = executeClone(downloadPath, repoUrl, cloneOptions); if (!status.ok()) { return status; } @@ -1560,9 +1532,8 @@ Status HfDownloader::downloadModel() { const bool useProxy = CheckIfProxySet(); std::string repoUrl = GetRepoUrl(); - std::string passRepoUrl = GetRepositoryUrlWithPassword(); - return handleFreshClone(this->downloadPath, repoUrl, passRepoUrl, useProxy, this->httpProxy, checkRepositoryStatusFn, removeReadonlyFn); + return handleFreshClone(this->downloadPath, repoUrl, useProxy, this->httpProxy, checkRepositoryStatusFn, removeReadonlyFn); } } // namespace ovms diff --git a/src/pull_module/libgit2.hpp b/src/pull_module/libgit2.hpp index d977d70b0d..3f157fec08 100644 --- a/src/pull_module/libgit2.hpp +++ b/src/pull_module/libgit2.hpp @@ -54,16 +54,14 @@ struct Libgt2InitGuard { class HfDownloader : public IModelDownloader { public: - HfDownloader(const std::string& sourceModel, const std::string& downloadPath, const std::string& hfEndpoint, const std::string& hfToken, const std::string& httpProxy, bool inOverwrite); + HfDownloader(const std::string& sourceModel, const std::string& downloadPath, const std::string& hfEndpoint, const std::string& httpProxy, bool inOverwrite); Status downloadModel() override; protected: const std::string hfEndpoint; - const std::string hfToken; const std::string httpProxy; std::string GetRepoUrl(); - std::string GetRepositoryUrlWithPassword(); bool CheckIfProxySet(); Status RemoveReadonlyFileAttributeFromDir(const std::string& directoryPath); Status CheckRepositoryStatus(bool checkUntracked); diff --git a/src/test/pull_hf_model_test.cpp b/src/test/pull_hf_model_test.cpp index 5995159f78..3a0c7e96b6 100644 --- a/src/test/pull_hf_model_test.cpp +++ b/src/test/pull_hf_model_test.cpp @@ -939,10 +939,9 @@ std::string sha256File(std::string_view path, std::error_code& ec) { class TestHfDownloader : public ovms::HfDownloader { public: - TestHfDownloader(const std::string& sourceModel, const std::string& downloadPath, const std::string& hfEndpoint, const std::string& hfToken, const std::string& httpProxy, bool overwrite) : - HfDownloader(sourceModel, downloadPath, hfEndpoint, hfToken, httpProxy, overwrite) {} + TestHfDownloader(const std::string& sourceModel, const std::string& downloadPath, const std::string& hfEndpoint, const std::string& /*hfToken*/, const std::string& httpProxy, bool overwrite) : + HfDownloader(sourceModel, downloadPath, hfEndpoint, httpProxy, overwrite) {} std::string GetRepoUrl() { return HfDownloader::GetRepoUrl(); } - std::string GetRepositoryUrlWithPassword() { return HfDownloader::GetRepositoryUrlWithPassword(); } bool CheckIfProxySet() { return HfDownloader::CheckIfProxySet(); } const std::string& getEndpoint() { return this->hfEndpoint; } const std::string& getProxy() { return this->httpProxy; } @@ -1832,7 +1831,6 @@ TEST(HfDownloaderClassTest, Methods) { EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).CheckIfProxySet(), false); ASSERT_EQ(hfDownloader->getEndpoint(), "www.new_hf.com/"); ASSERT_EQ(hfDownloader->GetRepoUrl(), "www.new_hf.com/model/name"); - ASSERT_EQ(hfDownloader->GetRepositoryUrlWithPassword(), "123$$o_O123!AAbb:123$$o_O123!AAbb@www.new_hf.com/model/name"); std::string expectedPath = downloadPath + "/" + modelName; #ifdef _WIN32 @@ -2117,29 +2115,29 @@ TEST_F(TestOptimumDownloaderSetup, PositiveOptimumExportCommandPassed) { ASSERT_EQ(optimumDownloader->downloadModel(), ovms::StatusCode::OK); } -TEST(HfDownloaderClassTest, ProtocollsWithPassword) { +TEST(HfDownloaderClassTest, ProtocolsWithoutPassword) { std::string modelName = "model/name"; std::string downloadPath = "/path/to/Download"; std::string hfEndpoint = "www.new_hf.com/"; std::string hfToken = ""; - EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "www.new_hf.com/model/name"); + EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepoUrl(), "www.new_hf.com/model/name"); hfEndpoint = "https://www.new_hf.com/"; - EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "https://www.new_hf.com/model/name"); + EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepoUrl(), "https://www.new_hf.com/model/name"); hfEndpoint = "www.new_hf.com/"; hfToken = "123!$token"; - EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "123!$token:123!$token@www.new_hf.com/model/name"); + EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepoUrl(), "www.new_hf.com/model/name"); hfEndpoint = "http://www.new_hf.com/"; hfToken = "123!$token"; - EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "http://123!$token:123!$token@www.new_hf.com/model/name"); + EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepoUrl(), "http://www.new_hf.com/model/name"); hfEndpoint = "git://www.new_hf.com/"; hfToken = "123!$token"; - EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "git://123!$token:123!$token@www.new_hf.com/model/name"); + EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepoUrl(), "git://www.new_hf.com/model/name"); hfEndpoint = "ssh://www.new_hf.com/"; hfToken = "123!$token"; - EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "ssh://123!$token:123!$token@www.new_hf.com/model/name"); + EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepoUrl(), "ssh://www.new_hf.com/model/name"); hfEndpoint = "what_ever_is_here://www.new_hf.com/"; hfToken = "123!$token"; - EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "what_ever_is_here://123!$token:123!$token@www.new_hf.com/model/name"); + EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepoUrl(), "what_ever_is_here://www.new_hf.com/model/name"); } TEST_F(HfPull, MethodsNegative) { @@ -2321,9 +2319,9 @@ TEST(Libgit2Framework, TimeoutTestProxy) { int e = git_libgit2_opts(GIT_OPT_SET_SERVER_CONNECT_TIMEOUT, 1000); EXPECT_EQ(e, 0); - std::string passRepoUrl = "https://huggingface.co/OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; + std::string repoUrl = "https://huggingface.co/OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov"; const char* path = "/tmp/model"; - int error = git_clone(&cloned_repo, passRepoUrl.c_str(), path, &clone_opts); + int error = git_clone(&cloned_repo, repoUrl.c_str(), path, &clone_opts); if (error != 0) { const git_error* err = git_error_last(); if (err) { diff --git a/third_party/libgit2/lfs.patch b/third_party/libgit2/lfs.patch index 887d700201..fd1408c0e6 100644 --- a/third_party/libgit2/lfs.patch +++ b/third_party/libgit2/lfs.patch @@ -427,7 +427,7 @@ new file mode 100644 index 000000000..18490e5ad --- /dev/null +++ b/src/libgit2/lfs_filter.c -@@ -0,0 +1,2014 @@ +@@ -0,0 +1,2029 @@ +/* +/ Copyright 2025 Intel Corporation +/ @@ -1775,6 +1775,20 @@ index 000000000..18490e5ad + status = setopt; \ + } + ++static CURLcode configure_hf_token_auth(CURL *curl) ++{ ++ const char *hf_token = getenv("HF_TOKEN"); ++ CURLcode status = CURLE_OK; ++ ++ if (!curl || !hf_token || !*hf_token) ++ return CURLE_OK; ++ ++ CURL_SETOPT(curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC)); ++ CURL_SETOPT(curl_easy_setopt(curl, CURLOPT_USERNAME, hf_token)); ++ CURL_SETOPT(curl_easy_setopt(curl, CURLOPT_PASSWORD, hf_token)); ++ return status; ++} ++ +/* + * print_curl_error_details + * ------------------------- @@ -2121,6 +2135,7 @@ index 000000000..18490e5ad + CURL_SETOPT(curl_easy_setopt( + info_curl, CURLOPT_ERRORBUFFER, info_error_buffer)); + CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_URL, lfs_info_url.ptr)); ++ CURL_SETOPT(configure_hf_token_auth(info_curl)); + /* Add cURL resiliency */ + /* unlimited data */ + CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_CONNECTTIMEOUT, 30L));