Move HF_TOKEN to libgit2 lib

src/pull_module/hf_pull_model_module.cpp

@@ -50,9 +50,9 @@ static std::string getEnvReturnOrDefaultIfNotSet(const std::string& envName, con
     const char* envValue = std::getenv(envName.c_str());
     if (envValue) {
         value = std::string(envValue);
-        SPDLOG_DEBUG("{} environment variable set. Using value: {};", envName, value);
+        SPDLOG_DEBUG("{} environment variable set.", envName);
     } else {
-        SPDLOG_DEBUG("{} environment variable not set. Using default value: {};", envName, defaultValue);
+        SPDLOG_DEBUG("{} environment variable not set. Using default value.", envName);
     }
     return value;
 }
@@ -244,7 +244,7 @@ Status HfPullModelModule::clone() {
             return std::get<Status>(guardOrError);
         }
 
-        downloader = std::make_unique<HfDownloader>(this->hfSettings.sourceModel, IModelDownloader::getGraphDirectory(this->hfSettings.downloadPath, this->hfSettings.sourceModel), this->GetHfEndpoint(), this->GetHfToken(), this->GetProxy(), this->hfSettings.overwriteModels);
+        downloader = std::make_unique<HfDownloader>(this->hfSettings.sourceModel, IModelDownloader::getGraphDirectory(this->hfSettings.downloadPath, this->hfSettings.sourceModel), this->GetHfEndpoint(), this->GetProxy(), this->hfSettings.overwriteModels);
     } else if (this->hfSettings.downloadType == OPTIMUM_CLI_DOWNLOAD) {
         downloader = std::make_unique<OptimumDownloader>(this->hfSettings.exportSettings, this->hfSettings.task, this->hfSettings.sourceModel, IModelDownloader::getGraphDirectory(this->hfSettings.downloadPath, this->hfSettings.sourceModel), this->hfSettings.overwriteModels);
     } else if (this->hfSettings.downloadType == GGUF_DOWNLOAD) {
@@ -265,7 +265,7 @@ Status HfPullModelModule::clone() {
     if (std::holds_alternative<TextGenGraphSettingsImpl>(this->hfSettings.graphSettings) && std::get<TextGenGraphSettingsImpl>(this->hfSettings.graphSettings).draftModelDirName.has_value()) {
         auto& graphSettings = std::get<TextGenGraphSettingsImpl>(this->hfSettings.graphSettings);
         std::unique_ptr<IModelDownloader> draftModelDownloader;
-        draftModelDownloader = std::make_unique<HfDownloader>(graphSettings.draftModelDirName.value(), GraphExport::getDraftModelDirectoryPath(graphDirectory, graphSettings.draftModelDirName.value()), this->GetHfEndpoint(), this->GetHfToken(), this->GetProxy(), this->hfSettings.overwriteModels);
+        draftModelDownloader = std::make_unique<HfDownloader>(graphSettings.draftModelDirName.value(), GraphExport::getDraftModelDirectoryPath(graphDirectory, graphSettings.draftModelDirName.value()), this->GetHfEndpoint(), this->GetProxy(), this->hfSettings.overwriteModels);
         status = draftModelDownloader->downloadModel();
         if (!status.ok()) {
             return status;

src/pull_module/libgit2.cpp

@@ -342,48 +342,21 @@ Libgt2InitGuard::~Libgt2InitGuard() {
     git_libgit2_shutdown();
 }
 
-const std::string PROTOCOL_SEPARATOR = "://";
-
 bool HfDownloader::CheckIfProxySet() {
     if (this->httpProxy != "")
         return true;
     return false;
 }
 
-std::string HfDownloader::GetRepositoryUrlWithPassword() {
-    std::string repoPass = "";
-    if (this->hfToken != "") {
-        repoPass += this->hfToken + ":" + this->hfToken + "@";
-    } else {
-        SPDLOG_DEBUG("HF_TOKEN environment variable not set");
-        return this->hfEndpoint + this->sourceModel;
-    }
-
-    std::string outputWithPass = "";
-    size_t match = this->hfEndpoint.find(PROTOCOL_SEPARATOR);
-    if (match != std::string::npos) {
-        // https://huggingface.co
-        // protocol[match]//address
-        std::string protocol = this->hfEndpoint.substr(0, match);
-        std::string address = this->hfEndpoint.substr(match + PROTOCOL_SEPARATOR.size());
-        outputWithPass = protocol + PROTOCOL_SEPARATOR + repoPass + address + this->sourceModel;
-    } else {
-        outputWithPass = repoPass + this->hfEndpoint + this->sourceModel;
-    }
-
-    return outputWithPass;
-}
-
 std::string HfDownloader::GetRepoUrl() {
     std::string repoUrl = "";
     repoUrl += this->hfEndpoint + this->sourceModel;
     return repoUrl;
 }
 
-HfDownloader::HfDownloader(const std::string& inSourceModel, const std::string& inDownloadPath, const std::string& inHfEndpoint, const std::string& inHfToken, const std::string& inHttpProxy, bool inOverwrite) :
+HfDownloader::HfDownloader(const std::string& inSourceModel, const std::string& inDownloadPath, const std::string& inHfEndpoint, const std::string& inHttpProxy, bool inOverwrite) :
     IModelDownloader(inSourceModel, inDownloadPath, inOverwrite),
     hfEndpoint(inHfEndpoint),
-    hfToken(inHfToken),
     httpProxy(inHttpProxy) {}
 
 Status HfDownloader::RemoveReadonlyFileAttributeFromDir(const std::string& directoryPath) {
@@ -1433,14 +1406,14 @@ void configureCloneOptions(git_clone_options& cloneOptions, bool useProxy, const
  * Executes git clone for a model repository and handles cancellation/error mapping.
  * 
  * @param downloadPath Destination repository path on local filesystem.
- * @param passRepoUrl Source repository URL (possibly with embedded credentials).
+ * @param repoUrl Source repository URL without embedded credentials.
  * @param cloneOptions Prepared libgit2 clone options.
  * @return StatusCode::OK on success, cancellation or clone failure status otherwise.
  * @note Connects to remote git endpoint and writes repository data to local filesystem.
  */
-Status executeClone(const std::string& downloadPath, const std::string& passRepoUrl, git_clone_options& cloneOptions) {
+Status executeClone(const std::string& downloadPath, const std::string& repoUrl, git_clone_options& cloneOptions) {
     git_repository* clonedRepo = nullptr;
-    const char* url = passRepoUrl.c_str();
+    const char* url = repoUrl.c_str();
     const char* path = downloadPath.c_str();
     SPDLOG_TRACE("Starting git clone to: {}", path);
     if (!libgit2::createLfsWipMarker(downloadPath)) {
@@ -1498,7 +1471,6 @@ Status finalizeAfterClone(const std::string& downloadPath,
 
 Status handleFreshClone(const std::string& downloadPath,
     const std::string& repoUrl,
-    const std::string& passRepoUrl,
     bool useProxy,
     const std::string& proxyUrl,
     const std::function<Status(bool)>& checkRepositoryStatusFn,
@@ -1513,7 +1485,7 @@ Status handleFreshClone(const std::string& downloadPath,
 
     SPDLOG_DEBUG("Downloading from url: {}", repoUrl.c_str());
 
-    auto status = executeClone(downloadPath, passRepoUrl, cloneOptions);
+    auto status = executeClone(downloadPath, repoUrl, cloneOptions);
     if (!status.ok()) {
         return status;
     }
@@ -1560,9 +1532,8 @@ Status HfDownloader::downloadModel() {
 
     const bool useProxy = CheckIfProxySet();
     std::string repoUrl = GetRepoUrl();
-    std::string passRepoUrl = GetRepositoryUrlWithPassword();
 
-    return handleFreshClone(this->downloadPath, repoUrl, passRepoUrl, useProxy, this->httpProxy, checkRepositoryStatusFn, removeReadonlyFn);
+    return handleFreshClone(this->downloadPath, repoUrl, useProxy, this->httpProxy, checkRepositoryStatusFn, removeReadonlyFn);
 }
 
 }  // namespace ovms

src/pull_module/libgit2.hpp

@@ -54,16 +54,14 @@ struct Libgt2InitGuard {
 
 class HfDownloader : public IModelDownloader {
 public:
-    HfDownloader(const std::string& sourceModel, const std::string& downloadPath, const std::string& hfEndpoint, const std::string& hfToken, const std::string& httpProxy, bool inOverwrite);
+    HfDownloader(const std::string& sourceModel, const std::string& downloadPath, const std::string& hfEndpoint, const std::string& httpProxy, bool inOverwrite);
     Status downloadModel() override;
 
 protected:
     const std::string hfEndpoint;
-    const std::string hfToken;
     const std::string httpProxy;
 
     std::string GetRepoUrl();
-    std::string GetRepositoryUrlWithPassword();
     bool CheckIfProxySet();
     Status RemoveReadonlyFileAttributeFromDir(const std::string& directoryPath);
     Status CheckRepositoryStatus(bool checkUntracked);

src/test/pull_hf_model_test.cpp

@@ -939,10 +939,9 @@ std::string sha256File(std::string_view path, std::error_code& ec) {
 
 class TestHfDownloader : public ovms::HfDownloader {
 public:
-    TestHfDownloader(const std::string& sourceModel, const std::string& downloadPath, const std::string& hfEndpoint, const std::string& hfToken, const std::string& httpProxy, bool overwrite) :
-        HfDownloader(sourceModel, downloadPath, hfEndpoint, hfToken, httpProxy, overwrite) {}
+    TestHfDownloader(const std::string& sourceModel, const std::string& downloadPath, const std::string& hfEndpoint, const std::string& /*hfToken*/, const std::string& httpProxy, bool overwrite) :
+        HfDownloader(sourceModel, downloadPath, hfEndpoint, httpProxy, overwrite) {}
     std::string GetRepoUrl() { return HfDownloader::GetRepoUrl(); }
-    std::string GetRepositoryUrlWithPassword() { return HfDownloader::GetRepositoryUrlWithPassword(); }
     bool CheckIfProxySet() { return HfDownloader::CheckIfProxySet(); }
     const std::string& getEndpoint() { return this->hfEndpoint; }
     const std::string& getProxy() { return this->httpProxy; }
@@ -1832,7 +1831,6 @@ TEST(HfDownloaderClassTest, Methods) {
     EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).CheckIfProxySet(), false);
     ASSERT_EQ(hfDownloader->getEndpoint(), "www.new_hf.com/");
     ASSERT_EQ(hfDownloader->GetRepoUrl(), "www.new_hf.com/model/name");
-    ASSERT_EQ(hfDownloader->GetRepositoryUrlWithPassword(), "123$$o_O123!AAbb:123$$o_O123!AAbb@www.new_hf.com/model/name");
 
     std::string expectedPath = downloadPath + "/" + modelName;
 #ifdef _WIN32
@@ -2117,29 +2115,29 @@ TEST_F(TestOptimumDownloaderSetup, PositiveOptimumExportCommandPassed) {
     ASSERT_EQ(optimumDownloader->downloadModel(), ovms::StatusCode::OK);
 }
 
-TEST(HfDownloaderClassTest, ProtocollsWithPassword) {
+TEST(HfDownloaderClassTest, ProtocolsWithoutPassword) {
     std::string modelName = "model/name";
     std::string downloadPath = "/path/to/Download";
     std::string hfEndpoint = "www.new_hf.com/";
     std::string hfToken = "";
-    EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "www.new_hf.com/model/name");
+    EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepoUrl(), "www.new_hf.com/model/name");
     hfEndpoint = "https://www.new_hf.com/";
-    EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "https://www.new_hf.com/model/name");
+    EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepoUrl(), "https://www.new_hf.com/model/name");
     hfEndpoint = "www.new_hf.com/";
     hfToken = "123!$token";
-    EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "123!$token:123!$token@www.new_hf.com/model/name");
+    EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepoUrl(), "www.new_hf.com/model/name");
     hfEndpoint = "http://www.new_hf.com/";
     hfToken = "123!$token";
-    EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "http://123!$token:123!$token@www.new_hf.com/model/name");
+    EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepoUrl(), "http://www.new_hf.com/model/name");
     hfEndpoint = "git://www.new_hf.com/";
     hfToken = "123!$token";
-    EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "git://123!$token:123!$token@www.new_hf.com/model/name");
+    EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepoUrl(), "git://www.new_hf.com/model/name");
     hfEndpoint = "ssh://www.new_hf.com/";
     hfToken = "123!$token";
-    EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "ssh://123!$token:123!$token@www.new_hf.com/model/name");
+    EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepoUrl(), "ssh://www.new_hf.com/model/name");
     hfEndpoint = "what_ever_is_here://www.new_hf.com/";
     hfToken = "123!$token";
-    EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepositoryUrlWithPassword(), "what_ever_is_here://123!$token:123!$token@www.new_hf.com/model/name");
+    EXPECT_EQ(TestHfDownloader(modelName, ovms::IModelDownloader::getGraphDirectory(downloadPath, modelName), hfEndpoint, hfToken, "", false).GetRepoUrl(), "what_ever_is_here://www.new_hf.com/model/name");
 }
 
 TEST_F(HfPull, MethodsNegative) {
@@ -2321,9 +2319,9 @@ TEST(Libgit2Framework, TimeoutTestProxy) {
     int e = git_libgit2_opts(GIT_OPT_SET_SERVER_CONNECT_TIMEOUT, 1000);
     EXPECT_EQ(e, 0);
 
-    std::string passRepoUrl = "https://huggingface.co/OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov";
+    std::string repoUrl = "https://huggingface.co/OpenVINO/Phi-3-mini-FastDraft-50M-int8-ov";
     const char* path = "/tmp/model";
-    int error = git_clone(&cloned_repo, passRepoUrl.c_str(), path, &clone_opts);
+    int error = git_clone(&cloned_repo, repoUrl.c_str(), path, &clone_opts);
     if (error != 0) {
         const git_error* err = git_error_last();
         if (err) {

third_party/libgit2/lfs.patch

@@ -427,7 +427,7 @@ new file mode 100644
 index 000000000..18490e5ad
 --- /dev/null
 +++ b/src/libgit2/lfs_filter.c
-@@ -0,0 +1,2014 @@
+@@ -0,0 +1,2029 @@
 +/*
 +/ Copyright 2025 Intel Corporation
 +/
@@ -1775,6 +1775,20 @@ index 000000000..18490e5ad
 +		status = setopt;  \
 +	}
 +
++static CURLcode configure_hf_token_auth(CURL *curl)
++{
++	const char *hf_token = getenv("HF_TOKEN");
++	CURLcode status = CURLE_OK;
++
++	if (!curl || !hf_token || !*hf_token)
++		return CURLE_OK;
++
++	CURL_SETOPT(curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC));
++	CURL_SETOPT(curl_easy_setopt(curl, CURLOPT_USERNAME, hf_token));
++	CURL_SETOPT(curl_easy_setopt(curl, CURLOPT_PASSWORD, hf_token));
++	return status;
++}
++
 +/*
 + * print_curl_error_details
 + * -------------------------
@@ -2121,6 +2135,7 @@ index 000000000..18490e5ad
 +	CURL_SETOPT(curl_easy_setopt(
 +	        info_curl, CURLOPT_ERRORBUFFER, info_error_buffer));
 +	CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_URL, lfs_info_url.ptr));
++	CURL_SETOPT(configure_hf_token_auth(info_curl));
 +	/* Add cURL resiliency */
 +	/* unlimited data */
 +	CURL_SETOPT(curl_easy_setopt(info_curl, CURLOPT_CONNECTTIMEOUT, 30L));