Spaces:
Running
on
Zero
Running
on
Zero
Update hf_merge.py
Browse files- hf_merge.py +12 -26
hf_merge.py
CHANGED
@@ -114,16 +114,6 @@ class ModelMerger:
|
|
114 |
copy_nontensor_files(base_model_path, output_dir)
|
115 |
save_tensor_map(self.tensor_map, output_dir)
|
116 |
|
117 |
-
def is_repo_exists(repo_id, token):
|
118 |
-
from huggingface_hub import HfApi
|
119 |
-
api = HfApi(token=token)
|
120 |
-
try:
|
121 |
-
if api.repo_exists(repo_id=repo_id): return True
|
122 |
-
else: return False
|
123 |
-
except Exception as e:
|
124 |
-
print(e)
|
125 |
-
print(f"Error: Failed to connect {repo_id}.")
|
126 |
-
return True # for safe
|
127 |
|
128 |
|
129 |
def upload_model(self, output_dir, commit_message):
|
@@ -134,25 +124,21 @@ class ModelMerger:
|
|
134 |
output_dir (str): The path to the output directory containing the merged model.
|
135 |
commit_message (str): The commit message for the upload.
|
136 |
"""
|
137 |
-
try:
|
138 |
-
# Create a new repository if it doesn't exist
|
139 |
-
if not self.api.repo_exists(repo_id=self.repo_id, token=self.token):
|
140 |
-
self.api.create_repo(repo_id=self.repo_id, token=self.token, private=True)
|
141 |
|
142 |
# Upload the folder to the repository
|
143 |
-
|
144 |
-
|
145 |
-
|
146 |
-
|
147 |
-
|
148 |
)
|
149 |
-
|
150 |
-
|
151 |
-
|
152 |
-
|
153 |
-
|
154 |
-
|
155 |
-
|
156 |
def get_max_vocab_size(repo_list):
|
157 |
"""
|
158 |
Get the maximum vocabulary size from a list of repositories.
|
|
|
114 |
copy_nontensor_files(base_model_path, output_dir)
|
115 |
save_tensor_map(self.tensor_map, output_dir)
|
116 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
117 |
|
118 |
|
119 |
def upload_model(self, output_dir, commit_message):
|
|
|
124 |
output_dir (str): The path to the output directory containing the merged model.
|
125 |
commit_message (str): The commit message for the upload.
|
126 |
"""
|
|
|
|
|
|
|
|
|
127 |
|
128 |
# Upload the folder to the repository
|
129 |
+
self.api.upload_folder(
|
130 |
+
repo_id=self.repo_id,
|
131 |
+
folder_path=output_dir,
|
132 |
+
commit_message=commit_message,
|
133 |
+
token=self.token
|
134 |
)
|
135 |
+
url = f"https://huggingface.co/{self.repo_id}"
|
136 |
+
logging.info(f"Model uploaded successfully to {url}")
|
137 |
+
return url
|
138 |
+
except Exception as e:
|
139 |
+
logging.error(f"Error: Failed to upload to {self.repo_id}.")
|
140 |
+
logging.error(e)
|
141 |
+
return ""
|
142 |
def get_max_vocab_size(repo_list):
|
143 |
"""
|
144 |
Get the maximum vocabulary size from a list of repositories.
|