Upload 2 files

app.py

@@ -21,6 +21,9 @@ HF_USER = os.environ.get("HF_USER") if os.environ.get("HF_USER") else "" # set y
 REGEX_HF_REPO = r'^[\w_\-\.]+/[\w_\-\.]+$'
 REGEX_HF_PATH = r'^[\w_\-\.]+/[\w_\-\.]+(/?:.+)?$'
 
+def debug(s: str):
+    print(s)
+
 def is_valid_reponame(repo_id: str):
     return re.fullmatch(REGEX_HF_REPO, repo_id)
 
@@ -38,9 +41,10 @@ def extract_src_reponame(source_repo: str):
         else:
             source_repo, target = re.findall(r'^(?:http.+\.co/)?(?:datasets)?(?:spaces)?([\w_\-\.]+/[\w_\-\.]+)/?(?:blob/main/)?(?:resolve/main/)?(.+)?$', source_repo)[0]
             target = urllib.parse.unquote(target.removesuffix("/"))
+        debug(f"Sourece repo_id:{source_repo} folder:{target}")
         return source_repo, target
     except Exception as e:
-        print(e)
+        debug(e)
         return source_repo, ""
 
 def extract_dst_reponame(dst_repo: str):
@@ -49,9 +53,10 @@ def extract_dst_reponame(dst_repo: str):
         else:
             dst_repo, subfolder = re.findall(r'^([\w_\-\.]+/[\w_\-\.]+)/?(.+)?$', dst_repo)[0]
             subfolder = subfolder.removesuffix("/")
+        debug(f"Destination repo_id:{dst_repo} folder:{subfolder}")
         return dst_repo, subfolder
     except Exception as e:
-        print(e)
+        debug(e)
         return dst_repo, ""
 
 def remove_repo_tags(repo_id: str, tags: list[str], repo_type: str, hf_token: str):
@@ -76,12 +81,12 @@ def duplicate(source_repo, dst_repo, repo_type, private, overwrite, auto_dir, re
     except Exception as e:
         raise gr.Error(f"""Oops, you forgot to login. Please use the loggin button on the top left to migrate your repo {e}""")
     
-    if not is_valid_path(dst_repo): raise gr.Error(f"Invalid dst_repo: {dst_repo}")
+    if not is_valid_path(extract_dst_reponame(dst_repo)[0]): raise gr.Error(f"Invalid dst_repo: {dst_repo}")
 
     try:
         source_repo, target = extract_src_reponame(source_repo)
         dst_repo, subfolder = extract_dst_reponame(dst_repo)
-        if auto_dir: subfolder = source_repo
+        if auto_dir: subfolder = f"{subfolder}/{source_repo}" if subfolder else source_repo
 
         if not overwrite and api.repo_exists(repo_id=dst_repo, repo_type=repo_type, token=hf_token): raise gr.Error(f"Repo already exists {dst_repo}")
 
@@ -90,12 +95,18 @@ def duplicate(source_repo, dst_repo, repo_type, private, overwrite, auto_dir, re
             create_repo(dst_repo, repo_type, private, hf_token)
             for path in api.list_repo_files(repo_id=source_repo, repo_type=repo_type, token=hf_token):
                 if target and target not in path: continue
+                path_in_repo = f"{subfolder}/{path}" if subfolder else path
+                if is_same_file_hf(source_repo, path, repo_type, dst_repo, path_in_repo, repo_type, hf_token):
+                    debug(f"{dst_repo}/{path_in_repo} is already exists. Skipping.")
+                    progress(0, desc=f"{dst_repo}/{path_in_repo} is already exists. Skipping.")
+                    continue
                 file = hf_hub_download(repo_id=source_repo, filename=path, repo_type=repo_type, local_dir=temp_dir, token=hf_token)
+                debug(f"Uploading {file} to {path_in_repo}")
                 if not Path(file).exists(): continue
                 if Path(file).is_dir(): # unused for now
-                    api.upload_folder(repo_id=dst_repo, folder_path=file, path_in_repo=f"{subfolder}/{path}" if subfolder else path, repo_type=repo_type, token=hf_token)
+                    api.upload_folder(repo_id=dst_repo, folder_path=file, path_in_repo=path_in_repo, repo_type=repo_type, token=hf_token)
                 elif Path(file).is_file():
-                    api.upload_file(repo_id=dst_repo, path_or_fileobj=file, path_in_repo=f"{subfolder}/{path}" if subfolder else path, repo_type=repo_type, token=hf_token)
+                    api.upload_file(repo_id=dst_repo, path_or_fileobj=file, path_in_repo=path_in_repo, repo_type=repo_type, token=hf_token)
                 if Path(file).exists(): Path(file).unlink()
             if repo_type == "dataset": repo_url = f"https://huggingface.co/datasets/{dst_repo}"
             elif repo_type == "space": repo_url = f"https://huggingface.co/spaces/{dst_repo}"
@@ -152,7 +163,7 @@ def duplicate_m2o(source_repos_str, dst_repo, repo_type, private, overwrite, oau
     except Exception as e:
         raise gr.Error(f"""Oops, you forgot to login. Please use the loggin button on the top left to migrate your repo {e}""")
     
-    if not is_valid_path(dst_repo): raise gr.Error(f"Invalid dst_repo: {dst_repo}")
+    if not is_valid_path(extract_dst_reponame(dst_repo)[0]): raise gr.Error(f"Invalid dst_repo: {dst_repo}")
 
     try:
         dst_repo, subfolder_prefix = extract_dst_reponame(dst_repo)
@@ -160,8 +171,7 @@ def duplicate_m2o(source_repos_str, dst_repo, repo_type, private, overwrite, oau
         source_repos = parse_repos(source_repos_str)
         for source_repo in source_repos:
             source_repo, target = extract_src_reponame(source_repo)
-
-            subfolder = subfolder_prefix + "/" + source_repo if subfolder_prefix else source_repo
+            subfolder = f"{subfolder_prefix}/{source_repo}" if subfolder_prefix else source_repo
 
             temp_dir = tempfile.mkdtemp()
             create_repo(dst_repo, repo_type, private, hf_token)
@@ -169,10 +179,12 @@ def duplicate_m2o(source_repos_str, dst_repo, repo_type, private, overwrite, oau
                 if target and target not in path: continue
                 path_in_repo = f"{subfolder}/{path}" if subfolder else path
                 if is_same_file_hf(source_repo, path, repo_type, dst_repo, path_in_repo, repo_type, hf_token):
-                    print(f"{dst_repo}/{path_in_repo} is already exists. Skipping.")
+                    debug(f"{dst_repo}/{path_in_repo} is already exists. Skipping.")
+                    progress(0, desc=f"{dst_repo}/{path_in_repo} is already exists. Skipping.")
                     continue
                 file = hf_hub_download(repo_id=source_repo, filename=path, repo_type=repo_type, local_dir=temp_dir, token=hf_token)
                 if not Path(file).exists(): continue
+                debug(f"Uploading {file} to {path_in_repo}")
                 if Path(file).is_dir(): # unused for now
                     api.upload_folder(repo_id=dst_repo, folder_path=file, path_in_repo=path_in_repo, repo_type=repo_type, token=hf_token)
                 elif Path(file).is_file():
@@ -270,6 +282,7 @@ with gr.Blocks(css=css) as demo:
                     with gr.Row():
                         submit_button = gr.Button("Submit", variant="primary")
                         clear_button = gr.Button("Clear", variant="secondary")
+                        stop_button = gr.Button("Stop", variant="stop")
                 with gr.Column():
                     output_md = gr.Markdown(label="output")
                     output_image = gr.Image(show_label=False)
@@ -292,6 +305,7 @@ with gr.Blocks(css=css) as demo:
                     with gr.Row():
                         m2o_submit_button = gr.Button("Submit", variant="primary")
                         m2o_clear_button = gr.Button("Clear", variant="secondary")
+                        m2o_stop_button = gr.Button("Stop", variant="stop")
                 with gr.Column():
                     m2o_output_md = gr.Markdown(label="output")
                     m2o_output_image = gr.Image(show_label=False)
@@ -318,20 +332,25 @@ with gr.Blocks(css=css) as demo:
                     with gr.Row():
                         m2m_submit_button = gr.Button("Submit", variant="primary")
                         m2m_clear_button = gr.Button("Clear", variant="secondary")
+                        m2m_stop_button = gr.Button("Stop", variant="stop")
                 with gr.Column():
                     m2m_output_md = gr.Markdown(label="output")
                     m2m_output_image = gr.Image(show_label=False)
     demo.load(fn=swap_visibilty, outputs=main_ui)
-    submit_button.click(duplicate, [search, dst_repo, repo_type, is_private, is_overwrite, is_subdir, is_remtag], [output_md, output_image])
+    submit_event = submit_button.click(duplicate, [search, dst_repo, repo_type, is_private, is_overwrite, is_subdir, is_remtag], [output_md, output_image])
+    stop_button.click(fn=None, inputs=None, outputs=None, cancels=[submit_event])
     clear_button.click(lambda: ("", HF_REPO, "model", True, True, True, True), None, [search, dst_repo, repo_type, is_private, is_overwrite, is_subdir, is_remtag], queue=False)
     m2o_search.submit(add_repo_text, [m2o_search, m2o_source_repos], [m2o_source_repos], queue=False)
-    m2o_submit_button.click(duplicate_m2o, [m2o_source_repos, m2o_dst_repo, m2o_repo_type, m2o_is_private, m2o_is_overwrite], [m2o_output_md, m2o_output_image])
+    m2o_submit_event = m2o_submit_button.click(duplicate_m2o, [m2o_source_repos, m2o_dst_repo, m2o_repo_type, m2o_is_private, m2o_is_overwrite], [m2o_output_md, m2o_output_image])
+    m2o_stop_button.click(fn=None, inputs=None, outputs=None, cancels=[m2o_submit_event])
     m2o_clear_button.click(lambda: ("", HF_REPO, "model", True, True, ""), None,
                            [m2o_search, m2o_dst_repo, m2o_repo_type, m2o_is_private, m2o_is_overwrite, m2o_source_repos], queue=False)
     m2m_search.submit(add_repo_text, [m2m_search, m2m_source_repos], [m2m_source_repos], queue=False)
-    m2m_submit_button.click(duplicate_m2m, [m2m_source_repos, m2m_user, m2m_repo_type, m2m_is_private, m2m_is_overwrite, m2m_is_remtag, m2m_prefix, m2m_suffix],
+    m2m_submit_event = m2m_submit_button.click(duplicate_m2m, [m2m_source_repos, m2m_user, m2m_repo_type, m2m_is_private, m2m_is_overwrite, m2m_is_remtag, m2m_prefix, m2m_suffix],
                             [m2m_output_md, m2m_output_image])
+    m2m_stop_button.click(fn=None, inputs=None, outputs=None, cancels=[m2m_submit_event])
     m2m_clear_button.click(lambda: ("", HF_USER, "model", True, False, True, "", HF_REPO_PREFIX, HF_REPO_SUFFIX), None,
                            [m2m_search, m2m_user, m2m_repo_type, m2m_is_private, m2m_is_overwrite, m2m_is_remtag, m2m_source_repos, m2m_prefix, m2m_suffix], queue=False)
+    
 
 demo.queue().launch()

requirements.txt

@@ -1,3 +1,5 @@
 huggingface_hub>=0.22.2
+hf_transfer
+hf_xet
 gradio_huggingfacehub_search==0.0.7
 pydantic==2.10.6