Initial import of NousResearch/hermes-agent

scripts/batch-push.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""分段推送到 Gitea：用 /contents API 分批上传文件
+
+策略：
+1. 每批上传最多 20MB 的文件，控制在 nginx 的 ~30MB 限制内
+2. 按目录结构分批上传，保持提交历史相对整洁
+3. 小文件合并到同一 commit，大文件单独 commit
+"""
+
+import os, base64, json, subprocess, sys, time
+
+GITEA_URL = "https://git666.u7f.cn"
+OWNER = "sawz"
+REPO = "hermes-agent"
+AUTH = "Basic c2F3ejo0NTMyNDYyMnd3"
+
+MAX_BATCH_SIZE = 20 * 1024 * 1024  # 20MB 批次上限（留余量）
+
+def run_curl(method, path, data=None):
+    """执行 curl 请求"""
+    url = f"{GITEA_URL}{path}"
+    cmd = ["curl", "-s", "-w", "\n%{http_code}", "-X", method, url,
+           "-H", f"Authorization: {AUTH}",
+           "-H", "Content-Type: application/json"]
+    if data:
+        cmd.extend(["-d", json.dumps(data)])
+    
+    for attempt in range(3):
+        try:
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
+            lines = result.stdout.strip().split("\n")
+            if len(lines) < 2:
+                return None, 0
+            http_code = int(lines[-1])
+            body = "\n".join(lines[:-1])
+            if body:
+                return json.loads(body), http_code
+            return {}, http_code
+        except Exception as e:
+            if attempt < 2:
+                time.sleep(2)
+                continue
+            print(f"  API 错误: {e}")
+            return None, 0
+
+def get_last_sha():
+    """获取当前 main 分支的最新 commit SHA"""
+    resp, code = run_curl("GET", f"/api/v1/repos/{OWNER}/{REPO}/branches/main")
+    if resp and "commit" in resp:
+        return resp["commit"]["id"]
+    return None
+
+def upload_file(local_path):
+    """上传单个文件到 Gitea"""
+    repo_path = os.path.relpath(local_path, repo_dir)
+    if repo_path.startswith("./"):
+        repo_path = repo_path[2:]
+    
+    with open(local_path, "rb") as f:
+        content = base64.b64encode(f.read()).decode()
+    
+    # 检查文件大小
+    file_size = len(content) * 3 // 4  # base64 膨胀约 1/3
+    if file_size > MAX_BATCH_SIZE:
+        print(f"  ⚠ 文件过大 ({file_size/1024/1024:.1f}MB): {repo_path}")
+        return None
+    
+    data = {
+        "content": content,
+        "message": f"add {repo_path}",
+        "branch": "main"
+    }
+    
+    resp, code = run_curl("PUT", f"/api/v1/repos/{OWNER}/{REPO}/contents/{repo_path}", data)
+    
+    if code == 201:
+        return resp.get("commit", {}).get("sha", "")
+    elif code == 404:
+        print(f"  ✗ 仓库不存在")
+        return None
+    elif code == 413:
+        print(f"  ✗ 413 太大: {repo_path} ({file_size/1024/1024:.1f}MB)")
+        return None
+    else:
+        # 可能是文件已存在
+        return None
+
+def get_all_files(repo_root):
+    """递归获取所有需要上传的文件"""
+    files = []
+    exclude_dirs = {".git", "__pycache__", ".venv", ".vscode", 
+                    ".hermes-docker", ".notebooklm-home", ".notebooklm-cli-venv",
+                    ".notebooklm-playwright", ".pip-cache", ".uv-cache", "venv"}
+    exclude_files = {".DS_Store"}
+    
+    for root, dirs, filenames in os.walk(repo_root):
+        dirs[:] = [d for d in dirs if d not in exclude_dirs]
+        for f in filenames:
+            if f in exclude_files or f.endswith(".pyc"):
+                continue
+            filepath = os.path.join(root, f)
+            filesize = os.path.getsize(filepath)
+            files.append((filepath, filesize))
+    
+    return files
+
+def main():
+    global repo_dir
+    repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    
+    # 先确保仓库和 main 分支存在
+    resp, code = run_curl("GET", f"/api/v1/repos/{OWNER}/{REPO}")
+    if not resp or "id" not in resp:
+        print("仓库不存在!")
+        sys.exit(1)
+    
+    # 检查 main 分支
+    branch, _ = run_curl("GET", f"/api/v1/repos/{OWNER}/{REPO}/branches/main")
+    if not branch:
+        # 用 README 初始化
+        print("初始化仓库 (README)...")
+        readme_b64 = base64.b64encode(b"# Hermes Agent\n\nBackup mirror").decode()
+        data = {
+            "content": readme_b64,
+            "message": "Initial commit",
+            "branch": "main"
+        }
+        resp, code = run_curl("PUT", f"/api/v1/repos/{OWNER}/{REPO}/contents/README.md", data)
+        if code == 201:
+            print("  初始化成功")
+        else:
+            print(f"  初始化失败: {code}")
+            sys.exit(1)
+    
+    # 获取文件列表
+    all_files = get_all_files(repo_dir)
+    total_files = len(all_files)
+    total_size = sum(sz for _, sz in all_files)
+    print(f"共 {total_files} 个文件, {total_size/1024/1024:.1f}MB")
+    
+    # 分批上传
+    uploaded = 0
+    skipped = 0
+    batch_files = []
+    batch_size = 0
+    
+    for filepath, filesize in all_files:
+        if filesize > MAX_BATCH_SIZE:
+            print(f"  ⚠ 超大文件跳过: {os.path.relpath(filepath, repo_dir)} ({filesize/1024/1024:.1f}MB)")
+            skipped += 1
+            continue
+        
+        if batch_size + filesize > MAX_BATCH_SIZE and batch_files:
+            # 提交当前批次
+            for fp in batch_files:
+                result = upload_file(fp)
+                if result:
+                    uploaded += 1
+                else:
+                    skipped += 1
+            batch_files = []
+            batch_size = 0
+            print(f"  进度: {uploaded}/{total_files} (跳过 {skipped})")
+        
+        batch_files.append(filepath)
+        batch_size += filesize
+    
+    # 提交最后一批
+    if batch_files:
+        for fp in batch_files:
+            result = upload_file(fp)
+            if result:
+                uploaded += 1
+            else:
+                skipped += 1
+    
+    print(f"\n完成！上传 {uploaded}, 跳过 {skipped}")
+
+if __name__ == "__main__":
+    main()