hermes-agent/scripts/batch-push.py

#!/usr/bin/env python3
"""分段推送到 Gitea：用 /contents API 分批上传文件

策略：
1. 每批上传最多 20MB 的文件，控制在 nginx 的 ~30MB 限制内
2. 按目录结构分批上传，保持提交历史相对整洁
3. 小文件合并到同一 commit，大文件单独 commit
"""

import os, base64, json, subprocess, sys, time

GITEA_URL = "https://git666.u7f.cn"
OWNER = "sawz"
REPO = "hermes-agent"
AUTH = "Basic c2F3ejo0NTMyNDYyMnd3"

MAX_BATCH_SIZE = 20 * 1024 * 1024  # 20MB 批次上限（留余量）

def run_curl(method, path, data=None):
    """执行 curl 请求"""
    url = f"{GITEA_URL}{path}"
    cmd = ["curl", "-s", "-w", "\n%{http_code}", "-X", method, url,
           "-H", f"Authorization: {AUTH}",
           "-H", "Content-Type: application/json"]
    if data:
        cmd.extend(["-d", json.dumps(data)])
    
    for attempt in range(3):
        try:
            result = subprocess.run(cmd, capture_output=True, text=True, timeout=60)
            lines = result.stdout.strip().split("\n")
            if len(lines) < 2:
                return None, 0
            http_code = int(lines[-1])
            body = "\n".join(lines[:-1])
            if body:
                return json.loads(body), http_code
            return {}, http_code
        except Exception as e:
            if attempt < 2:
                time.sleep(2)
                continue
            print(f"  API 错误: {e}")
            return None, 0

def get_last_sha():
    """获取当前 main 分支的最新 commit SHA"""
    resp, code = run_curl("GET", f"/api/v1/repos/{OWNER}/{REPO}/branches/main")
    if resp and "commit" in resp:
        return resp["commit"]["id"]
    return None

def upload_file(local_path):
    """上传单个文件到 Gitea"""
    repo_path = os.path.relpath(local_path, repo_dir)
    if repo_path.startswith("./"):
        repo_path = repo_path[2:]
    
    with open(local_path, "rb") as f:
        content = base64.b64encode(f.read()).decode()
    
    # 检查文件大小
    file_size = len(content) * 3 // 4  # base64 膨胀约 1/3
    if file_size > MAX_BATCH_SIZE:
        print(f"  ⚠ 文件过大 ({file_size/1024/1024:.1f}MB): {repo_path}")
        return None
    
    data = {
        "content": content,
        "message": f"add {repo_path}",
        "branch": "main"
    }
    
    resp, code = run_curl("PUT", f"/api/v1/repos/{OWNER}/{REPO}/contents/{repo_path}", data)
    
    if code == 201:
        return resp.get("commit", {}).get("sha", "")
    elif code == 404:
        print(f"  ✗ 仓库不存在")
        return None
    elif code == 413:
        print(f"  ✗ 413 太大: {repo_path} ({file_size/1024/1024:.1f}MB)")
        return None
    else:
        # 可能是文件已存在
        return None

def get_all_files(repo_root):
    """递归获取所有需要上传的文件"""
    files = []
    exclude_dirs = {".git", "__pycache__", ".venv", ".vscode", 
                    ".hermes-docker", ".notebooklm-home", ".notebooklm-cli-venv",
                    ".notebooklm-playwright", ".pip-cache", ".uv-cache", "venv"}
    exclude_files = {".DS_Store"}
    
    for root, dirs, filenames in os.walk(repo_root):
        dirs[:] = [d for d in dirs if d not in exclude_dirs]
        for f in filenames:
            if f in exclude_files or f.endswith(".pyc"):
                continue
            filepath = os.path.join(root, f)
            filesize = os.path.getsize(filepath)
            files.append((filepath, filesize))
    
    return files

def main():
    global repo_dir
    repo_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    
    # 先确保仓库和 main 分支存在
    resp, code = run_curl("GET", f"/api/v1/repos/{OWNER}/{REPO}")
    if not resp or "id" not in resp:
        print("仓库不存在!")
        sys.exit(1)
    
    # 检查 main 分支
    branch, _ = run_curl("GET", f"/api/v1/repos/{OWNER}/{REPO}/branches/main")
    if not branch:
        # 用 README 初始化
        print("初始化仓库 (README)...")
        readme_b64 = base64.b64encode(b"# Hermes Agent\n\nBackup mirror").decode()
        data = {
            "content": readme_b64,
            "message": "Initial commit",
            "branch": "main"
        }
        resp, code = run_curl("PUT", f"/api/v1/repos/{OWNER}/{REPO}/contents/README.md", data)
        if code == 201:
            print("  初始化成功")
        else:
            print(f"  初始化失败: {code}")
            sys.exit(1)
    
    # 获取文件列表
    all_files = get_all_files(repo_dir)
    total_files = len(all_files)
    total_size = sum(sz for _, sz in all_files)
    print(f"共 {total_files} 个文件, {total_size/1024/1024:.1f}MB")
    
    # 分批上传
    uploaded = 0
    skipped = 0
    batch_files = []
    batch_size = 0
    
    for filepath, filesize in all_files:
        if filesize > MAX_BATCH_SIZE:
            print(f"  ⚠ 超大文件跳过: {os.path.relpath(filepath, repo_dir)} ({filesize/1024/1024:.1f}MB)")
            skipped += 1
            continue
        
        if batch_size + filesize > MAX_BATCH_SIZE and batch_files:
            # 提交当前批次
            for fp in batch_files:
                result = upload_file(fp)
                if result:
                    uploaded += 1
                else:
                    skipped += 1
            batch_files = []
            batch_size = 0
            print(f"  进度: {uploaded}/{total_files} (跳过 {skipped})")
        
        batch_files.append(filepath)
        batch_size += filesize
    
    # 提交最后一批
    if batch_files:
        for fp in batch_files:
            result = upload_file(fp)
            if result:
                uploaded += 1
            else:
                skipped += 1
    
    print(f"\n完成！上传 {uploaded}, 跳过 {skipped}")

if __name__ == "__main__":
    main()