gitmirror/gitmirror.py

#!/usr/bin/env python3

import os
import sys
from subprocess import run,Popen, PIPE



source_server="git@git.annax.de"
local_base=f"./repositories"
mirror_base=f"{local_base}/{source_server}"
COM=sys.argv[0]



def git_info(source_server):
    proc = Popen(
        ["ssh",source_server,"info"], 
        stdout=PIPE,
        stderr=PIPE,
        encoding="utf-8")
    for line in proc.stdout.readlines():
        yield line

def repos_from_lines(lines):
    for line in lines:
        if line.find("gitolite") >= 0:
            continue
        fields = line.split()
        if len(fields) < 3:
            continue
        yield fields[2]


async def run_parallel(target, args=[], procs=4, debug=False):
    import asyncio
    loop = asyncio.new_event_loop()
    jobs = []
    lock = asyncio.Lock()
    async def addJobs(target, args):
        for arg in args:
            print("addJobs:", len(jobs))
            while len(jobs) >= procs:
                lock.release()
                await asyncio.sleep(0.5)
            await lock.acquire()
            print("adding Job with arg:", arg)
            jobs.append(loop.create_task(target(arg)))

    jobmon = loop.create_task(addJobs(target, args))
    while True:
        while len(jobs)==0:
            await lock.acquire()
            if jobmon.done():
                break
        result = await jobs[0]
        print("Job done with result:", result, flush=True)
        jobs.pop(0)
        lock.release()
    print(jobmon)
    await jobmon



def main2():
    import asyncio
    lines = git_info(source_server)
    repos = repos_from_lines(lines)
    asyncio.run(run_parallel(mirror_repo, repos, debug=True))

def main():
    lines = git_info(source_server)
    repos = repos_from_lines(lines)
    for repo in repos:
        mirror_repo(repo)


def mirror_repo(repo):
    print(f"{COM}: checking: {repo}")
    source_repo = f"{source_server}:{repo}"
    local_repo = f"{mirror_base}/{repo}"
    if os.path.isdir(f"{local_repo}/refs"):
        update_local_repo(local_repo)
    else:
        clone_repo(source_repo, local_repo)
    return True

def update_local_repo(local_repo):
    print(f"{COM}: {local_repo} exists, updating...")
    run(["git","fetch","-p","origin"], cwd=local_repo)

def clone_repo(source_repo, local_repo):
    print(f"{COM}: Cloning {source_repo} => {local_repo}")
    run(["rm","-rf",local_repo])
    run(["mkdir","-p",local_repo])
    run(["git","clone","--bare",source_repo,"."], cwd=local_repo)

if __name__ == "__main__":
    main()