gitmirror/gitmirror.py

#!/usr/bin/env python3

import os
import sys
from subprocess import run,Popen, PIPE



source_server="git@git.annax.de"
local_base=f"./repositories"
mirror_base=f"{local_base}/{source_server}"
COM=sys.argv[0]



def git_info(source_server):
    proc = Popen(
        ["ssh",source_server,"info"], 
        stdout=PIPE,
        stderr=PIPE,
        encoding="utf-8")
    for line in proc.stdout.readlines():
        yield line

def repos_from_lines(lines):
    for line in lines:
        if line.find("gitolite") >= 0:
            continue
        fields = line.split()
        if len(fields) < 3:
            continue
        yield fields[2]


def run_threaded(target, args=[], procs=4, debug=False):
    import threading
    import queue
    inq = queue.Queue()
    outq = queue.Queue()
    jobs = []
    j = 0
    i = 0
    def worker(j):
        import time
        while True:
            arg = inq.get(block=True)
            if arg == None:
                break
            j += 1
            print(f"{COM}: mirror task ({j}/{i}): {source_repo(arg)} => {local_repo(arg)}")
            outq.put(target(source_repo(arg), local_repo(arg)))
    while len(jobs) < procs:
        th = threading.Thread(target=worker, args=(j,))
        th.start()
        jobs.append(th)
    i = 0
    for arg in args:
        i += 1
        inq.put(arg)
    for job in jobs:
        inq.put(None)
    for job in jobs:
        job.join()


def main():
    repos = repos_from_lines(git_info(source_server))
    from multiprocessing import Pool
    pool = Pool(processes=4)
    pool.map(pull_repo, repos)


def main3():
    repos = repos_from_lines(git_info(source_server))
    run_threaded(mirror_repo, repos, debug=True)

def main2():
    lines = git_info(source_server)
    repos = repos_from_lines(lines)
    for repo in repos:
        mirror_repo(repo)


def source_repo(repo):
    return f"{source_server}:{repo}"

def local_repo(repo):
    return f"{mirror_base}/{repo}.git"

processed = 0
def pull_repo(repo):
    global processed
    processed += 1
    print(f"{COM}: mirror task {processed}: {source_repo(repo)} => {local_repo(repo)}")
    return mirror_repo(source_repo(repo), local_repo(repo))

def mirror_repo(source_repo,local_repo):
    if os.path.isdir(f"{local_repo}/refs"):
        update_local_repo(local_repo)
    else:
        clone_repo(source_repo, local_repo)
    return True

def update_local_repo(local_repo):
    proc = run(["git","fetch","-p","origin"], cwd=local_repo)
    if not proc.returncode == 0:
        print(f"Error: {proc}, {local_repo}")
        _rm(local_repo)

def clone_repo(source_repo, local_repo):
    run(["rm","-rf",local_repo])
    run(["mkdir","-p",local_repo])
    proc = run(["git","clone","--bare",source_repo,"."], cwd=local_repo)
    if not proc.returncode == 0:
        print(f"Error: {proc}, {local_repo}")
        _rm(local_repo)

def _rm(di):
    run(["rm","-rf",local_repo])

if __name__ == "__main__":
    main()