gitmirror/gitmirror.py

#!/usr/bin/env python3

import os
import sys
from subprocess import run,Popen, PIPE



source_server="git@git.annax.de"
local_base=f"./repositories"
mirror_base=f"{local_base}/{source_server}"
COM=sys.argv[0]


def main4():
    repos = repos_from_lines(git_info(source_server))
    from multiprocessing import Pool
    pool = Pool(processes=4)
    pool.map(pull_repo, repos)


def git_info(source_server):
    proc = Popen(
        ["ssh",source_server,"info"], 
        stdout=PIPE,
        stderr=PIPE,
        encoding="utf-8")
    for line in iter(proc.stdout.readline, ""):
        yield line
    proc.wait()
    print("GIT_INFO DONE!!!!!!!!!!!!!!!")

def repos_from_lines(lines):
    for line in lines:
        if line.find("gitolite") >= 0:
            continue
        fields = line.split()
        if len(fields) < 3:
            continue
        yield fields[2]


def run_threaded(target, args=[], procs=4, debug=False):
    import threading
    import queue
    inq = queue.Queue()
    outq = queue.Queue()
    jobs = []
    def worker():
        import time
        while True:
            num, arg = inq.get(block=True)
            if num == None:
                break
            print(f"{COM}: threaded task ({num}/{num+inq.qsize()}): {arg}")
            result = target(arg)
            outq.put(result)
    while len(jobs) < procs:
        th = threading.Thread(target=worker)
        th.start()
        jobs.append(th)
    def filler():
        i = 0
        for arg in args:
            i += 1
            inq.put((i,arg))
        for job in jobs:
            inq.put((None,None))
    fillThread = threading.Thread(target=filler)
    fillThread.start()
    while fillThread.is_alive():
        yield outq.get(block=True)
    fillThread.join()
    for job in jobs:
        job.join()
    while not outq.empty():
        yield outq.get(block=True)

def print_sleep(args):
    import time
    print("Job Started: ",args)
    time.sleep(3)
    print("Job Done!")
    return args

def main():
    repos = repos_from_lines(git_info(source_server))
    for output in run_threaded(pull_repo, repos, debug=True):
        pass

def main2():
    lines = git_info(source_server)
    repos = repos_from_lines(lines)
    for repo in repos:
        mirror_repo(repo)


def source_repo(repo):
    return f"{source_server}:{repo}"

def local_repo(repo):
    return f"{mirror_base}/{repo}.git"

def pull_repo(repo):
    return mirror_repo(source_repo(repo), local_repo(repo))

def mirror_repo(source_repo,local_repo):
    if not os.path.isdir(f"{local_repo}/refs"):
        init_repo(source_repo, local_repo)
    update_local_repo(local_repo)
    return True

def update_local_repo(local_repo):
    proc = run(["git","fetch","-p","origin"], cwd=local_repo)
    if not proc.returncode == 0:
        print(f"Error: {proc}, {local_repo}")
        _rm(local_repo)

def init_repo(source_repo, local_repo):
    _rm(local_repo)
    run(["mkdir","-p",local_repo])
    proc1 = run(["git","init","--bare"], cwd=local_repo)
    proc2 = run(["git","remote","add","origin",source_repo], cwd=local_repo)
    if not proc1.returncode == 0 and proc2.returncode == 0:
        print(f"Error: {proc1}, {proc2}, {local_repo}")
        _rm(local_repo)

def _rm(di):
    run(["rm","-rf", di])

if __name__ == "__main__":
    main()