gitmirror/gitmirror.py

#!/usr/bin/env python3

import os
import sys
from subprocess import run,Popen, PIPE



local_base=f"./repositories"
COM=sys.argv[0]

def parser():
    from argparse import ArgumentParser
    p = ArgumentParser()
    p.add_argument("source")
    p.add_argument("-b", "--base", default="./repositories",help="local basedir for mirror")
    return p

def main():
    p = parser()
    args = p.parse_args()
    repos = repos_from_lines(git_info(args.source))
    repo_obj = [Repo(args.source,r,args.base) for r in repos]
    for output in map_threaded(
            pull_repo,
            repo_obj,
            debug=lambda cur,tot,arg:print(f"{COM}: {cur}/{tot}: {arg}"),
        ):
        pass

class Repo():
    def __init__(self,remote,path,base):
        self.remote = remote
        self.path   = path
        self.base   = base

    def source_uri(self):
        return f"{self.remote}:{self.path}"

    def local_path(self):
        return os.path.join(self.base, self.remote, self.path+".git")

    def __repr__(self):
        return f"Repo('{self.remote}','{self.path}','{self.base}')"

def git_info(source_server):
    proc = Popen(
        ["ssh",source_server,"info"],
        stdout=PIPE,
        stderr=PIPE,
        encoding="utf-8")
    for line in iter(proc.stdout.readline, ""):
        yield line
    proc.wait()
    print("GIT_INFO DONE!!!!!!!!!!!!!!!")

def repos_from_lines(lines):
    for line in lines:
        if line.find("gitolite") >= 0:
            continue
        fields = line.split()
        if len(fields) < 3:
            continue
        yield fields[2]


def pull_repo(repo):

    if not os.path.isdir(os.path.join(repo.local_path() , "refs")):

        init_repo(repo.source_uri(), repo.local_path())

    update_local_repo(repo.local_path())

    return True

'''
:param target: A function to be mapped.
:param args: An Array or Generator to be mapped.
:return: Generator of function return values, but not in same order.
'''
def map_threaded(target, args=[], procs=4, debug=None):
    import threading
    import queue
    inq = queue.Queue()
    outq = queue.Queue()
    jobs = []
    def worker():
        while True:
            try:
                num, arg = inq.get(block=False)
            except:
                continue
            if num == None:
                break
            if debug:
                debug( num, num+inq.qsize(), arg)
            result = target(arg)
            outq.put(result)
    while len(jobs) < procs:
        th = threading.Thread(target=worker)
        th.start()
        jobs.append(th)
    def filler():
        i = 0
        for arg in args:
            i += 1
            inq.put((i,arg))
        for job in jobs:
            inq.put((None,None))
    fillThread = threading.Thread(target=filler)
    fillThread.start()
    while fillThread.is_alive():
        yield outq.get(block=True)
    fillThread.join()
    for job in jobs:
        job.join()
    while not outq.empty():
        yield outq.get(block=True)

def update_local_repo(local_repo):
    proc = run(["git","fetch","-p","origin"], cwd=local_repo)
    if not proc.returncode == 0:
        print(f"Error: {proc}, {local_repo}")
        _rm(local_repo)

def init_repo(source_repo, local_repo):
    _rm(local_repo)
    run(["mkdir","-p",local_repo])
    proc1 = run(["git","init","--bare"], cwd=local_repo)
    proc2 = run(["git","remote","add","origin",source_repo], cwd=local_repo)
    if not proc1.returncode == 0 and proc2.returncode == 0:
        print(f"Error: {proc1}, {proc2}, {local_repo}")
        _rm(local_repo)

def _rm(di):
    run(["rm","-rf", di])

if __name__ == "__main__":
    main()