better logging

gitmirror.py

@@ -6,23 +6,46 @@ from subprocess import run,Popen, PIPE
 
 
 
-source_server="git@git.annax.de"
 local_base=f"./repositories"
-mirror_base=f"{local_base}/{source_server}"
 COM=sys.argv[0]
 
 def parser():
-    from argparse import Parser
+    from argparse import ArgumentParser
+    p = ArgumentParser()
+    p.add_argument("source")
+    p.add_argument("-b", "--base", default="./repositories",help="local basedir for mirror")
+    return p
 
 def main():
-    repos = repos_from_lines(git_info(source_server))
-    for output in map_threaded(pull_repo, repos, debug=True):
+    p = parser()
+    args = p.parse_args()
+    repos = repos_from_lines(git_info(args.source))
+    repo_obj = [Repo(args.source,r,args.base) for r in repos]
+    for output in map_threaded(
+            pull_repo,
+            repo_obj,
+            debug=lambda cur,tot,arg:print(f"{COM}: {cur}/{tot}: {arg}"),
+        ):
         pass
 
+class Repo():
+    def __init__(self,remote,path,base):
+        self.remote = remote
+        self.path   = path
+        self.base   = base
+
+    def source_uri(self):
+        return f"{self.remote}:{self.path}"
+
+    def local_path(self):
+        return os.path.join(self.base, self.remote, self.path+".git")
+
+    def __repr__(self):
+        return f"Repo('{self.remote}','{self.path}','{self.base}')"
 
 def git_info(source_server):
     proc = Popen(
-        ["ssh",source_server,"info"], 
+        ["ssh",source_server,"info"],
         stdout=PIPE,
         stderr=PIPE,
         encoding="utf-8")
@@ -40,24 +63,38 @@ def repos_from_lines(lines):
             continue
         yield fields[2]
 
+
+def pull_repo(repo):
+
+    if not os.path.isdir(os.path.join(repo.local_path() , "refs")):
+
+        init_repo(repo.source_uri(), repo.local_path())
+
+    update_local_repo(repo.local_path())
+
+    return True
+
 '''
 :param target: A function to be mapped.
 :param args: An Array or Generator to be mapped.
 :return: Generator of function return values, but not in same order.
 '''
-def map_threaded(target, args=[], procs=4, debug=False):
+def map_threaded(target, args=[], procs=4, debug=None):
     import threading
     import queue
     inq = queue.Queue()
     outq = queue.Queue()
     jobs = []
     def worker():
-        import time
         while True:
-            num, arg = inq.get(block=True)
+            try:
+                num, arg = inq.get(block=False)
+            except:
+                continue
             if num == None:
                 break
-            print(f"{COM}: threaded task ({num}/{num+inq.qsize()}): {arg}")
+            if debug:
+                debug( num, num+inq.qsize(), arg)
             result = target(arg)
             outq.put(result)
     while len(jobs) < procs:
@@ -81,35 +118,6 @@ def map_threaded(target, args=[], procs=4, debug=False):
     while not outq.empty():
         yield outq.get(block=True)
 
-def print_sleep(args):
-    import time
-    print("Job Started: ",args)
-    time.sleep(3)
-    print("Job Done!")
-    return args
-
-def main2():
-    lines = git_info(source_server)
-    repos = repos_from_lines(lines)
-    for repo in repos:
-        mirror_repo(repo)
-
-
-def source_repo(repo):
-    return f"{source_server}:{repo}"
-
-def local_repo(repo):
-    return f"{mirror_base}/{repo}.git"
-
-def pull_repo(repo):
-    return mirror_repo(source_repo(repo), local_repo(repo))
-
-def mirror_repo(source_repo,local_repo):
-    if not os.path.isdir(f"{local_repo}/refs"):
-        init_repo(source_repo, local_repo)
-    update_local_repo(local_repo)
-    return True
-
 def update_local_repo(local_repo):
     proc = run(["git","fetch","-p","origin"], cwd=local_repo)
     if not proc.returncode == 0: