|
|
@@ -0,0 +1,100 @@
|
|
|
+#!/usr/bin/env python3
|
|
|
+
|
|
|
+import os
|
|
|
+import sys
|
|
|
+from subprocess import run,Popen, PIPE
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+source_server="git@git.annax.de"
|
|
|
+local_base=f"./repositories"
|
|
|
+mirror_base=f"{local_base}/{source_server}"
|
|
|
+COM=sys.argv[0]
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+def git_info(source_server):
|
|
|
+ proc = Popen(
|
|
|
+ ["ssh",source_server,"info"],
|
|
|
+ stdout=PIPE,
|
|
|
+ stderr=PIPE,
|
|
|
+ encoding="utf-8")
|
|
|
+ for line in proc.stdout.readlines():
|
|
|
+ yield line
|
|
|
+
|
|
|
+def repos_from_lines(lines):
|
|
|
+ for line in lines:
|
|
|
+ if line.find("gitolite") >= 0:
|
|
|
+ continue
|
|
|
+ fields = line.split()
|
|
|
+ if len(fields) < 3:
|
|
|
+ continue
|
|
|
+ yield fields[2]
|
|
|
+
|
|
|
+
|
|
|
+async def run_parallel(target, args=[], procs=4, debug=False):
|
|
|
+ import asyncio
|
|
|
+ loop = asyncio.new_event_loop()
|
|
|
+ jobs = []
|
|
|
+ lock = asyncio.Lock()
|
|
|
+ async def addJobs(target, args):
|
|
|
+ for arg in args:
|
|
|
+ print("addJobs:", len(jobs))
|
|
|
+ while len(jobs) >= procs:
|
|
|
+ lock.release()
|
|
|
+ await asyncio.sleep(0.5)
|
|
|
+ await lock.acquire()
|
|
|
+ print("adding Job with arg:", arg)
|
|
|
+ jobs.append(loop.create_task(target(arg)))
|
|
|
+
|
|
|
+ jobmon = loop.create_task(addJobs(target, args))
|
|
|
+ while True:
|
|
|
+ while len(jobs)==0:
|
|
|
+ await lock.acquire()
|
|
|
+ if jobmon.done():
|
|
|
+ break
|
|
|
+ result = await jobs[0]
|
|
|
+ print("Job done with result:", result, flush=True)
|
|
|
+ jobs.pop(0)
|
|
|
+ lock.release()
|
|
|
+ print(jobmon)
|
|
|
+ await jobmon
|
|
|
+
|
|
|
+
|
|
|
+
|
|
|
+def main2():
|
|
|
+ import asyncio
|
|
|
+ lines = git_info(source_server)
|
|
|
+ repos = repos_from_lines(lines)
|
|
|
+ asyncio.run(run_parallel(mirror_repo, repos, debug=True))
|
|
|
+
|
|
|
+def main():
|
|
|
+ lines = git_info(source_server)
|
|
|
+ repos = repos_from_lines(lines)
|
|
|
+ for repo in repos:
|
|
|
+ mirror_repo(repo)
|
|
|
+
|
|
|
+
|
|
|
+def mirror_repo(repo):
|
|
|
+ print(f"{COM}: checking: {repo}")
|
|
|
+ source_repo = f"{source_server}:{repo}"
|
|
|
+ local_repo = f"{mirror_base}/{repo}"
|
|
|
+ if os.path.isdir(f"{local_repo}/refs"):
|
|
|
+ update_local_repo(local_repo)
|
|
|
+ else:
|
|
|
+ clone_repo(source_repo, local_repo)
|
|
|
+ return True
|
|
|
+
|
|
|
+def update_local_repo(local_repo):
|
|
|
+ print(f"{COM}: {local_repo} exists, updating...")
|
|
|
+ run(["git","fetch","-p","origin"], cwd=local_repo)
|
|
|
+
|
|
|
+def clone_repo(source_repo, local_repo):
|
|
|
+ print(f"{COM}: Cloning {source_repo} => {local_repo}")
|
|
|
+ run(["rm","-rf",local_repo])
|
|
|
+ run(["mkdir","-p",local_repo])
|
|
|
+ run(["git","clone","--bare",source_repo,"."], cwd=local_repo)
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+ main()
|
|
|
+
|
