use threading for git clone

abe_setup.py

@@ -19,6 +19,9 @@ def main():
 
     sub = AbeSubmodule([args.output_dir, url.geturl(), args.tag], AbeSubType.SUBMODULE)
 
+    last_parent = None
+    jobs = []
+
     if args.abe_command:
         executeAbeCommand( args.output_dir, args.abe_command )
     else:
@@ -37,13 +40,31 @@ def main():
                     sys.exit(err.returncode)
 
             else:
-                clone_repo_cached( sub, cache_dir=args.cache_dir, fetch_origin=(not args.no_fetch_origin))
+                from threading import Thread
+                job = Thread(target=sub.clone_repo, kwargs={'cache_dir':args.cache_dir, 'fetch_origin':(not args.no_fetch_origin)})
+                job.start()
+                jobs.append(job)
+                while len(jobs) >= args.jobs:
+                    to_remove = None
+                    for j in jobs:
+                        if not j.is_alive():
+                            j.join()
+                            to_remove = j
+                    if to_remove:
+                        jobs.remove(to_remove)
+                if sub.last:
+                    while len(jobs):
+                        jobs.pop().join()
 
 
 def parser():
     import argparse
     p = argparse.ArgumentParser()
     p.add_argument('-d', '--debug', action='count', default=1)
+    p.add_argument('-j', '--jobs',
+                   default=1,
+                   type=int,
+                   help='use threaded clone with n jobs')
     p.add_argument('-q', '--quiet', action='store_true')
     p.add_argument('-o', '--output_dir',
                    default='.',
@@ -94,11 +115,19 @@ def recurse_abe_submodules(parent):
     subm = get_abe_subtree( parent.fullpath )
 
     acc = []
+    last = None
     for su in subm:
         su.parent = parent
         print(f"Repo: \"{parent.fullpath}\" has Submodule: \"{su.fullpath}\" Type: \"{su.subtype}\" From: \"{su.url.geturl()}\"") if DEBUG else None
         acc.append(su)
-        yield su
+        if last != None:
+            last.last = False
+            yield last
+        last = su
+
+    if last != None:
+        last.last = True
+        yield last
 
     for sub in acc:
         for ret in recurse_abe_submodules( sub ):
@@ -135,8 +164,14 @@ class ProgressPrinter(RemoteProgress):
     def update(self,op_code,cur_count,max_count=None,message=''):
         if DEBUG:
             cur_count_int = round(cur_count)
-            max_count_int = round(max_count or 100)
-            print(f"\033[2K{cur_count_int}/{max_count_int}", str(round(100*cur_count / (max_count or 100)))+'%', end='\r')
+            if not max_count:
+                div = f"cur_count_int/?"
+                perc = '??%'
+            else:
+                max_count_int = round(max_count)
+                div = f"{cur_count_int}/{max_count_int}"
+                perc = str(round(100*cur_count / (max_count or 100)))+'%'
+            print(f"\033[2K" + div, perc, message, end='\r')
             sys.stdout.flush()
 
 
@@ -239,6 +274,7 @@ class AbeSubType(Enum):
 
 class AbeSubmodule():
     subtype = AbeSubType.SUBMODULE
+    last = True
     def __init__(self, sub_info, subtype=None, parent=None):
         if subtype != None:
             self.subtype = subtype
@@ -275,6 +311,9 @@ class AbeSubmodule():
         import os.path
         return os.path.normpath(os.path.join( self.parent.fullpath, self.path))
 
+    def clone_repo(self, cache_dir=None, fetch_origin=True):
+        return clone_repo_cached( self, cache_dir=cache_dir, fetch_origin=fetch_origin)
+
 def get_abe_subtree(repo_dir):
     import itertools
     subfile_generators = [