consequently use Object props

abe_setup.py

@@ -13,22 +13,20 @@ def main():
 
     print(args) if DEBUG>1 else None
 
-    source_url = sane_origin_url( args.source_url, args.output_dir )
+    count_submodules = 0
+
+    for sub in abe_submodules( args.output_dir, args.source_url, args.tag):
+        count_submodules += 1
+        if DEBUG > 0:
+            print("Count:",count_submodules)
 
-    if args.execute_command:
-        executeAbeCommand( args.output_dir, args.execute_command )
-    else:
-        clone_repo_cached( args.output_dir, source_url.geturl(), args.tag,
-            cache_dir=args.cache_dir, fetch_origin=(not args.no_fetch_origin))
-    for ret in recurse_abe_submodules( args.output_dir, source_url, args.tag):
         if args.abe_command:
-            executeAbeCommand( ret.path, args.execute_command )
+            executeAbeCommand( sub.path, args.execute_command )
         else:
-            clone_repo_cached( ret.path, ret.remote, ret.ref,
-                cache_dir=args.cache_dir, fetch_origin=(not args.no_fetch_origin))
+            clone_repo_cached( sub, cache_dir=args.cache_dir, fetch_origin=(not args.no_fetch_origin))
         if args.execute_command:
             from subprocess import run
-            run(args.execute_command, shell=True,check=True, cwd=ret.path)
+            run(args.execute_command, shell=True,check=True, cwd=sub.path)
 
 
 def parser():
@@ -53,21 +51,27 @@ def executeAbeCommand( repo_path, command_name ):
     if os.path.isfile(commandFile):
         run([ commandFileRel ], check=True, cwd=repo_path)
 
-def recurse_abe_submodules(path, remote, ref, parent=None):
-    import os
-    from urllib.parse import urlparse
 
+def abe_submodules(path, remote, ref, parent=None):
     url = sane_origin_url( remote, path )
 
-    sub = AbeSubmodule([path, url.geturl(), ref], AbeSubType.SUBMODULE, parent)
+    sub = AbeSubmodule([path, url.geturl(), ref], AbeSubType.SUBMODULE)
     yield sub
+    for s in recurse_abe_submodules(sub):
+        yield s
 
-    subm = get_abe_subtree( path )
 
+def recurse_abe_submodules(parent):
+
+    subm = get_abe_subtree( parent.fullpath )
+
+    acc = []
     for su in subm:
-        su.parent = sub
-        for ret in recurse_abe_submodules( su.fullpath, su.url, su.ref, sub):
-            print(f"Repo: \"{path}\" has Submodule: \"{su.fullpath}\" Type: \"{su.subtype}\" From: \"{su.url.geturl()}\"") if DEBUG else None
+        su.parent = parent
+        print(f"Repo: \"{parent.fullpath}\" has Submodule: \"{su.fullpath}\" Type: \"{su.subtype}\" From: \"{su.url.geturl()}\"") if DEBUG else None
+        acc.append(su)
+        yield su
+        for ret in recurse_abe_submodules( su ):
             yield ret
 
 
@@ -94,58 +98,50 @@ def origin_url(path):
     for url in repo.remotes.origin.urls:
         return url
 
-def cache_path(cache, remote):
-    import os.path
-    from urllib.parse import urlparse
-    for url in remote:
-        sane_url = urlparse(url)
-        sane_url = sane_url._replace(
-            path = str(sane_url.path).lstrip('/'),
-            netloc = str(sane_url.netloc).replace('/','_').replace( '@', '_').replace(':', '_')
-        )
-        return os.path.join( cache, sane_url.netloc, sane_url.path)
-
-def real_relpath(dest, source='.'):
-    import os.path
-    real_dest = os.path.realpath(dest)
-    real_source = os.path.realpath(source)
-    return os.path.relpath(real_dest, real_source)
 
 from git import RemoteProgress
+import sys
 class ProgressPrinter(RemoteProgress):
     def update(self,op_code,cur_count,max_count=None,message=''):
-        cur_count_int = round(cur_count)
-        max_count_int = round(max_count or 100)
-        print(f"\033[2K{cur_count_int}/{max_count_int}", str(round(100*cur_count / (max_count or 100)))+'%', end="\r")
+        if DEBUG:
+            cur_count_int = round(cur_count)
+            max_count_int = round(max_count or 100)
+            print(f"\033[2K{cur_count_int}/{max_count_int}", str(round(100*cur_count / (max_count or 100)))+'%', end='\r')
+            sys.stdout.flush()
+
 
-def clone_repo_cached( path, remote=None, ref=None,
-            cache_dir=None, bare=False, fetch_origin=True):
-    print('Repo:', path) if DEBUG>1 else None
+def clone_repo_cached( sub, cache_dir=None, bare=False, fetch_origin=True):
+    print('Repo:', sub.path) if DEBUG>1 else None
     from git import Repo
-    repo = Repo.init(path, bare=bare)
+    ref = sub.ref
+    url = sub.url.geturl()
+    repo = Repo.init(sub.fullpath, bare=bare)
     if 'origin' in repo.remotes:
         origin = repo.remotes['origin']
     else:
-        origin = repo.create_remote('origin', remote)
+        origin = repo.create_remote('origin', url)
 
     if cache_dir != None:
         cache_repo_path = cache_path(cache_dir, origin.urls)
-        print(f"Cacheing from: {cache_repo_path}, origin: {remote}") if DEBUG else None
+        cache_link = real_relpath( cache_repo_path, sub.fullpath)
+        print(f"Cacheing from: {cache_repo_path}, origin: {url}") if DEBUG else None
+        cache_repo = AbeSubmodule((cache_repo_path, url, sub.ref), AbeSubType.SUBMODULE, parent=None)
         clone_repo_cached(
-            cache_repo_path,
-            remote = remote,
-            ref = ref,
+            cache_repo,
             cache_dir = None,
             bare = True,
             fetch_origin = fetch_origin)
         if 'cache' in repo.remotes:
             cache = repo.remotes['cache']
         else:
-            cache = repo.create_remote('cache', real_relpath( cache_repo_path, path))
-            cache.set_url("no_push" , '--push')
+            print(f"Setting up cache: {cache_link}, for Repo: {path}") if DEBUG>2 else None
+            cache = repo.create_remote('cache', cache_link)
+            # cache.set_url("no_push" , '--push')
         cache.fetch(refspec="+refs/remotes/origin/*:refs/remotes/origin/*",progress=ProgressPrinter())
         print()
-    elif fetch_origin:
+    elif fetch_origin or (
+        (ref not in origin.refs) and
+        (ref not in repo.tags)):
         origin.fetch(progress=ProgressPrinter())
         print()
 
@@ -159,7 +155,7 @@ def clone_repo_cached( path, remote=None, ref=None,
             if tracking_branch_name in repo.heads:
                 active_branch = repo.heads[tracking_branch_name]
             else:
-                active_branch = repo.create_head('local_tag_branch/'+ref, tracking_ref)
+                active_branch = repo.create_head(tracking_branch_name, tracking_ref)
         elif ref in repo.heads:
             tracking_ref = origin.refs[ref]
             active_branch = repo.heads[ref]
@@ -168,17 +164,16 @@ def clone_repo_cached( path, remote=None, ref=None,
             tracking_ref = origin.refs[ref]
             active_branch = repo.create_head(ref, tracking_ref)
             active_branch.set_tracking_branch(tracking_ref)
-        elif ref in origin.refs:
-            tracking_ref = origin.refs[ref]
-            active_branch = repo.create_head(ref, tracking_ref)
-            active_branch.set_tracking_branch(tracking_ref)
         else:
-            try:
-                tracking_ref = ref
-                tracking_branch_name = 'local_commit_branch/'+ref
-                active_branch = repo.create_head(tracking_branch_name, tracking_ref)
-            except Exception:
-                raise Exception(f"Branch/Tag/Commit \"{ref}\" not found")
+            tracking_ref = ref
+            tracking_branch_name = 'local_commit_branch/'+ref
+            if tracking_branch_name in repo.heads:
+                active_branch = repo.heads[tracking_branch_name]
+            else:
+                try:
+                    active_branch = repo.create_head(tracking_branch_name, tracking_ref)
+                except Exception:
+                    raise Exception(f"Branch/Tag/Commit \"{ref}\" not found")
         print('Active Branch:', active_branch) if DEBUG else None
         print('Tracking Ref:', tracking_ref, '\n') if DEBUG else None
         active_branch.checkout()
@@ -186,6 +181,25 @@ def clone_repo_cached( path, remote=None, ref=None,
     return repo
 
 
+def cache_path(cache, remote):
+    import os.path
+    from urllib.parse import urlparse
+    for url in remote:
+        sane_url = urlparse(url)
+        sane_url = sane_url._replace(
+            path = str(sane_url.path).lstrip('/'),
+            netloc = str(sane_url.netloc).replace('/','_').replace( '@', '_').replace(':', '_')
+        )
+        return os.path.join( cache, sane_url.netloc, sane_url.path)
+
+
+def real_relpath(dest, source='.'):
+    import os.path
+    real_dest = os.path.realpath(dest)
+    real_source = os.path.realpath(source)
+    return os.path.relpath(real_dest, real_source)
+
+
 from enum import Enum
 class AbeSubType(Enum):
     SUBMODULE = 1
@@ -229,14 +243,14 @@ class AbeSubmodule():
         if not self.parent:
             return self.path
         import os.path
-        return os.path.normpath(os.path.join( self.parent.path, self.path))
+        return os.path.normpath(os.path.join( self.parent.fullpath, self.path))
 
 def get_abe_subtree(repo_dir):
     import itertools
     subfile_generators = [
-        get_abe_bsps(repo_dir),
-        get_abe_bootloaders(repo_dir),
         get_abe_submodules(repo_dir),
+        get_abe_bootloaders(repo_dir),
+        get_abe_bsps(repo_dir),
     ]
     return itertools.chain(*subfile_generators)