use threading

multigource.py

@@ -3,6 +3,7 @@
 from argparse import ArgumentParser,Namespace
 import re
 
+GIT_CMD = ["git", "log", "--all", '--pretty=format:user:%aN%n%ct', "--reverse", "--raw", "--encoding=UTF-8", "--no-renames"]
 STMT_CREATE='''
 CREATE TABLE IF NOT EXISTS gourcelog (
     time DATETIME NOT NULL,
@@ -36,41 +37,54 @@ def parser():
     p_join.add_argument('source')
     p_join.add_argument('output')
     p_join.add_argument('-p','--prefix')
+    p_select = sub.add_parser('select')
+    p_select.add_argument('output')
+    p_select.set_defaults(func=select)
     return p
 
 def gitjoin(args: Namespace):
     print(args)
-    import os
-    import subprocess
     import sqlite3
-    GIT_CMD = ["git", "log", "--all", '--pretty=format:user:%aN%n%ct', "--reverse", "--raw", "--encoding=UTF-8", "--no-renames"]
+    import multiprocessing
+    from functools import partial
+
+    pool = multiprocessing.Pool()
 
     with sqlite3.connect(args.output) as db:
         db.execute(STMT_CREATE)
         db.commit()
-        for fp in gitwalk(args.source):
-            p = fp[len(args.source):]
-            proc = subprocess.run(GIT_CMD, cwd=fp, stdout=subprocess.PIPE)
-            if not proc.returncode == 0:
+
+        for fp,gitlog in pool.imap(get_gitlog, gitwalk_rec(args.source)):
+            if fp == None:
                 continue
             print(fp)
-            try:
-                gitlog = proc.stdout.decode()
-            except Exception as ex:
-                print(ex)
-                continue
-            for l in gource_format(gitlog):
-                l[3] = os.path.join(p, l[3].lstrip('/'))
-                db.execute(STMT_INSERT, {
-                    'time': int(l[0]),
-                    'user': l[1],
-                    'action': l[2],
-                    'path': l[3],
-                })
-                # print('|'.join(l))
-        db.commit()
-    for it in db.execute(STMT_SELECT):
-        print(it)
+            p = fp[len(args.source):]
+
+            list(map(partial(sqlite_insert, p=p, db=db), gource_format(gitlog)))
+            db.commit()
+
+def get_gitlog(fp):
+    import subprocess
+    proc = subprocess.run(GIT_CMD, cwd=fp, stdout=subprocess.PIPE)
+    if not proc.returncode == 0:
+        return None,None
+    try:
+        gitlog = proc.stdout.decode(errors='ignore')
+    except Exception as ex:
+        print(ex)
+        return None,None
+    return fp,gitlog
+
+def sqlite_insert(l, p=None, db=None):
+    import os
+    l[3] = os.path.join(p, l[3].lstrip('/'))
+    db.execute(STMT_INSERT, {
+        'time': int(l[0]),
+        'user': l[1],
+        'action': l[2],
+        'path': l[3],
+    })
+    # print('|'.join(l))
 
 def gource_format(inp: str):
     import subprocess
@@ -86,7 +100,7 @@ def gource_format(inp: str):
         yield ls
     # sed "s, \([ACDMRTU]\)\t, \1\t$REL/," | gource  --log-format git --output-custom-log - -
 
-def gitwalk(root: str):
+def gitwalk_rec(root: str):
     import os
     if not os.path.isdir(root):
         return
@@ -96,12 +110,14 @@ def gitwalk(root: str):
         ls = os.listdir(root)
         for d in ls:
             dj = os.path.join(root,d)
-            for p in gitwalk(dj):
+            for p in gitwalk_rec(dj):
                 yield p
 
-def tmpid(key: str):
-    import hashlib
-    return hashlib.sha256(key).hexdigest()
+def select(args: Namespace):
+    import sqlite3
+    with sqlite3.connect(args.output) as db:
+        for it in db.execute(STMT_SELECT):
+            print('|'.join(it))
 
 def render(args: Namespace):
     print(args)