Changeset 3185

Timestamp:

02/06/08 08:39:49 (1 year ago)

Author:

hvr

Message:

• changed next revision/previous revision to point to next/previous changeset in a flattened history as provided by git-rev-list --all
  
• implemented in-memory commit tree cache in order to speed up typical Trac repository access patterns (addresses #746)
  
• allow to wrap GitRepository in a CachedRepository, and thus store meta-data in Trac's sql db (addresses #746)
  
• implemented new [git] options to control caching:
  

  [git]
  
  cached_repository = true
  
  persistent_cache = true
  

• various other fixes and cleanups
  

Files:

• gitplugin/0.11/gitplugin/git_fs.py (modified) (14 diffs)
• gitplugin/0.11/gitplugin/PyGIT.py (modified) (9 diffs)

gitplugin/0.11/gitplugin/git_fs.py

@@ -19 +19 @@
-    Changeset, Node, Repository, IRepositoryConnector, NoSuchChangeset, NoSuchNode
-from trac.wiki import IWikiSyntaxProvider
+from trac.versioncontrol.cache import CachedRepository
-from trac.versioncontrol.web_ui import IPropertyRenderer
+from trac.config import _TRUE_VALUES as TRUE
-
-from genshi.builder import tag
@@ -29 +31 @@
-pkg_resources.require('Trac>=0.11dev')
-
-from genshi.builder import tag
-
-import PyGIT
-
-class GitConnector(Component):
-        implements(IRepositoryConnector, IWikiSyntaxProvider, IPropertyRenderer)
+
+        def __init__(self):
+                self._version = None
-
-        def _format_sha_link(self, formatter, ns, sha, label, fullmatch=None):
@@ -53 +56 @@
-
-        def match_property(self, name, mode):
-== 'Parents'
+in ('Parents','Children')
-                        return 8 # default renderer has priority 1
-                return 0
-
-        def render_property(self, name, mode, context, props):
-== 'Parents'
+in ('Parents','Children')
-
-                revs = props[name]
@@ -87 +90 @@
-
-        def get_repository(self, type, dir, authname):
+                """GitRepository factory method"""
+                if not self._version:
+                        self._version = PyGIT.git_version()
+                        self.env.systeminfo.append(('GIT', self._version))
+
-                options = dict(self.config.options(type))
-
+
+
+
+
+
+
+
+
+
+
+
+
-
-class GitRepository(Repository):
-        def __init__(self, path, log, options):
+                self.logger = log
-                self.gitrepo = path
-
+
+
+
+
-                Repository.__init__(self, "git:"+path, None, log)
-
@@ -99 +122 @@
-                self.git = None
-
+        def clear(self, youngest_rev=None):
+                self.youngest = None
+                if youngest_rev is not None:
+                        self.youngest = self.normalize_rev(youngest_rev)
+                self.oldest = None
+
-        def get_youngest_rev(self):
-
+
+
+
+
-
-        def normalize_path(self, path):
@@ -116 +148 @@
-                return self.git.shortrev(self.normalize_rev(rev))
-
-        def get_oldest_rev(self):
-                return ""
-
-        def get_node(self, path, rev=None):
-                #print "get_node", path, rev
@@ -128 +157 @@
-                        return time.mktime(dt.timetuple()) + dt.microsecond/1e6
-
-all
+timerange
-                        yield self.get_changeset(rev)
-
-        def get_changeset(self, rev):
-                """GitChangeset factory method"""
-                #print "get_changeset", rev
-                return GitChangeset(self.git, rev)
-
@@ -142 +170 @@
-
-                for chg in self.git.diff_tree(old_rev, new_rev, self.normalize_path(new_path)):
-                        #print chg
-                        (mode1,mode2,obj1,obj2,action,path) = chg
-
+
-                        if mode2[0] == '1' or mode2[0] == '1':
-                                kind = Node.DIRECTORY
-
-                        if action == 'A':
-                                change = Changeset.ADD
-                        elif action == 'M':
-                                change = Changeset.EDIT
-                        elif action == 'D':
-                                change = Changeset.DELETE
-                        else:
-
+
+
+
-
-                        old_node = None
@@ -168 +190 @@
-
-        def next_rev(self, rev, path=''):
-
-
-
-
+
-
-        def previous_rev(self, rev):
-
-
-
-
+
-
-        def rev_older_than(self, rev1, rev2):
-
-
+
-                return rc
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
-
-class GitNode(Node):
@@ -270 +292 @@
-
-class GitChangeset(Changeset):
+
+        action_map = {
+                'A': Changeset.ADD,
+                'M': Changeset.EDIT,
+                'D': Changeset.DELETE 
+                }
+
-        def __init__(self, git, sha):
-                self.git = git
@@ -279 +308 @@
-
-                committer = props['committer'][0]
+
+                assert 'children' not in props
+                _children = list(git.children(sha))
+                if _children:
+                        props['children'] = _children
+
-                (user,time,tz) = committer.rsplit(None, 2)
-
@@ -288 +323 @@
-                if 'parent' in self.props:
-                        properties['Parents'] = self.props['parent']
+                if 'children' in self.props:
+                        properties['Children'] = self.props['children']
-                if 'committer' in self.props:
-                        properties['git-committer'] = "\n".join(self.props['committer'])
@@ -302 +339 @@
-                prev = self.props.has_key('parent') and self.props['parent'][0] or None
-                for chg in self.git.diff_tree(prev, self.rev):
-                        #print chg
-                        (mode1,mode2,obj1,obj2,action,path) = chg
-                        kind = Node.FILE
@@ -308 +344 @@
-                                kind = Node.DIRECTORY
-
-
-
-
-
-
-
-
-
+
-
-                        yield (path, kind, change, path, prev)

gitplugin/0.11/gitplugin/PyGIT.py

@@ -13 +13 @@
-# GNU General Public License for more details.
-
-
+
+
-#from traceback import print_stack
-
@@ -24 +25 @@
-    pass
-
+def git_version():
+    try:
+        (input, output, error) = os.popen3('git --version')
+        [v] = output.readlines()
+        [a,b,c] = v.strip().split()
+        return c
+    except:
+        raise GitError
+
+class StorageFactory:
+    __dict = weakref.WeakValueDictionary()
+    __dict_nonweak = dict()
+    __dict_lock = threading.Lock()
+
+    def __init__(self, repo, log, weak=True):
+        self.logger = log
+
+        StorageFactory.__dict_lock.acquire()
+
+        try:
+            i = StorageFactory.__dict[repo]
+        except KeyError:
+            i = Storage(repo, log)
+            StorageFactory.__dict[repo] = i
+
+        # create or remove additional reference depending on 'weak' argument
+        if weak:
+            try:
+                del StorageFactory.__dict_nonweak[repo]
+            except KeyError:
+                pass
+        else:
+            StorageFactory.__dict_nonweak[repo] = i
+
+        StorageFactory.__dict_lock.release()
+
+        self.__inst = i
+        self.__repo = repo
+
+    def getInstance(self):
+        is_weak = self.__repo not in StorageFactory.__dict_nonweak
+        self.logger.debug("requested %sPyGIT.Storage instance %d for '%s'"
+                          % (("","weak ")[is_weak], id(self.__inst), self.__repo))
+        return self.__inst
+
-class Storage:
-
+
+
+
+
-        self.repo = repo
-        self.commit_encoding = None
+
+        self._lock = threading.Lock()
+        self.last_youngest_rev = -1
+        self._invalidate_caches()
+
+    def __del__(self):
+        self.logger.debug("PyGIT.Storage instance %d destructed" % id(self))
+
+    def _invalidate_caches(self,youngest_rev=None):
+        self._lock.acquire()
+
+        rc = False
+
+        if self.last_youngest_rev != youngest_rev:
+            self.logger.debug("invalidated caches (%s != %s)" % (self.last_youngest_rev, youngest_rev))
+            rc = True
+            self._commit_db = None
+            self._oldest_rev = None
+            self.last_youngest_rev = None
+
+        self._lock.release()
+        return rc
+
+    def get_commits(self):
+        self._lock.acquire()
+        if self._commit_db is None:
+            self.logger.debug("triggered rebuild of commit tree db for %d" % id(self))
+            new_db = {}
+            parent = None
+            youngest = None
+            ord_rev = 0
+            for revs in self._git_call_f("git-rev-list --parents --all").readlines():
+                revs = revs.strip().split()
+
+                rev = revs[0]
+                parents = set(revs[1:])
+
+                ord_rev += 1
+
+                if not youngest:
+                    youngest = rev
+
+                # new_db[rev] = (children(rev), parents(rev), ordinal_id(rev))
+                if new_db.has_key(rev):
+                    _children,_parents,_ord_rev = new_db[rev]
+                    assert _children
+                    assert not _parents
+                    assert _ord_rev == 0
+                    new_db[rev] = (_children, parents, ord_rev)
+                else:
+                    new_db[rev] = (set(), parents, ord_rev)
+
+                # update all parents(rev)'s children
+                for parent in parents:
+                    if new_db.has_key(parent):
+                        new_db[parent][0].add(rev)
+                    else:
+                        new_db[parent] = (set([rev]), set(), 0) # dummy ordinal_id
+
+            self._commit_db = new_db, parent
+            self.last_youngest_rev = youngest
+            self.logger.debug("rebuilt commit tree db for %d with %d entries" % (id(self),len(new_db)))
+
+        self._lock.release()
+
+        assert self._commit_db[1] is not None
+        assert self._commit_db[0] is not None
+
+        return self._commit_db[0]
+
+    def sync(self):
+        rev = self._git_call("git-rev-list -n1 --all").strip()
+        return self._invalidate_caches(rev)
+
+    def oldest_rev(self):
+        self.get_commits() # trigger commit tree db build
+        return self._commit_db[1]
+        #return self._git_call("git-rev-list --reverse --all | head -1").strip()
+
+    def youngest_rev(self):
+        self.get_commits() # trigger commit tree db build
+        return self.last_youngest_rev
+
+    def history_relative_rev(self, sha, rel_pos):
+        db = self.get_commits()
+
+        if sha not in db:
+            raise GitErrorSha
+
+        if rel_pos == 0:
+            return sha
+
+        lin_rev = db[sha][2] + rel_pos
+
+        if lin_rev < 1 or lin_rev > len(db):
+            return None
+
+        for k,v in db.iteritems():
+            if v[2] == lin_rev:
+                return k
+
+        # should never be reached if db is consistent
+        raise GitError
+
+    def hist_next_revision(self, sha):
+        return self.history_relative_rev(sha, -1)
+
+    def hist_prev_revision(self, sha):
+        return self.history_relative_rev(sha, +1)
-
-    def _git_call_f(self,cmd):
@@ -38 +196 @@
-
-        if _profile_git_calls:
-
+ # doesn't work actually, as popen3 runs async
-            print >>sys.stderr, "GIT: took %6.2fs for '%s'" % (t, cmd)
-            pass
@@ -54 +212 @@
-        return self.commit_encoding
-
+
-    def head(self):
-        "get current HEAD commit id"
@@ -60 +219 @@
-    def verifyrev(self,rev):
-        "verify/lookup given revision object and return a sha id or None if lookup failed"
+
+        db = self.get_commits()
+        if db.has_key(rev):
+            return rev
+
-        rc=self._git_call("git-rev-parse --verify '%s'" % rev).strip()
-        if len(rc)==0:
@@ -86 +250 @@
-
-    def read_commit(self, sha):
+        db = self.get_commits()
+        if sha not in db:
+            self.logger.info("read_commit failed for '%s'" % sha)
+            raise GitErrorSha
+
-        raw = self._git_call("git-cat-file commit "+sha)
-        raw = unicode(raw, self.get_commit_encoding(), 'replace')
@@ -110 +279 @@
-        return int(self._git_call("git-cat-file -s "+sha).strip())
-
+    def children(self, sha):
+        db = self.get_commits()
+
+        try:
+            return list(db[sha][0])
+        except KeyError:
+            return []
+
+    def children_recursive(self, sha):
+        db = self.get_commits()
+
+        work_list = deque()
+        seen = set()
+
+        seen.update(db[sha][0])
+        work_list.extend(db[sha][0])
+
+        while work_list:
+            p = work_list.popleft()
+            yield p
+
+            #_children = db[p][0]
+            _children = db[p][0] - seen
+
+            seen.update(_children)
+            work_list.extend(_children)
+
+        assert len(work_list) == 0
+
-    def parents(self, sha):
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
-
-    def history(self, sha, path, limit=None, skip=0):
@@ -133 +326 @@
-            yield rev.strip()
-
-
+
+
+
+
-        for rev in self._git_call_f("git-rev-list --reverse --max-age=%d --min-age=%d --all" \
-                                        % (start,stop)).readlines():
-            yield rev.strip()
-
-
+
+
-        rev1 = rev1.strip()
-        rev2 = rev2.strip()
-
-
-
-
+
-
-    def last_change(self, sha, path):
@@ -172 +366 @@
-    print g.read_commit(g.head())
-    print "--------------"
-
-
+
+
+
+
+
-    print "--------------"
-    print g.get_commit_encoding()
-    print "--------------"
-    print g.get_branches()
+    print "--------------"
+    print g.hist_prev_revision(g.oldest_rev()), g.oldest_rev(), g.hist_next_revision(g.oldest_rev())
+
+    print "--------------"
+    p = g.youngest_rev()
+    print g.hist_prev_revision(p), p, g.hist_next_revision(p)
+    print "--------------"
+    p = g.head()
+    for i in range(-5,5):
+        print i, g.history_relative_rev(p, i)
+
+    # check for loops
+    def check4loops(head):
+        print "check4loops", head
+        seen = set([head])
+        for sha in g.children_recursive(head):
+            if sha in seen:
+                print "dupe detected :-/", sha, len(seen)
+                #print seen
+                #break
+            seen.add(sha)
+        return seen
+
+    print len(check4loops(g.parents(g.head())[0]))
+
+    #print len(check4loops(g.oldest_rev()))
+
+    #print len(list(g.children_recursive(g.oldest_rev())))