D7835: nodemap: write nodemap data on disk

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
14 messages Options
Reply | Threaded
Open this post in threaded view
|

D7835: nodemap: write nodemap data on disk

martinvonz (Martin von Zweigbergk)
marmoute created this revision.
Herald added a reviewer: indygreg.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REVISION SUMMARY
  Let us start writing data on disk (so that we can read it from there later).
  This series of changeset is going to focus first on having data on disk and
  updating it.
 
  Right now the data is written right next to the revlog data, in the store. We
  might move it to cache (with proper cache validation mechanism) later, but for
  now revlog have a storevfs instance and it is simpler to us it. The right
  location for this data is not the focus of this series.

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=245760, sha256=5dbe62ab98a26668b544063d4d674ac4452ba903ee8895c52fd21d9bbd771e09
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=245760, sha256=5dbe62ab98a26668b544063d4d674ac4452ba903ee8895c52fd21d9bbd771e09
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -21,6 +21,35 @@
         raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+    if revlog.nodemap_file is None:
+        return  # we do not use persistent_nodemap on this revlog
+    callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+    if tr.hasfinalize(callback_id):
+        return  # no need to register again
+    tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+    """Write nodemap data on disk for a given revlog
+    """
+    if getattr(revlog, 'filteredrevs', ()):
+        raise error.ProgrammingError(
+            "cannot persist nodemap of a filtered changelog"
+        )
+    if revlog.nodemap_file is None:
+        msg = "calling persist nodemap on a revlog without the feature enableb"
+        raise error.ProgrammingError(msg)
+    data = persistent_data(revlog.index)
+    # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+    # store vfs
+    with revlog.opener(revlog.nodemap_file, 'w') as f:
+        f.write(data)
+    # EXP-TODO: if the transaction abort, we should remove the new data and
+    # reinstall the old one. (This will be simpler when the file format get a
+    # bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and serialise a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
         mmaplargeindex=False,
         censorable=False,
         upperboundcomp=None,
+        persistentnodemap=False,
     ):
         """
         create a revlog object
@@ -418,6 +419,10 @@
         self.upperboundcomp = upperboundcomp
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + b".d")
+        self.nodemap_file = None
+        if persistentnodemap:
+            self.nodemap_file = indexfile[:-2] + b".n"
+
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
             ifh.write(data[0])
             ifh.write(data[1])
             self._enforceinlinesize(transaction, ifh)
+        nodemaputil.setup_persistent_nodemap(transaction, self)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -929,6 +929,8 @@
 
     if ui.configbool(b'experimental', b'rust.index'):
         options[b'rust.index'] = True
+    if ui.configbool('experimental', 'exp-persistent-nodemap'):
+        options[b'exp-persistent-nodemap'] = True
 
     return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
     b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@
             datafile=datafile,
             checkambig=True,
             mmaplargeindex=True,
+            persistentnodemap=opener.options.get(
+                'exp-persistent-nodemap', False
+            ),
         )
 
         if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1):



To: marmoute, indygreg, #hg-reviewers
Cc: mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7835: nodemap: write nodemap data on disk

martinvonz (Martin von Zweigbergk)
marmoute updated this revision to Diff 19172.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19155&id=19172

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=245760, sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=245760, sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -21,6 +21,39 @@
         raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+    """Install whatever is needed transaction side to persist a nodemap on disk
+
+    (only actually persist the nodemap if this is relevant for this revlog)
+    """
+    if revlog.nodemap_file is None:
+        return  # we do not use persistent_nodemap on this revlog
+    callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+    if tr.hasfinalize(callback_id):
+        return  # no need to register again
+    tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+    """Write nodemap data on disk for a given revlog
+    """
+    if getattr(revlog, 'filteredrevs', ()):
+        raise error.ProgrammingError(
+            "cannot persist nodemap of a filtered changelog"
+        )
+    if revlog.nodemap_file is None:
+        msg = "calling persist nodemap on a revlog without the feature enableb"
+        raise error.ProgrammingError(msg)
+    data = persistent_data(revlog.index)
+    # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+    # store vfs
+    with revlog.opener(revlog.nodemap_file, 'w') as f:
+        f.write(data)
+    # EXP-TODO: if the transaction abort, we should remove the new data and
+    # reinstall the old one. (This will be simpler when the file format get a
+    # bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and serialise a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
         mmaplargeindex=False,
         censorable=False,
         upperboundcomp=None,
+        persistentnodemap=False,
     ):
         """
         create a revlog object
@@ -418,6 +419,10 @@
         self.upperboundcomp = upperboundcomp
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + b".d")
+        self.nodemap_file = None
+        if persistentnodemap:
+            self.nodemap_file = indexfile[:-2] + b".n"
+
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
             ifh.write(data[0])
             ifh.write(data[1])
             self._enforceinlinesize(transaction, ifh)
+        nodemaputil.setup_persistent_nodemap(transaction, self)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -929,6 +929,8 @@
 
     if ui.configbool(b'experimental', b'rust.index'):
         options[b'rust.index'] = True
+    if ui.configbool('experimental', 'exp-persistent-nodemap'):
+        options[b'exp-persistent-nodemap'] = True
 
     return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
     b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@
             datafile=datafile,
             checkambig=True,
             mmaplargeindex=True,
+            persistentnodemap=opener.options.get(
+                'exp-persistent-nodemap', False
+            ),
         )
 
         if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1):



To: marmoute, indygreg, #hg-reviewers
Cc: mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7835: nodemap: write nodemap data on disk

martinvonz (Martin von Zweigbergk)
In reply to this post by martinvonz (Martin von Zweigbergk)
marmoute updated this revision to Diff 19185.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19172&id=19185

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=245760, sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=245760, sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -21,6 +21,39 @@
         raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+    """Install whatever is needed transaction side to persist a nodemap on disk
+
+    (only actually persist the nodemap if this is relevant for this revlog)
+    """
+    if revlog.nodemap_file is None:
+        return  # we do not use persistent_nodemap on this revlog
+    callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+    if tr.hasfinalize(callback_id):
+        return  # no need to register again
+    tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+    """Write nodemap data on disk for a given revlog
+    """
+    if getattr(revlog, 'filteredrevs', ()):
+        raise error.ProgrammingError(
+            "cannot persist nodemap of a filtered changelog"
+        )
+    if revlog.nodemap_file is None:
+        msg = "calling persist nodemap on a revlog without the feature enableb"
+        raise error.ProgrammingError(msg)
+    data = persistent_data(revlog.index)
+    # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+    # store vfs
+    with revlog.opener(revlog.nodemap_file, 'w') as f:
+        f.write(data)
+    # EXP-TODO: if the transaction abort, we should remove the new data and
+    # reinstall the old one. (This will be simpler when the file format get a
+    # bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and serialise a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
         mmaplargeindex=False,
         censorable=False,
         upperboundcomp=None,
+        persistentnodemap=False,
     ):
         """
         create a revlog object
@@ -418,6 +419,10 @@
         self.upperboundcomp = upperboundcomp
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + b".d")
+        self.nodemap_file = None
+        if persistentnodemap:
+            self.nodemap_file = indexfile[:-2] + b".n"
+
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
             ifh.write(data[0])
             ifh.write(data[1])
             self._enforceinlinesize(transaction, ifh)
+        nodemaputil.setup_persistent_nodemap(transaction, self)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -929,6 +929,8 @@
 
     if ui.configbool(b'experimental', b'rust.index'):
         options[b'rust.index'] = True
+    if ui.configbool('experimental', 'exp-persistent-nodemap'):
+        options[b'exp-persistent-nodemap'] = True
 
     return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
     b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@
             datafile=datafile,
             checkambig=True,
             mmaplargeindex=True,
+            persistentnodemap=opener.options.get(
+                'exp-persistent-nodemap', False
+            ),
         )
 
         if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1):



To: marmoute, indygreg, #hg-reviewers
Cc: mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7835: nodemap: write nodemap data on disk

martinvonz (Martin von Zweigbergk)
In reply to this post by martinvonz (Martin von Zweigbergk)
marmoute updated this revision to Diff 19425.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19185&id=19425

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=245760, sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=245760, sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -21,6 +21,39 @@
         raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+    """Install whatever is needed transaction side to persist a nodemap on disk
+
+    (only actually persist the nodemap if this is relevant for this revlog)
+    """
+    if revlog.nodemap_file is None:
+        return  # we do not use persistent_nodemap on this revlog
+    callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+    if tr.hasfinalize(callback_id):
+        return  # no need to register again
+    tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+    """Write nodemap data on disk for a given revlog
+    """
+    if getattr(revlog, 'filteredrevs', ()):
+        raise error.ProgrammingError(
+            "cannot persist nodemap of a filtered changelog"
+        )
+    if revlog.nodemap_file is None:
+        msg = "calling persist nodemap on a revlog without the feature enableb"
+        raise error.ProgrammingError(msg)
+    data = persistent_data(revlog.index)
+    # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+    # store vfs
+    with revlog.opener(revlog.nodemap_file, 'w') as f:
+        f.write(data)
+    # EXP-TODO: if the transaction abort, we should remove the new data and
+    # reinstall the old one. (This will be simpler when the file format get a
+    # bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and serialise a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
         mmaplargeindex=False,
         censorable=False,
         upperboundcomp=None,
+        persistentnodemap=False,
     ):
         """
         create a revlog object
@@ -418,6 +419,10 @@
         self.upperboundcomp = upperboundcomp
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + b".d")
+        self.nodemap_file = None
+        if persistentnodemap:
+            self.nodemap_file = indexfile[:-2] + b".n"
+
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
             ifh.write(data[0])
             ifh.write(data[1])
             self._enforceinlinesize(transaction, ifh)
+        nodemaputil.setup_persistent_nodemap(transaction, self)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -929,6 +929,8 @@
 
     if ui.configbool(b'experimental', b'rust.index'):
         options[b'rust.index'] = True
+    if ui.configbool('experimental', 'exp-persistent-nodemap'):
+        options[b'exp-persistent-nodemap'] = True
 
     return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
     b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@
             datafile=datafile,
             checkambig=True,
             mmaplargeindex=True,
+            persistentnodemap=opener.options.get(
+                'exp-persistent-nodemap', False
+            ),
         )
 
         if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1):



To: marmoute, indygreg, #hg-reviewers
Cc: mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7835: nodemap: write nodemap data on disk

martinvonz (Martin von Zweigbergk)
In reply to this post by martinvonz (Martin von Zweigbergk)
martinvonz added inline comments.

INLINE COMMENTS

> nodemap.py:42
> +        raise error.ProgrammingError(
> +            "cannot persist nodemap of a filtered changelog"
> +        )

need b'' here and below for py3?

> nodemap.py:49
> +    # EXP-TODO: if this is a cache, this should use a cache vfs, not a
> +    # store vfs
> +    with revlog.opener(revlog.nodemap_file, 'w') as f:

and here? this is a series that introduced a lot of new code, so try running all (previously passing) py3 test at least at the end of the series

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

To: marmoute, indygreg, #hg-reviewers
Cc: martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7835: nodemap: write nodemap data on disk

martinvonz (Martin von Zweigbergk)
In reply to this post by martinvonz (Martin von Zweigbergk)
martinvonz added inline comments.

INLINE COMMENTS

> nodemap.py:50-51
> +    # store vfs
> +    with revlog.opener(revlog.nodemap_file, 'w') as f:
> +        f.write(data)
> +    # EXP-TODO: if the transaction abort, we should remove the new data and

Actually, does `revlog.opener.write(revlog.nodemap_file, data)` work?

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

To: marmoute, indygreg, #hg-reviewers
Cc: martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7835: nodemap: write nodemap data on disk

martinvonz (Martin von Zweigbergk)
In reply to this post by martinvonz (Martin von Zweigbergk)
martinvonz added inline comments.

INLINE COMMENTS

> martinvonz wrote in nodemap.py:42
> need b'' here and below for py3?

not needed here, it seems, because `ProgrammingError` converts it for us

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

To: marmoute, indygreg, #hg-reviewers
Cc: martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7835: nodemap: write nodemap data on disk

martinvonz (Martin von Zweigbergk)
In reply to this post by martinvonz (Martin von Zweigbergk)
marmoute added inline comments.

INLINE COMMENTS

> martinvonz wrote in nodemap.py:49
> and here? this is a series that introduced a lot of new code, so try running all (previously passing) py3 test at least at the end of the series

If you can taje the series starting at D8011 <https://phab.mercurial-scm.org/D8011>, if would help me to make sure all angles are covered.

> martinvonz wrote in nodemap.py:50-51
> Actually, does `revlog.opener.write(revlog.nodemap_file, data)` work?

probably, but we will need the context manager approach later in the series so I don't think it is worth updating.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

To: marmoute, indygreg, #hg-reviewers
Cc: martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7835: nodemap: write nodemap data on disk

martinvonz (Martin von Zweigbergk)
In reply to this post by martinvonz (Martin von Zweigbergk)
marmoute updated this revision to Diff 19751.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19425&id=19751

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=245760, sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=245760, sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -21,6 +21,39 @@
         raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+    """Install whatever is needed transaction side to persist a nodemap on disk
+
+    (only actually persist the nodemap if this is relevant for this revlog)
+    """
+    if revlog.nodemap_file is None:
+        return  # we do not use persistent_nodemap on this revlog
+    callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+    if tr.hasfinalize(callback_id):
+        return  # no need to register again
+    tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+    """Write nodemap data on disk for a given revlog
+    """
+    if getattr(revlog, 'filteredrevs', ()):
+        raise error.ProgrammingError(
+            "cannot persist nodemap of a filtered changelog"
+        )
+    if revlog.nodemap_file is None:
+        msg = "calling persist nodemap on a revlog without the feature enableb"
+        raise error.ProgrammingError(msg)
+    data = persistent_data(revlog.index)
+    # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+    # store vfs
+    with revlog.opener(revlog.nodemap_file, 'w') as f:
+        f.write(data)
+    # EXP-TODO: if the transaction abort, we should remove the new data and
+    # reinstall the old one. (This will be simpler when the file format get a
+    # bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
         mmaplargeindex=False,
         censorable=False,
         upperboundcomp=None,
+        persistentnodemap=False,
     ):
         """
         create a revlog object
@@ -418,6 +419,10 @@
         self.upperboundcomp = upperboundcomp
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + b".d")
+        self.nodemap_file = None
+        if persistentnodemap:
+            self.nodemap_file = indexfile[:-2] + b".n"
+
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
             ifh.write(data[0])
             ifh.write(data[1])
             self._enforceinlinesize(transaction, ifh)
+        nodemaputil.setup_persistent_nodemap(transaction, self)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -931,6 +931,8 @@
 
     if ui.configbool(b'experimental', b'rust.index'):
         options[b'rust.index'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
+        options[b'exp-persistent-nodemap'] = True
 
     return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
     b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@
             datafile=datafile,
             checkambig=True,
             mmaplargeindex=True,
+            persistentnodemap=opener.options.get(
+                b'exp-persistent-nodemap', False
+            ),
         )
 
         if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1):



To: marmoute, indygreg, #hg-reviewers
Cc: martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7835: nodemap: write nodemap data on disk

martinvonz (Martin von Zweigbergk)
In reply to this post by martinvonz (Martin von Zweigbergk)
marmoute updated this revision to Diff 19780.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19751&id=19780

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=245760, sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=245760, sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -22,6 +22,39 @@
         raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+    """Install whatever is needed transaction side to persist a nodemap on disk
+
+    (only actually persist the nodemap if this is relevant for this revlog)
+    """
+    if revlog.nodemap_file is None:
+        return  # we do not use persistent_nodemap on this revlog
+    callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+    if tr.hasfinalize(callback_id):
+        return  # no need to register again
+    tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+    """Write nodemap data on disk for a given revlog
+    """
+    if getattr(revlog, 'filteredrevs', ()):
+        raise error.ProgrammingError(
+            "cannot persist nodemap of a filtered changelog"
+        )
+    if revlog.nodemap_file is None:
+        msg = "calling persist nodemap on a revlog without the feature enableb"
+        raise error.ProgrammingError(msg)
+    data = persistent_data(revlog.index)
+    # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+    # store vfs
+    with revlog.opener(revlog.nodemap_file, b'w') as f:
+        f.write(data)
+    # EXP-TODO: if the transaction abort, we should remove the new data and
+    # reinstall the old one. (This will be simpler when the file format get a
+    # bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
         mmaplargeindex=False,
         censorable=False,
         upperboundcomp=None,
+        persistentnodemap=False,
     ):
         """
         create a revlog object
@@ -418,6 +419,10 @@
         self.upperboundcomp = upperboundcomp
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + b".d")
+        self.nodemap_file = None
+        if persistentnodemap:
+            self.nodemap_file = indexfile[:-2] + b".n"
+
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
             ifh.write(data[0])
             ifh.write(data[1])
             self._enforceinlinesize(transaction, ifh)
+        nodemaputil.setup_persistent_nodemap(transaction, self)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -931,6 +931,8 @@
 
     if ui.configbool(b'experimental', b'rust.index'):
         options[b'rust.index'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
+        options[b'exp-persistent-nodemap'] = True
 
     return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
     b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@
             datafile=datafile,
             checkambig=True,
             mmaplargeindex=True,
+            persistentnodemap=opener.options.get(
+                b'exp-persistent-nodemap', False
+            ),
         )
 
         if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1):



To: marmoute, indygreg, #hg-reviewers
Cc: martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7835: nodemap: write nodemap data on disk

martinvonz (Martin von Zweigbergk)
In reply to this post by martinvonz (Martin von Zweigbergk)
marmoute added a comment.
marmoute updated this revision to Diff 19825.


  small doc update on .#s[1]

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19780&id=19825

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=245760, sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=245760, sha256=bc400bf49f11e83bbd25630439feee6628a80a8602d2e38972eac44cc3efe10c
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -22,6 +22,39 @@
         raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+    """Install whatever is needed transaction side to persist a nodemap on disk
+
+    (only actually persist the nodemap if this is relevant for this revlog)
+    """
+    if revlog.nodemap_file is None:
+        return  # we do not use persistent_nodemap on this revlog
+    callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+    if tr.hasfinalize(callback_id):
+        return  # no need to register again
+    tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+    """Write nodemap data on disk for a given revlog
+    """
+    if getattr(revlog, 'filteredrevs', ()):
+        raise error.ProgrammingError(
+            "cannot persist nodemap of a filtered changelog"
+        )
+    if revlog.nodemap_file is None:
+        msg = "calling persist nodemap on a revlog without the feature enableb"
+        raise error.ProgrammingError(msg)
+    data = persistent_data(revlog.index)
+    # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+    # store vfs
+    with revlog.opener(revlog.nodemap_file, b'w') as f:
+        f.write(data)
+    # EXP-TODO: if the transaction abort, we should remove the new data and
+    # reinstall the old one. (This will be simpler when the file format get a
+    # bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
         mmaplargeindex=False,
         censorable=False,
         upperboundcomp=None,
+        persistentnodemap=False,
     ):
         """
         create a revlog object
@@ -418,6 +419,10 @@
         self.upperboundcomp = upperboundcomp
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + b".d")
+        self.nodemap_file = None
+        if persistentnodemap:
+            self.nodemap_file = indexfile[:-2] + b".n"
+
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
             ifh.write(data[0])
             ifh.write(data[1])
             self._enforceinlinesize(transaction, ifh)
+        nodemaputil.setup_persistent_nodemap(transaction, self)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -931,6 +931,8 @@
 
     if ui.configbool(b'experimental', b'rust.index'):
         options[b'rust.index'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
+        options[b'exp-persistent-nodemap'] = True
 
     return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
     b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@
             datafile=datafile,
             checkambig=True,
             mmaplargeindex=True,
+            persistentnodemap=opener.options.get(
+                b'exp-persistent-nodemap', False
+            ),
         )
 
         if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1):



To: marmoute, indygreg, #hg-reviewers
Cc: martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7835: nodemap: write nodemap data on disk

martinvonz (Martin von Zweigbergk)
In reply to this post by martinvonz (Martin von Zweigbergk)
marmoute added a comment.
marmoute updated this revision to Diff 19883.


  rebase to latest default

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19825&id=19883

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff fa c2 ff ff ff ff |................|
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -22,6 +22,39 @@
         raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+    """Install whatever is needed transaction side to persist a nodemap on disk
+
+    (only actually persist the nodemap if this is relevant for this revlog)
+    """
+    if revlog.nodemap_file is None:
+        return  # we do not use persistent_nodemap on this revlog
+    callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+    if tr.hasfinalize(callback_id):
+        return  # no need to register again
+    tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+    """Write nodemap data on disk for a given revlog
+    """
+    if getattr(revlog, 'filteredrevs', ()):
+        raise error.ProgrammingError(
+            "cannot persist nodemap of a filtered changelog"
+        )
+    if revlog.nodemap_file is None:
+        msg = "calling persist nodemap on a revlog without the feature enableb"
+        raise error.ProgrammingError(msg)
+    data = persistent_data(revlog.index)
+    # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+    # store vfs
+    with revlog.opener(revlog.nodemap_file, b'w') as f:
+        f.write(data)
+    # EXP-TODO: if the transaction abort, we should remove the new data and
+    # reinstall the old one. (This will be simpler when the file format get a
+    # bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
         mmaplargeindex=False,
         censorable=False,
         upperboundcomp=None,
+        persistentnodemap=False,
     ):
         """
         create a revlog object
@@ -418,6 +419,10 @@
         self.upperboundcomp = upperboundcomp
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + b".d")
+        self.nodemap_file = None
+        if persistentnodemap:
+            self.nodemap_file = indexfile[:-2] + b".n"
+
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
             ifh.write(data[0])
             ifh.write(data[1])
             self._enforceinlinesize(transaction, ifh)
+        nodemaputil.setup_persistent_nodemap(transaction, self)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -931,6 +931,8 @@
 
     if ui.configbool(b'experimental', b'rust.index'):
         options[b'rust.index'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
+        options[b'exp-persistent-nodemap'] = True
 
     return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
     b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@
             datafile=datafile,
             checkambig=True,
             mmaplargeindex=True,
+            persistentnodemap=opener.options.get(
+                b'exp-persistent-nodemap', False
+            ),
         )
 
         if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1):



To: marmoute, indygreg, #hg-reviewers
Cc: martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7835: nodemap: write nodemap data on disk

martinvonz (Martin von Zweigbergk)
In reply to this post by martinvonz (Martin von Zweigbergk)
marmoute updated this revision to Diff 20016.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=19883&id=20016

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff fa c2 ff ff ff ff |................|
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -22,6 +22,39 @@
         raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+    """Install whatever is needed transaction side to persist a nodemap on disk
+
+    (only actually persist the nodemap if this is relevant for this revlog)
+    """
+    if revlog.nodemap_file is None:
+        return  # we do not use persistent_nodemap on this revlog
+    callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+    if tr.hasfinalize(callback_id):
+        return  # no need to register again
+    tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+    """Write nodemap data on disk for a given revlog
+    """
+    if getattr(revlog, 'filteredrevs', ()):
+        raise error.ProgrammingError(
+            "cannot persist nodemap of a filtered changelog"
+        )
+    if revlog.nodemap_file is None:
+        msg = "calling persist nodemap on a revlog without the feature enableb"
+        raise error.ProgrammingError(msg)
+    data = persistent_data(revlog.index)
+    # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+    # store vfs
+    with revlog.opener(revlog.nodemap_file, b'w') as f:
+        f.write(data)
+    # EXP-TODO: if the transaction abort, we should remove the new data and
+    # reinstall the old one. (This will be simpler when the file format get a
+    # bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
         mmaplargeindex=False,
         censorable=False,
         upperboundcomp=None,
+        persistentnodemap=False,
     ):
         """
         create a revlog object
@@ -418,6 +419,10 @@
         self.upperboundcomp = upperboundcomp
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + b".d")
+        self.nodemap_file = None
+        if persistentnodemap:
+            self.nodemap_file = indexfile[:-2] + b".n"
+
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
             ifh.write(data[0])
             ifh.write(data[1])
             self._enforceinlinesize(transaction, ifh)
+        nodemaputil.setup_persistent_nodemap(transaction, self)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -932,6 +932,8 @@
 
     if ui.configbool(b'experimental', b'rust.index'):
         options[b'rust.index'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
+        options[b'exp-persistent-nodemap'] = True
 
     return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
     b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@
             datafile=datafile,
             checkambig=True,
             mmaplargeindex=True,
+            persistentnodemap=opener.options.get(
+                b'exp-persistent-nodemap', False
+            ),
         )
 
         if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1):



To: marmoute, indygreg, #hg-reviewers
Cc: martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7835: nodemap: write nodemap data on disk

martinvonz (Martin von Zweigbergk)
In reply to this post by martinvonz (Martin von Zweigbergk)
Closed by commit rHG5962fd0d1045: nodemap: write nodemap data on disk (authored by marmoute).
This revision was automatically updated to reflect the committed changes.
This revision was not accepted when it landed; it landed in state "Needs Review".

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7835?vs=20016&id=20103

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7835/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7835

AFFECTED FILES
  mercurial/changelog.py
  mercurial/configitems.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -5,8 +5,14 @@
 
   $ hg init test-repo
   $ cd test-repo
+  $ cat << EOF >> .hg/hgrc
+  > [experimental]
+  > exp-persistent-nodemap=yes
+  > EOF
   $ hg debugbuilddag .+5000
-  $ hg debugnodemap --dump | f --sha256 --bytes=256 --hexdump --size
+  $ hg debugnodemap --dump | f --sha256 --size
+  size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
+  $ f --sha256 --bytes=256 --hexdump --size < .hg/store/00changelog.n
   size=122880, sha256=b961925120e1c9bc345c199b2cc442abc477029fdece37ef9d99cbe59c0558b7
   0000: ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff ff |................|
   0010: ff ff ff ff ff ff ff ff ff ff fa c2 ff ff ff ff |................|
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -22,6 +22,39 @@
         raise error.RevlogError(b'unknown node: %s' % x)
 
 
+def setup_persistent_nodemap(tr, revlog):
+    """Install whatever is needed transaction side to persist a nodemap on disk
+
+    (only actually persist the nodemap if this is relevant for this revlog)
+    """
+    if revlog.nodemap_file is None:
+        return  # we do not use persistent_nodemap on this revlog
+    callback_id = b"revlog-persistent-nodemap-%s" % revlog.nodemap_file
+    if tr.hasfinalize(callback_id):
+        return  # no need to register again
+    tr.addfinalize(callback_id, lambda tr: _persist_nodemap(tr, revlog))
+
+
+def _persist_nodemap(tr, revlog):
+    """Write nodemap data on disk for a given revlog
+    """
+    if getattr(revlog, 'filteredrevs', ()):
+        raise error.ProgrammingError(
+            "cannot persist nodemap of a filtered changelog"
+        )
+    if revlog.nodemap_file is None:
+        msg = "calling persist nodemap on a revlog without the feature enableb"
+        raise error.ProgrammingError(msg)
+    data = persistent_data(revlog.index)
+    # EXP-TODO: if this is a cache, this should use a cache vfs, not a
+    # store vfs
+    with revlog.opener(revlog.nodemap_file, b'w') as f:
+        f.write(data)
+    # EXP-TODO: if the transaction abort, we should remove the new data and
+    # reinstall the old one. (This will be simpler when the file format get a
+    # bit more advanced)
+
+
 ### Nodemap Trie
 #
 # This is a simple reference implementation to compute and persist a nodemap
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -407,6 +407,7 @@
         mmaplargeindex=False,
         censorable=False,
         upperboundcomp=None,
+        persistentnodemap=False,
     ):
         """
         create a revlog object
@@ -418,6 +419,10 @@
         self.upperboundcomp = upperboundcomp
         self.indexfile = indexfile
         self.datafile = datafile or (indexfile[:-2] + b".d")
+        self.nodemap_file = None
+        if persistentnodemap:
+            self.nodemap_file = indexfile[:-2] + b".n"
+
         self.opener = opener
         #  When True, indexfile is opened with checkambig=True at writing, to
         #  avoid file stat ambiguity.
@@ -2286,6 +2291,7 @@
             ifh.write(data[0])
             ifh.write(data[1])
             self._enforceinlinesize(transaction, ifh)
+        nodemaputil.setup_persistent_nodemap(transaction, self)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         """
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -932,6 +932,8 @@
 
     if ui.configbool(b'experimental', b'rust.index'):
         options[b'rust.index'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
+        options[b'exp-persistent-nodemap'] = True
 
     return options
 
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -660,6 +660,9 @@
     b'experimental', b'rust.index', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap', default=False,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(
diff --git a/mercurial/changelog.py b/mercurial/changelog.py
--- a/mercurial/changelog.py
+++ b/mercurial/changelog.py
@@ -385,6 +385,9 @@
             datafile=datafile,
             checkambig=True,
             mmaplargeindex=True,
+            persistentnodemap=opener.options.get(
+                b'exp-persistent-nodemap', False
+            ),
         )
 
         if self._initempty and (self.version & 0xFFFF == revlog.REVLOGV1):



To: marmoute, indygreg, #hg-reviewers
Cc: martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel