D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
19 messages Options
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
marmoute created this revision.
Herald added a subscriber: mercurial-devel.
Herald added a reviewer: hg-reviewers.

REVISION SUMMARY
  The performance and memory benefit is much greater if we don't have to copy all
  the data in memory for each information. So we introduce an option (on by
  default) to read the data using mmap.
 
  This changeset is the last one definition the API for index support nodemap
  data. (they have to be able to use the mmaping).

REPOSITORY
  rHG Mercurial

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

AFFECTED FILES
  mercurial/configitems.py
  mercurial/debugcommands.py
  mercurial/localrepo.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -84,3 +84,37 @@
   $ hg debugnodemap --check
   revision in index:   5002
   revision in nodemap: 5002
+
+Test code path without mmap
+---------------------------
+
+  $ echo bar > bar
+  $ hg add bar
+  $ hg ci -m 'bar' --config experimental.exp-persistent-nodemap.mmap=no
+
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=yes
+  revision in index:   5003
+  revision in nodemap: 5003
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=no
+  revision in index:   5003
+  revision in nodemap: 5003
+
+
+#if pure
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 246656
+  data-unused: 768
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=246656, sha256=8221807a0860a7a65002d2d3e0d33512d28aa6db2433db966e56aa17dcf6329f (glob)
+
+#else
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 245888
+  data-unused: 0
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=245888, sha256=42233b63e5567fd362fb7847fa7a9f4d40ad93c28f8571197b356a69fe8bd271 (glob)
+#endif
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -8,6 +8,7 @@
 
 from __future__ import absolute_import
 
+import errno
 import os
 import re
 import struct
@@ -46,10 +47,17 @@
 
     filename = _rawdata_filepath(revlog, docket)
     data = revlog.opener.tryread(filename)
+    try:
+        with revlog.opener(filename) as fd:
+            if revlog.opener.options.get("exp-persistent-nodemap.mmap"):
+                data = fd.read(data_length)
+            else:
+                data = util.buffer(util.mmapread(fd, data_length))
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
     if len(data) < data_length:
         return None
-    elif len(data) > data_length:
-        data = data[:data_length]
     return docket, data
 
 
@@ -81,6 +89,8 @@
 
     can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
     ondisk_docket = revlog._nodemap_docket
+    feed_data = util.safehasattr(revlog.index, "update_nodemap_data")
+    use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
 
     data = None
     # first attemp an incremental update of the data
@@ -97,12 +107,18 @@
             datafile = _rawdata_filepath(revlog, target_docket)
             # EXP-TODO: if this is a cache, this should use a cache vfs, not a
             # store vfs
+            new_length = target_docket.data_length + len(data)
             with revlog.opener(datafile, 'r+') as fd:
                 fd.seek(target_docket.data_length)
                 fd.write(data)
-                fd.seek(0)
-                new_data = fd.read(target_docket.data_length + len(data))
-            target_docket.data_length += len(data)
+                if feed_data:
+                    if use_mmap:
+                        fd.seek(0)
+                        new_data = fd.read(new_length)
+                    else:
+                        fd.flush()
+                        new_data = util.buffer(util.mmapread(fd, new_length))
+            target_docket.data_length = new_length
             target_docket.data_unused += data_changed_count
 
     if data is None:
@@ -115,9 +131,14 @@
             data = persistent_data(revlog.index)
         # EXP-TODO: if this is a cache, this should use a cache vfs, not a
         # store vfs
-        new_data = data
-        with revlog.opener(datafile, 'w') as fd:
+        with revlog.opener(datafile, 'w+') as fd:
             fd.write(data)
+            if feed_data:
+                if use_mmap:
+                    new_data = data
+                else:
+                    fd.flush()
+                    new_data = util.buffer(util.mmapread(fd, len(data)))
         target_docket.data_length = len(data)
     target_docket.tip_rev = revlog.tiprev()
     # EXP-TODO: if this is a cache, this should use a cache vfs, not a
@@ -125,7 +146,7 @@
     with revlog.opener(revlog.nodemap_file, 'w', atomictemp=True) as fp:
         fp.write(target_docket.serialize())
     revlog._nodemap_docket = target_docket
-    if util.safehasattr(revlog.index, "update_nodemap_data"):
+    if feed_data:
         revlog.index.update_nodemap_data(target_docket, new_data)
 
     # EXP-TODO: if the transaction abort, we should remove the new data and
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -931,6 +931,8 @@
         options[b'rust.index'] = True
     if ui.configbool('experimental', 'exp-persistent-nodemap'):
         options[b'exp-persistent-nodemap'] = True
+    if ui.configbool('experimental', 'exp-persistent-nodemap.mmap'):
+        options[b'exp-persistent-nodemap.mmap'] = True
     if ui.configbool('devel', 'persistent-nodemap'):
         options[b'devel-force-nodemap'] = True
 
diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py
--- a/mercurial/debugcommands.py
+++ b/mercurial/debugcommands.py
@@ -2115,7 +2115,7 @@
         nm_data = nodemap.persisted_data(cl)
         if nm_data is not None:
             docket, data = nm_data
-            ui.write(data)
+            ui.write(data[:])
     elif args['check']:
         unfi = repo.unfiltered()
         cl = unfi.changelog
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -666,6 +666,9 @@
     b'experimental', b'exp-persistent-nodemap', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap.mmap', default=True,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(



To: marmoute, #hg-reviewers
Cc: mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
martinvonz added a comment.


  How much does this patch help performance?
 
  I would also like to see performance numbers (even just rough ones) for the Rust version. Sorry about a possibly stupid question, but why will this on-disk nodemap be faster than building it from the index? Is it that the file is smaller and thus faster to read? Or is it more the building of the tree than the reading that's slow? You mentioned you use some private repo for testing this. How large is the `00changelog.n` file in that repo and how large is `00changelog.i`?

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

To: marmoute, #hg-reviewers
Cc: martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
marmoute added a comment.
marmoute added a subscriber: gracinet.


  In D7894#116397 <https://phab.mercurial-scm.org/D7894#116397>, @martinvonz wrote:
 
  > How much does this patch help performance?
  > I would also like to see performance numbers (even just rough ones) for the Rust version. Sorry about a possibly stupid question, but why will this on-disk nodemap be faster than building it from the index? Is it that the file is smaller and thus faster to read? Or is it more the building of the tree than the reading that's slow? You mentioned you use some private repo for testing this. How large is the `00changelog.n` file in that repo and how large is `00changelog.i`?
 
  This save hundreds of milli second at initialization of large repositories. The repositories we are looking at are about 2 millions revisions. (but this will help smaller repository too). Mozilla try is a public repository in that range. It 00changelog.i is 103MB
 
  The information in the .i files is just a flat list of node. So anything that need a mapping needs to build it. Building a mapping for millions of revision is slow. (I think Georges mentionned 300ms to build the mozilla-try nodemap). The nodemap we write on disk is directly usage as such. So we just need to mmap the files (mostly instant is the repository have been busy recently, eg: on server) and directly query the data from disk. We will get more data once the latest version of this series and the latest version of the Rust series (from @gracinet) are connected again. But this greatly boost many operation (small operation, discovery, etc).

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

To: marmoute, #hg-reviewers
Cc: gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
martinvonz added a comment.


  In D7894#116407 <https://phab.mercurial-scm.org/D7894#116407>, @marmoute wrote:
 
  > In D7894#116397 <https://phab.mercurial-scm.org/D7894#116397>, @martinvonz wrote:
  >
  >> How much does this patch help performance?
  >> I would also like to see performance numbers (even just rough ones) for the Rust version. Sorry about a possibly stupid question, but why will this on-disk nodemap be faster than building it from the index? Is it that the file is smaller and thus faster to read? Or is it more the building of the tree than the reading that's slow? You mentioned you use some private repo for testing this. How large is the `00changelog.n` file in that repo and how large is `00changelog.i`?
  >
  > This save hundreds of milli second at initialization of large repositories. The repositories we are looking at are about 2 millions revisions. (but this will help smaller repository too). Mozilla try is a public repository in that range. It 00changelog.i is 103MB
 
  And 00changelog.n?
 
  > The information in the .i files is just a flat list of node. So anything that need a mapping needs to build it. Building a mapping for millions of revision is slow. (I think Georges mentionned 300ms to build the mozilla-try nodemap). The nodemap we write on disk is directly usage as such. So we just need to mmap the files (mostly instant is the repository have been busy recently, eg: on server) and directly query the data from disk.
 
  Ah, that's what I was wondering. I was wondering while reviewing this series if your plan was to lazily from disk but I didn't see any mention of that. I guess this mmap business should have been that a hint :)

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

To: marmoute, #hg-reviewers
Cc: gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
marmoute added a comment.


  In D7894#116540 <https://phab.mercurial-scm.org/D7894#116540>, @martinvonz wrote:
 
  > In D7894#116407 <https://phab.mercurial-scm.org/D7894#116407>, @marmoute wrote:
  >
  >> In D7894#116397 <https://phab.mercurial-scm.org/D7894#116397>, @martinvonz wrote:
  >>
  >>> How much does this patch help performance?
  >>> I would also like to see performance numbers (even just rough ones) for the Rust version. Sorry about a possibly stupid question, but why will this on-disk nodemap be faster than building it from the index? Is it that the file is smaller and thus faster to read? Or is it more the building of the tree than the reading that's slow? You mentioned you use some private repo for testing this. How large is the `00changelog.n` file in that repo and how large is `00changelog.i`?
  >>
  >> This save hundreds of milli second at initialization of large repositories. The repositories we are looking at are about 2 millions revisions. (but this will help smaller repository too). Mozilla try is a public repository in that range. It 00changelog.i is 103MB
  >
  > And 00changelog.n?
 
 
 
    106973952 bytes for changelog.i
     83123200 bytes for the nodemap rawfiles
 
 
 
  >> The information in the .i files is just a flat list of node. So anything that need a mapping needs to build it. Building a mapping for millions of revision is slow. (I think Georges mentionned 300ms to build the mozilla-try nodemap). The nodemap we write on disk is directly usage as such. So we just need to mmap the files (mostly instant is the repository have been busy recently, eg: on server) and directly query the data from disk.
  >
  > Ah, that's what I was wondering. I was wondering while reviewing this series if your plan was to lazily from disk but I didn't see any mention of that. I guess this mmap business should have been that a hint :)
 
  Yes, so everythign we are doing is not so really "serialization" since we never actualy "deserialize" it in practice.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

To: marmoute, #hg-reviewers
Cc: gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
marmoute updated this revision to Diff 19435.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7894?vs=19312&id=19435

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

AFFECTED FILES
  mercurial/configitems.py
  mercurial/debugcommands.py
  mercurial/localrepo.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -84,3 +84,37 @@
   $ hg debugnodemap --check
   revision in index:   5002
   revision in nodemap: 5002
+
+Test code path without mmap
+---------------------------
+
+  $ echo bar > bar
+  $ hg add bar
+  $ hg ci -m 'bar' --config experimental.exp-persistent-nodemap.mmap=no
+
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=yes
+  revision in index:   5003
+  revision in nodemap: 5003
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=no
+  revision in index:   5003
+  revision in nodemap: 5003
+
+
+#if pure
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 246656
+  data-unused: 768
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=246656, sha256=8221807a0860a7a65002d2d3e0d33512d28aa6db2433db966e56aa17dcf6329f (glob)
+
+#else
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 245888
+  data-unused: 0
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=245888, sha256=42233b63e5567fd362fb7847fa7a9f4d40ad93c28f8571197b356a69fe8bd271 (glob)
+#endif
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -8,6 +8,7 @@
 
 from __future__ import absolute_import
 
+import errno
 import os
 import re
 import struct
@@ -46,10 +47,17 @@
 
     filename = _rawdata_filepath(revlog, docket)
     data = revlog.opener.tryread(filename)
+    try:
+        with revlog.opener(filename) as fd:
+            if revlog.opener.options.get("exp-persistent-nodemap.mmap"):
+                data = fd.read(data_length)
+            else:
+                data = util.buffer(util.mmapread(fd, data_length))
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
     if len(data) < data_length:
         return None
-    elif len(data) > data_length:
-        data = data[:data_length]
     return docket, data
 
 
@@ -81,6 +89,8 @@
 
     can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
     ondisk_docket = revlog._nodemap_docket
+    feed_data = util.safehasattr(revlog.index, "update_nodemap_data")
+    use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
 
     data = None
     # first attemp an incremental update of the data
@@ -97,12 +107,18 @@
             datafile = _rawdata_filepath(revlog, target_docket)
             # EXP-TODO: if this is a cache, this should use a cache vfs, not a
             # store vfs
+            new_length = target_docket.data_length + len(data)
             with revlog.opener(datafile, 'r+') as fd:
                 fd.seek(target_docket.data_length)
                 fd.write(data)
-                fd.seek(0)
-                new_data = fd.read(target_docket.data_length + len(data))
-            target_docket.data_length += len(data)
+                if feed_data:
+                    if use_mmap:
+                        fd.seek(0)
+                        new_data = fd.read(new_length)
+                    else:
+                        fd.flush()
+                        new_data = util.buffer(util.mmapread(fd, new_length))
+            target_docket.data_length = new_length
             target_docket.data_unused += data_changed_count
 
     if data is None:
@@ -115,9 +131,14 @@
             data = persistent_data(revlog.index)
         # EXP-TODO: if this is a cache, this should use a cache vfs, not a
         # store vfs
-        new_data = data
-        with revlog.opener(datafile, 'w') as fd:
+        with revlog.opener(datafile, 'w+') as fd:
             fd.write(data)
+            if feed_data:
+                if use_mmap:
+                    new_data = data
+                else:
+                    fd.flush()
+                    new_data = util.buffer(util.mmapread(fd, len(data)))
         target_docket.data_length = len(data)
     target_docket.tip_rev = revlog.tiprev()
     # EXP-TODO: if this is a cache, this should use a cache vfs, not a
@@ -125,7 +146,7 @@
     with revlog.opener(revlog.nodemap_file, 'w', atomictemp=True) as fp:
         fp.write(target_docket.serialize())
     revlog._nodemap_docket = target_docket
-    if util.safehasattr(revlog.index, "update_nodemap_data"):
+    if feed_data:
         revlog.index.update_nodemap_data(target_docket, new_data)
 
     # EXP-TODO: if the transaction abort, we should remove the new data and
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -931,6 +931,8 @@
         options[b'rust.index'] = True
     if ui.configbool('experimental', 'exp-persistent-nodemap'):
         options[b'exp-persistent-nodemap'] = True
+    if ui.configbool('experimental', 'exp-persistent-nodemap.mmap'):
+        options[b'exp-persistent-nodemap.mmap'] = True
     if ui.configbool('devel', 'persistent-nodemap'):
         options[b'devel-force-nodemap'] = True
 
diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py
--- a/mercurial/debugcommands.py
+++ b/mercurial/debugcommands.py
@@ -2115,7 +2115,7 @@
         nm_data = nodemap.persisted_data(cl)
         if nm_data is not None:
             docket, data = nm_data
-            ui.write(data)
+            ui.write(data[:])
     elif opts['check']:
         unfi = repo.unfiltered()
         cl = unfi.changelog
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -666,6 +666,9 @@
     b'experimental', b'exp-persistent-nodemap', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap.mmap', default=True,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(



To: marmoute, #hg-reviewers
Cc: gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
marmoute added a comment.


  hooo, I am a bot

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

To: marmoute, #hg-reviewers
Cc: gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
marmoute updated this revision to Diff 19775.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7894?vs=19435&id=19775

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

AFFECTED FILES
  mercurial/configitems.py
  mercurial/debugcommands.py
  mercurial/localrepo.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -84,3 +84,37 @@
   $ hg debugnodemap --check
   revision in index:   5002
   revision in nodemap: 5002
+
+Test code path without mmap
+---------------------------
+
+  $ echo bar > bar
+  $ hg add bar
+  $ hg ci -m 'bar' --config experimental.exp-persistent-nodemap.mmap=no
+
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=yes
+  revision in index:   5003
+  revision in nodemap: 5003
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=no
+  revision in index:   5003
+  revision in nodemap: 5003
+
+
+#if pure
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 246656
+  data-unused: 768
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=246656, sha256=8221807a0860a7a65002d2d3e0d33512d28aa6db2433db966e56aa17dcf6329f (glob)
+
+#else
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 245888
+  data-unused: 0
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=245888, sha256=42233b63e5567fd362fb7847fa7a9f4d40ad93c28f8571197b356a69fe8bd271 (glob)
+#endif
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -8,6 +8,7 @@
 
 from __future__ import absolute_import
 
+import errno
 import os
 import re
 import struct
@@ -46,10 +47,17 @@
 
     filename = _rawdata_filepath(revlog, docket)
     data = revlog.opener.tryread(filename)
+    try:
+        with revlog.opener(filename) as fd:
+            if revlog.opener.options.get(b"exp-persistent-nodemap.mmap"):
+                data = fd.read(data_length)
+            else:
+                data = util.buffer(util.mmapread(fd, data_length))
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
     if len(data) < data_length:
         return None
-    elif len(data) > data_length:
-        data = data[:data_length]
     return docket, data
 
 
@@ -81,6 +89,8 @@
 
     can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
     ondisk_docket = revlog._nodemap_docket
+    feed_data = util.safehasattr(revlog.index, "update_nodemap_data")
+    use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
 
     data = None
     # first attemp an incremental update of the data
@@ -97,12 +107,18 @@
             datafile = _rawdata_filepath(revlog, target_docket)
             # EXP-TODO: if this is a cache, this should use a cache vfs, not a
             # store vfs
+            new_length = target_docket.data_length + len(data)
             with revlog.opener(datafile, 'r+') as fd:
                 fd.seek(target_docket.data_length)
                 fd.write(data)
-                fd.seek(0)
-                new_data = fd.read(target_docket.data_length + len(data))
-            target_docket.data_length += len(data)
+                if feed_data:
+                    if use_mmap:
+                        fd.seek(0)
+                        new_data = fd.read(new_length)
+                    else:
+                        fd.flush()
+                        new_data = util.buffer(util.mmapread(fd, new_length))
+            target_docket.data_length = new_length
             target_docket.data_unused += data_changed_count
 
     if data is None:
@@ -115,9 +131,14 @@
             data = persistent_data(revlog.index)
         # EXP-TODO: if this is a cache, this should use a cache vfs, not a
         # store vfs
-        new_data = data
-        with revlog.opener(datafile, 'w') as fd:
+        with revlog.opener(datafile, 'w+') as fd:
             fd.write(data)
+            if feed_data:
+                if use_mmap:
+                    new_data = data
+                else:
+                    fd.flush()
+                    new_data = util.buffer(util.mmapread(fd, len(data)))
         target_docket.data_length = len(data)
     target_docket.tip_rev = revlog.tiprev()
     # EXP-TODO: if this is a cache, this should use a cache vfs, not a
@@ -125,7 +146,7 @@
     with revlog.opener(revlog.nodemap_file, 'w', atomictemp=True) as fp:
         fp.write(target_docket.serialize())
     revlog._nodemap_docket = target_docket
-    if util.safehasattr(revlog.index, "update_nodemap_data"):
+    if feed_data:
         revlog.index.update_nodemap_data(target_docket, new_data)
 
     # EXP-TODO: if the transaction abort, we should remove the new data and
@@ -284,7 +305,7 @@
     changed_block, trie = _update_trie(index, root, last_rev)
     return (
         changed_block * S_BLOCK.size,
-        _persist_block(trie, existing_idx=max_idx),
+        _persist_trie(trie, existing_idx=max_idx),
     )
 
 
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -933,6 +933,8 @@
         options[b'rust.index'] = True
     if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
         options[b'exp-persistent-nodemap'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap.mmap'):
+        options[b'exp-persistent-nodemap.mmap'] = True
     if ui.configbool(b'devel', b'persistent-nodemap'):
         options[b'devel-force-nodemap'] = True
 
diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py
--- a/mercurial/debugcommands.py
+++ b/mercurial/debugcommands.py
@@ -2121,7 +2121,7 @@
         nm_data = nodemap.persisted_data(cl)
         if nm_data is not None:
             docket, data = nm_data
-            ui.write(data)
+            ui.write(data[:])
     elif opts['check']:
         unfi = repo.unfiltered()
         cl = unfi.changelog
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -666,6 +666,9 @@
     b'experimental', b'exp-persistent-nodemap', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap.mmap', default=True,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(



To: marmoute, #hg-reviewers
Cc: gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
marmoute updated this revision to Diff 19802.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7894?vs=19775&id=19802

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

AFFECTED FILES
  mercurial/configitems.py
  mercurial/debugcommands.py
  mercurial/localrepo.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -84,3 +84,37 @@
   $ hg debugnodemap --check
   revision in index:   5002
   revision in nodemap: 5002
+
+Test code path without mmap
+---------------------------
+
+  $ echo bar > bar
+  $ hg add bar
+  $ hg ci -m 'bar' --config experimental.exp-persistent-nodemap.mmap=no
+
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=yes
+  revision in index:   5003
+  revision in nodemap: 5003
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=no
+  revision in index:   5003
+  revision in nodemap: 5003
+
+
+#if pure
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 246656
+  data-unused: 768
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=246656, sha256=8221807a0860a7a65002d2d3e0d33512d28aa6db2433db966e56aa17dcf6329f (glob)
+
+#else
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 245888
+  data-unused: 0
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=245888, sha256=42233b63e5567fd362fb7847fa7a9f4d40ad93c28f8571197b356a69fe8bd271 (glob)
+#endif
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -8,6 +8,7 @@
 
 from __future__ import absolute_import
 
+import errno
 import os
 import re
 import struct
@@ -46,10 +47,17 @@
 
     filename = _rawdata_filepath(revlog, docket)
     data = revlog.opener.tryread(filename)
+    try:
+        with revlog.opener(filename) as fd:
+            if revlog.opener.options.get(b"exp-persistent-nodemap.mmap"):
+                data = fd.read(data_length)
+            else:
+                data = util.buffer(util.mmapread(fd, data_length))
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
     if len(data) < data_length:
         return None
-    elif len(data) > data_length:
-        data = data[:data_length]
     return docket, data
 
 
@@ -81,6 +89,8 @@
 
     can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
     ondisk_docket = revlog._nodemap_docket
+    feed_data = util.safehasattr(revlog.index, "update_nodemap_data")
+    use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
 
     data = None
     # first attemp an incremental update of the data
@@ -97,12 +107,18 @@
             datafile = _rawdata_filepath(revlog, target_docket)
             # EXP-TODO: if this is a cache, this should use a cache vfs, not a
             # store vfs
+            new_length = target_docket.data_length + len(data)
             with revlog.opener(datafile, b'r+') as fd:
                 fd.seek(target_docket.data_length)
                 fd.write(data)
-                fd.seek(0)
-                new_data = fd.read(target_docket.data_length + len(data))
-            target_docket.data_length += len(data)
+                if feed_data:
+                    if use_mmap:
+                        fd.seek(0)
+                        new_data = fd.read(new_length)
+                    else:
+                        fd.flush()
+                        new_data = util.buffer(util.mmapread(fd, new_length))
+            target_docket.data_length = new_length
             target_docket.data_unused += data_changed_count
 
     if data is None:
@@ -115,9 +131,14 @@
             data = persistent_data(revlog.index)
         # EXP-TODO: if this is a cache, this should use a cache vfs, not a
         # store vfs
-        new_data = data
-        with revlog.opener(datafile, b'w') as fd:
+        with revlog.opener(datafile, b'w+') as fd:
             fd.write(data)
+            if feed_data:
+                if use_mmap:
+                    new_data = data
+                else:
+                    fd.flush()
+                    new_data = util.buffer(util.mmapread(fd, len(data)))
         target_docket.data_length = len(data)
     target_docket.tip_rev = revlog.tiprev()
     # EXP-TODO: if this is a cache, this should use a cache vfs, not a
@@ -125,7 +146,7 @@
     with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp:
         fp.write(target_docket.serialize())
     revlog._nodemap_docket = target_docket
-    if util.safehasattr(revlog.index, "update_nodemap_data"):
+    if feed_data:
         revlog.index.update_nodemap_data(target_docket, new_data)
 
     # EXP-TODO: if the transaction abort, we should remove the new data and
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -933,6 +933,8 @@
         options[b'rust.index'] = True
     if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
         options[b'exp-persistent-nodemap'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap.mmap'):
+        options[b'exp-persistent-nodemap.mmap'] = True
     if ui.configbool(b'devel', b'persistent-nodemap'):
         options[b'devel-force-nodemap'] = True
 
diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py
--- a/mercurial/debugcommands.py
+++ b/mercurial/debugcommands.py
@@ -2121,7 +2121,7 @@
         nm_data = nodemap.persisted_data(cl)
         if nm_data is not None:
             docket, data = nm_data
-            ui.write(data)
+            ui.write(data[:])
     elif opts['check']:
         unfi = repo.unfiltered()
         cl = unfi.changelog
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -666,6 +666,9 @@
     b'experimental', b'exp-persistent-nodemap', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap.mmap', default=True,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(



To: marmoute, #hg-reviewers
Cc: gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
marmoute added a comment.
marmoute updated this revision to Diff 19846.


  small doc update on .#s[1]

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7894?vs=19802&id=19846

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

AFFECTED FILES
  mercurial/configitems.py
  mercurial/debugcommands.py
  mercurial/localrepo.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -84,3 +84,37 @@
   $ hg debugnodemap --check
   revision in index:   5002
   revision in nodemap: 5002
+
+Test code path without mmap
+---------------------------
+
+  $ echo bar > bar
+  $ hg add bar
+  $ hg ci -m 'bar' --config experimental.exp-persistent-nodemap.mmap=no
+
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=yes
+  revision in index:   5003
+  revision in nodemap: 5003
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=no
+  revision in index:   5003
+  revision in nodemap: 5003
+
+
+#if pure
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 246656
+  data-unused: 768
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=246656, sha256=8221807a0860a7a65002d2d3e0d33512d28aa6db2433db966e56aa17dcf6329f (glob)
+
+#else
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 245888
+  data-unused: 0
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=245888, sha256=42233b63e5567fd362fb7847fa7a9f4d40ad93c28f8571197b356a69fe8bd271 (glob)
+#endif
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -8,6 +8,7 @@
 
 from __future__ import absolute_import
 
+import errno
 import os
 import re
 import struct
@@ -46,10 +47,17 @@
 
     filename = _rawdata_filepath(revlog, docket)
     data = revlog.opener.tryread(filename)
+    try:
+        with revlog.opener(filename) as fd:
+            if revlog.opener.options.get(b"exp-persistent-nodemap.mmap"):
+                data = fd.read(data_length)
+            else:
+                data = util.buffer(util.mmapread(fd, data_length))
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
     if len(data) < data_length:
         return None
-    elif len(data) > data_length:
-        data = data[:data_length]
     return docket, data
 
 
@@ -81,6 +89,8 @@
 
     can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
     ondisk_docket = revlog._nodemap_docket
+    feed_data = util.safehasattr(revlog.index, "update_nodemap_data")
+    use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
 
     data = None
     # first attemp an incremental update of the data
@@ -97,12 +107,18 @@
             datafile = _rawdata_filepath(revlog, target_docket)
             # EXP-TODO: if this is a cache, this should use a cache vfs, not a
             # store vfs
+            new_length = target_docket.data_length + len(data)
             with revlog.opener(datafile, b'r+') as fd:
                 fd.seek(target_docket.data_length)
                 fd.write(data)
-                fd.seek(0)
-                new_data = fd.read(target_docket.data_length + len(data))
-            target_docket.data_length += len(data)
+                if feed_data:
+                    if use_mmap:
+                        fd.seek(0)
+                        new_data = fd.read(new_length)
+                    else:
+                        fd.flush()
+                        new_data = util.buffer(util.mmapread(fd, new_length))
+            target_docket.data_length = new_length
             target_docket.data_unused += data_changed_count
 
     if data is None:
@@ -115,9 +131,14 @@
             data = persistent_data(revlog.index)
         # EXP-TODO: if this is a cache, this should use a cache vfs, not a
         # store vfs
-        new_data = data
-        with revlog.opener(datafile, b'w') as fd:
+        with revlog.opener(datafile, b'w+') as fd:
             fd.write(data)
+            if feed_data:
+                if use_mmap:
+                    new_data = data
+                else:
+                    fd.flush()
+                    new_data = util.buffer(util.mmapread(fd, len(data)))
         target_docket.data_length = len(data)
     target_docket.tip_rev = revlog.tiprev()
     # EXP-TODO: if this is a cache, this should use a cache vfs, not a
@@ -125,7 +146,7 @@
     with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp:
         fp.write(target_docket.serialize())
     revlog._nodemap_docket = target_docket
-    if util.safehasattr(revlog.index, "update_nodemap_data"):
+    if feed_data:
         revlog.index.update_nodemap_data(target_docket, new_data)
 
     # EXP-TODO: if the transaction abort, we should remove the new data and
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -933,6 +933,8 @@
         options[b'rust.index'] = True
     if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
         options[b'exp-persistent-nodemap'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap.mmap'):
+        options[b'exp-persistent-nodemap.mmap'] = True
     if ui.configbool(b'devel', b'persistent-nodemap'):
         options[b'devel-force-nodemap'] = True
 
diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py
--- a/mercurial/debugcommands.py
+++ b/mercurial/debugcommands.py
@@ -2121,7 +2121,7 @@
         nm_data = nodemap.persisted_data(cl)
         if nm_data is not None:
             docket, data = nm_data
-            ui.write(data)
+            ui.write(data[:])
     elif opts['check']:
         unfi = repo.unfiltered()
         cl = unfi.changelog
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -666,6 +666,9 @@
     b'experimental', b'exp-persistent-nodemap', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap.mmap', default=True,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(



To: marmoute, #hg-reviewers
Cc: gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
marmoute added a comment.
marmoute updated this revision to Diff 19906.


  rebase to latest default

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7894?vs=19846&id=19906

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

AFFECTED FILES
  mercurial/configitems.py
  mercurial/debugcommands.py
  mercurial/localrepo.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -84,3 +84,37 @@
   $ hg debugnodemap --check
   revision in index:   5002
   revision in nodemap: 5002
+
+Test code path without mmap
+---------------------------
+
+  $ echo bar > bar
+  $ hg add bar
+  $ hg ci -m 'bar' --config experimental.exp-persistent-nodemap.mmap=no
+
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=yes
+  revision in index:   5003
+  revision in nodemap: 5003
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=no
+  revision in index:   5003
+  revision in nodemap: 5003
+
+
+#if pure
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 123328
+  data-unused: 384
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=123328, sha256=10d26e9776b6596af0f89143a54eba8cc581e929c38242a02a7b0760698c6c70 (glob)
+
+#else
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 122944
+  data-unused: 0
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=122944, sha256=755976b22b64ab680401b45395953504e64e7fa8c31ac570f58dee21e15f9bc0 (glob)
+#endif
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -8,6 +8,7 @@
 
 from __future__ import absolute_import
 
+import errno
 import os
 import re
 import struct
@@ -46,10 +47,17 @@
 
     filename = _rawdata_filepath(revlog, docket)
     data = revlog.opener.tryread(filename)
+    try:
+        with revlog.opener(filename) as fd:
+            if revlog.opener.options.get(b"exp-persistent-nodemap.mmap"):
+                data = fd.read(data_length)
+            else:
+                data = util.buffer(util.mmapread(fd, data_length))
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
     if len(data) < data_length:
         return None
-    elif len(data) > data_length:
-        data = data[:data_length]
     return docket, data
 
 
@@ -81,6 +89,8 @@
 
     can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
     ondisk_docket = revlog._nodemap_docket
+    feed_data = util.safehasattr(revlog.index, "update_nodemap_data")
+    use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
 
     data = None
     # first attemp an incremental update of the data
@@ -97,12 +107,18 @@
             datafile = _rawdata_filepath(revlog, target_docket)
             # EXP-TODO: if this is a cache, this should use a cache vfs, not a
             # store vfs
+            new_length = target_docket.data_length + len(data)
             with revlog.opener(datafile, b'r+') as fd:
                 fd.seek(target_docket.data_length)
                 fd.write(data)
-                fd.seek(0)
-                new_data = fd.read(target_docket.data_length + len(data))
-            target_docket.data_length += len(data)
+                if feed_data:
+                    if use_mmap:
+                        fd.seek(0)
+                        new_data = fd.read(new_length)
+                    else:
+                        fd.flush()
+                        new_data = util.buffer(util.mmapread(fd, new_length))
+            target_docket.data_length = new_length
             target_docket.data_unused += data_changed_count
 
     if data is None:
@@ -115,9 +131,14 @@
             data = persistent_data(revlog.index)
         # EXP-TODO: if this is a cache, this should use a cache vfs, not a
         # store vfs
-        new_data = data
-        with revlog.opener(datafile, b'w') as fd:
+        with revlog.opener(datafile, b'w+') as fd:
             fd.write(data)
+            if feed_data:
+                if use_mmap:
+                    new_data = data
+                else:
+                    fd.flush()
+                    new_data = util.buffer(util.mmapread(fd, len(data)))
         target_docket.data_length = len(data)
     target_docket.tip_rev = revlog.tiprev()
     # EXP-TODO: if this is a cache, this should use a cache vfs, not a
@@ -125,7 +146,7 @@
     with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp:
         fp.write(target_docket.serialize())
     revlog._nodemap_docket = target_docket
-    if util.safehasattr(revlog.index, "update_nodemap_data"):
+    if feed_data:
         revlog.index.update_nodemap_data(target_docket, new_data)
 
     # EXP-TODO: if the transaction abort, we should remove the new data and
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -933,6 +933,8 @@
         options[b'rust.index'] = True
     if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
         options[b'exp-persistent-nodemap'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap.mmap'):
+        options[b'exp-persistent-nodemap.mmap'] = True
     if ui.configbool(b'devel', b'persistent-nodemap'):
         options[b'devel-force-nodemap'] = True
 
diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py
--- a/mercurial/debugcommands.py
+++ b/mercurial/debugcommands.py
@@ -2121,7 +2121,7 @@
         nm_data = nodemap.persisted_data(cl)
         if nm_data is not None:
             docket, data = nm_data
-            ui.write(data)
+            ui.write(data[:])
     elif opts['check']:
         unfi = repo.unfiltered()
         cl = unfi.changelog
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -666,6 +666,9 @@
     b'experimental', b'exp-persistent-nodemap', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap.mmap', default=True,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(



To: marmoute, #hg-reviewers
Cc: gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
marmoute updated this revision to Diff 20018.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7894?vs=19906&id=20018

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

AFFECTED FILES
  mercurial/configitems.py
  mercurial/debugcommands.py
  mercurial/localrepo.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -84,3 +84,37 @@
   $ hg debugnodemap --check
   revision in index:   5002
   revision in nodemap: 5002
+
+Test code path without mmap
+---------------------------
+
+  $ echo bar > bar
+  $ hg add bar
+  $ hg ci -m 'bar' --config experimental.exp-persistent-nodemap.mmap=no
+
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=yes
+  revision in index:   5003
+  revision in nodemap: 5003
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=no
+  revision in index:   5003
+  revision in nodemap: 5003
+
+
+#if pure
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 123328
+  data-unused: 384
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=123328, sha256=10d26e9776b6596af0f89143a54eba8cc581e929c38242a02a7b0760698c6c70 (glob)
+
+#else
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 122944
+  data-unused: 0
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=122944, sha256=755976b22b64ab680401b45395953504e64e7fa8c31ac570f58dee21e15f9bc0 (glob)
+#endif
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -8,6 +8,7 @@
 
 from __future__ import absolute_import
 
+import errno
 import os
 import re
 import struct
@@ -46,10 +47,17 @@
 
     filename = _rawdata_filepath(revlog, docket)
     data = revlog.opener.tryread(filename)
+    try:
+        with revlog.opener(filename) as fd:
+            if revlog.opener.options.get(b"exp-persistent-nodemap.mmap"):
+                data = fd.read(data_length)
+            else:
+                data = util.buffer(util.mmapread(fd, data_length))
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
     if len(data) < data_length:
         return None
-    elif len(data) > data_length:
-        data = data[:data_length]
     return docket, data
 
 
@@ -81,6 +89,8 @@
 
     can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
     ondisk_docket = revlog._nodemap_docket
+    feed_data = util.safehasattr(revlog.index, "update_nodemap_data")
+    use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
 
     data = None
     # first attemp an incremental update of the data
@@ -97,12 +107,18 @@
             datafile = _rawdata_filepath(revlog, target_docket)
             # EXP-TODO: if this is a cache, this should use a cache vfs, not a
             # store vfs
+            new_length = target_docket.data_length + len(data)
             with revlog.opener(datafile, b'r+') as fd:
                 fd.seek(target_docket.data_length)
                 fd.write(data)
-                fd.seek(0)
-                new_data = fd.read(target_docket.data_length + len(data))
-            target_docket.data_length += len(data)
+                if feed_data:
+                    if use_mmap:
+                        fd.seek(0)
+                        new_data = fd.read(new_length)
+                    else:
+                        fd.flush()
+                        new_data = util.buffer(util.mmapread(fd, new_length))
+            target_docket.data_length = new_length
             target_docket.data_unused += data_changed_count
 
     if data is None:
@@ -115,9 +131,14 @@
             data = persistent_data(revlog.index)
         # EXP-TODO: if this is a cache, this should use a cache vfs, not a
         # store vfs
-        new_data = data
-        with revlog.opener(datafile, b'w') as fd:
+        with revlog.opener(datafile, b'w+') as fd:
             fd.write(data)
+            if feed_data:
+                if use_mmap:
+                    new_data = data
+                else:
+                    fd.flush()
+                    new_data = util.buffer(util.mmapread(fd, len(data)))
         target_docket.data_length = len(data)
     target_docket.tip_rev = revlog.tiprev()
     # EXP-TODO: if this is a cache, this should use a cache vfs, not a
@@ -125,7 +146,7 @@
     with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp:
         fp.write(target_docket.serialize())
     revlog._nodemap_docket = target_docket
-    if util.safehasattr(revlog.index, "update_nodemap_data"):
+    if feed_data:
         revlog.index.update_nodemap_data(target_docket, new_data)
 
     # EXP-TODO: if the transaction abort, we should remove the new data and
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -934,6 +934,8 @@
         options[b'rust.index'] = True
     if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
         options[b'exp-persistent-nodemap'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap.mmap'):
+        options[b'exp-persistent-nodemap.mmap'] = True
     if ui.configbool(b'devel', b'persistent-nodemap'):
         options[b'devel-force-nodemap'] = True
 
diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py
--- a/mercurial/debugcommands.py
+++ b/mercurial/debugcommands.py
@@ -2121,7 +2121,7 @@
         nm_data = nodemap.persisted_data(cl)
         if nm_data is not None:
             docket, data = nm_data
-            ui.write(data)
+            ui.write(data[:])
     elif opts['check']:
         unfi = repo.unfiltered()
         cl = unfi.changelog
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -666,6 +666,9 @@
     b'experimental', b'exp-persistent-nodemap', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap.mmap', default=True,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(



To: marmoute, #hg-reviewers
Cc: gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
marmoute added a comment.


  Here are quick performance number on our mozilla-try reference:
 
  For `perfindex --rev tip` I now get
 
    c:    ! wall 0.000067 comb 0.000000 user 0.000000 sys 0.000000 (median of 36546)
    rust: ! wall 0.000197 comb 0.000000 user 0.000000 sys 0.000000 (median of 13090)
 
  For `perfindex --rev '-10000: + 0'` I now get
 
    c:    ! wall 0.567621 comb 0.570000 user 0.560000 sys 0.010000 (median of 18)
    rust: ! wall 0.011586 comb 0.000000 user 0.000000 sys 0.000000 (median of 254)
 
  More throughful benchmarking in progress.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

To: marmoute, #hg-reviewers
Cc: gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
marmoute added a comment.


  More throughful benchmarking has arrived.
 
  This is a large win all over the board. The win for the larger repository very significant.
 
    +         3.9e-05          7.1e-05     1.82  internal.index.no_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, False) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         3.8e-05            7e-05     1.84  internal.index.no_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, True) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         3.7e-05            7e-05     1.89  internal.index.no_lookup.track_time('mercurial-2018-08-01', 'zstd', 'default', True, True, True, True, True) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.002178         0.000111     0.05  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, False, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         6.4e-05          9.8e-05     1.53  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, False, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000561          7.9e-05     0.14  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, False, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.003858         0.000108     0.03  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, False, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.003848           0.0001     0.03  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, False, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         6.3e-05          7.8e-05     1.24  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, False, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.002184          0.00011     0.05  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, True, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         6.6e-05          9.9e-05     1.50  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, True, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000567            8e-05     0.14  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, True, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -         0.00386         0.000106     0.03  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, True, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.003892          9.9e-05     0.03  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, True, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         6.2e-05          7.7e-05     1.24  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, True, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.002187         0.000108     0.05  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zstd', 'default', True, True, True, True, True, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         6.5e-05          9.7e-05     1.49  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zstd', 'default', True, True, True, True, True, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000561          7.9e-05     0.14  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zstd', 'default', True, True, True, True, True, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.003846          0.00011     0.03  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zstd', 'default', True, True, True, True, True, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.003854          9.9e-05     0.03  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zstd', 'default', True, True, True, True, True, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         6.3e-05          7.8e-05     1.24  internal.index.small_lookup.track_time('mercurial-2018-08-01', 'zstd', 'default', True, True, True, True, True, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.006298         0.004007     0.64  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, False, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.006272         0.003874     0.62  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, False, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.004634         0.003636     0.78  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, False, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.004998          0.00365     0.73  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, False, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.007712         0.003678     0.48  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, False, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.006313         0.003732     0.59  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, True, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -         0.00629         0.003769     0.60  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, True, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.004632         0.003654     0.79  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, True, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.005032         0.003735     0.74  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, True, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.007575          0.00379     0.50  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, True, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.006367         0.003707     0.58  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zstd', 'default', True, True, True, True, True, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.006252         0.003716     0.59  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zstd', 'default', True, True, True, True, True, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.004609         0.003648     0.79  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zstd', 'default', True, True, True, True, True, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.005023         0.003715     0.74  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zstd', 'default', True, True, True, True, True, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.007644         0.003778     0.49  internal.index.large_lookup.track_time('mercurial-2018-08-01', 'zstd', 'default', True, True, True, True, True, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
   
    +         3.8e-05          7.3e-05     1.92  internal.index.no_lookup.track_time('mercurial-filtered-2019-11-22', 'zstd', 'default', True, True, True, True, True) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.008179         0.000131     0.02  internal.index.small_lookup.track_time('mercurial-filtered-2019-11-22', 'zstd', 'default', True, True, True, True, True, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
             0.000106         0.000115     1.08  internal.index.small_lookup.track_time('mercurial-filtered-2019-11-22', 'zstd', 'default', True, True, True, True, True, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.001654          8.3e-05     0.05  internal.index.small_lookup.track_time('mercurial-filtered-2019-11-22', 'zstd', 'default', True, True, True, True, True, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.013476         0.000134     0.01  internal.index.small_lookup.track_time('mercurial-filtered-2019-11-22', 'zstd', 'default', True, True, True, True, True, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.013184         0.000112     0.01  internal.index.small_lookup.track_time('mercurial-filtered-2019-11-22', 'zstd', 'default', True, True, True, True, True, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000102            8e-05     0.78  internal.index.small_lookup.track_time('mercurial-filtered-2019-11-22', 'zstd', 'default', True, True, True, True, True, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.012199         0.004018     0.33  internal.index.large_lookup.track_time('mercurial-filtered-2019-11-22', 'zstd', 'default', True, True, True, True, True, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.011962         0.003782     0.32  internal.index.large_lookup.track_time('mercurial-filtered-2019-11-22', 'zstd', 'default', True, True, True, True, True, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.004641         0.004017     0.87  internal.index.large_lookup.track_time('mercurial-filtered-2019-11-22', 'zstd', 'default', True, True, True, True, True, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.006088          0.00382     0.63  internal.index.large_lookup.track_time('mercurial-filtered-2019-11-22', 'zstd', 'default', True, True, True, True, True, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -         0.01875         0.003775     0.20  internal.index.large_lookup.track_time('mercurial-filtered-2019-11-22', 'zstd', 'default', True, True, True, True, True, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
   
    +         3.9e-05          7.2e-05     1.85  internal.index.no_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, False) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         3.9e-05          7.3e-05     1.87  internal.index.no_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, True) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         3.8e-05          7.1e-05     1.87  internal.index.no_lookup.track_time('pypy-2018-08-01', 'zstd', 'default', True, True, True, True, True) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.006601          0.00013     0.02  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, False, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         9.6e-05          0.00011     1.15  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, False, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.001394          7.9e-05     0.06  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, False, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.010903          0.00013     0.01  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, False, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.010888         0.000115     0.01  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, False, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
              7.4e-05            8e-05     1.08  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, False, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.006601         0.000129     0.02  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, True, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         9.7e-05         0.000108     1.11  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, True, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -          0.0014            8e-05     0.06  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, True, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.010844         0.000128     0.01  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, True, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.010786         0.000106     0.01  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, True, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -         9.3e-05            8e-05     0.86  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, True, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.006475         0.000131     0.02  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zstd', 'default', True, True, True, True, True, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         9.6e-05          0.00011     1.15  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zstd', 'default', True, True, True, True, True, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.001397          8.2e-05     0.06  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zstd', 'default', True, True, True, True, True, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.010938         0.000133     0.01  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zstd', 'default', True, True, True, True, True, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.010665         0.000106     0.01  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zstd', 'default', True, True, True, True, True, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         6.5e-05          8.2e-05     1.26  internal.index.small_lookup.track_time('pypy-2018-08-01', 'zstd', 'default', True, True, True, True, True, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.010683         0.003773     0.35  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, False, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.010673         0.003735     0.35  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, False, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -         0.00463          0.00382     0.83  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, False, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.005921         0.003757     0.63  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, False, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.014701         0.004016     0.27  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, False, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.010648         0.003816     0.36  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, True, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.010652         0.003766     0.35  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, True, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.004969         0.003773     0.76  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, True, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.005925         0.003782     0.64  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, True, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -         0.01466         0.003777     0.26  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zlib', 'default', True, True, True, True, True, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.010633          0.00386     0.36  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zstd', 'default', True, True, True, True, True, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.010727         0.003785     0.35  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zstd', 'default', True, True, True, True, True, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.004668         0.003688     0.79  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zstd', 'default', True, True, True, True, True, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.005846         0.004064     0.70  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zstd', 'default', True, True, True, True, True, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.014876          0.00384     0.26  internal.index.large_lookup.track_time('pypy-2018-08-01', 'zstd', 'default', True, True, True, True, True, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
   
    +         3.7e-05          7.5e-05     2.03  internal.index.no_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, False) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         3.8e-05            7e-05     1.84  internal.index.no_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, True) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         3.9e-05          7.2e-05     1.85  internal.index.no_lookup.track_time('netbeans-2018-08-01', 'zstd', 'default', True, True, True, True, True) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -         0.02847         0.000145     0.01  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, False, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000239         0.000116     0.49  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, False, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.004983            8e-05     0.02  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, False, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.042669         0.000143     0.00  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, False, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.042118         0.000117     0.00  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, False, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000244          8.3e-05     0.34  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, False, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.027695         0.000146     0.01  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, True, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000168         0.000117     0.70  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, True, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.004967          8.2e-05     0.02  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, True, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.042328         0.000144     0.00  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, True, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.042587         0.000119     0.00  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, True, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000238          8.1e-05     0.34  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, True, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.028557         0.000145     0.01  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zstd', 'default', True, True, True, True, True, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000165          0.00012     0.73  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zstd', 'default', True, True, True, True, True, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.004941          8.4e-05     0.02  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zstd', 'default', True, True, True, True, True, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.042682          0.00014     0.00  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zstd', 'default', True, True, True, True, True, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.042114         0.000118     0.00  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zstd', 'default', True, True, True, True, True, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000236          8.1e-05     0.34  internal.index.small_lookup.track_time('netbeans-2018-08-01', 'zstd', 'default', True, True, True, True, True, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.032823         0.004066     0.12  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, False, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.031798         0.003967     0.12  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, False, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.004849         0.003907     0.81  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, False, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.009529         0.004034     0.42  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, False, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.046217         0.004013     0.09  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, False, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.031647         0.004058     0.13  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, True, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.032553         0.004017     0.12  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, True, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.004805         0.004136     0.86  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, True, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.009515         0.004009     0.42  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, True, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.046954         0.003983     0.08  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zlib', 'default', True, True, True, True, True, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -         0.03245         0.004078     0.13  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zstd', 'default', True, True, True, True, True, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.031685         0.003989     0.13  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zstd', 'default', True, True, True, True, True, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -         0.00484         0.003889     0.80  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zstd', 'default', True, True, True, True, True, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.009477         0.004019     0.42  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zstd', 'default', True, True, True, True, True, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -         0.04654         0.004023     0.09  internal.index.large_lookup.track_time('netbeans-2018-08-01', 'zstd', 'default', True, True, True, True, True, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
   
    +         4.1e-05          7.3e-05     1.78  internal.index.no_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, False) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         3.7e-05          7.1e-05     1.92  internal.index.no_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, True) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         3.9e-05            7e-05     1.79  internal.index.no_lookup.track_time('mozilla-central-2018-08-01', 'zstd', 'default', True, True, True, True, True) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.041165         0.000153     0.00  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, False, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000166         0.000119     0.72  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, False, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.006994          8.1e-05     0.01  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, False, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.060687         0.000157     0.00  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, False, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.061438         0.000118     0.00  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, False, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000275          8.1e-05     0.29  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, False, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.039947         0.000154     0.00  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, True, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000164         0.000119     0.73  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, True, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.007001          8.3e-05     0.01  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, True, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -         0.06026         0.000156     0.00  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, True, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.061074         0.000121     0.00  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, True, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000275          8.3e-05     0.30  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, True, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.040962         0.000153     0.00  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zstd', 'default', True, True, True, True, True, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000164         0.000121     0.74  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zstd', 'default', True, True, True, True, True, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.007109          8.5e-05     0.01  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zstd', 'default', True, True, True, True, True, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.061388         0.000152     0.00  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zstd', 'default', True, True, True, True, True, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -         0.06147         0.000125     0.00  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zstd', 'default', True, True, True, True, True, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.000274          8.2e-05     0.30  internal.index.small_lookup.track_time('mozilla-central-2018-08-01', 'zstd', 'default', True, True, True, True, True, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.045977         0.004112     0.09  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, False, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.045198         0.004046     0.09  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, False, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.004805         0.004118     0.86  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, False, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.011548         0.004232     0.37  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, False, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.066339          0.00408     0.06  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, False, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.046125         0.004158     0.09  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, True, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.046382         0.004132     0.09  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, True, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.004815          0.00402     0.83  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, True, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.011542         0.004066     0.35  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, True, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.066069         0.004116     0.06  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zlib', 'default', True, True, True, True, True, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.044966         0.004142     0.09  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zstd', 'default', True, True, True, True, True, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.045773         0.004113     0.09  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zstd', 'default', True, True, True, True, True, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -         0.00484         0.004036     0.83  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zstd', 'default', True, True, True, True, True, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.011581         0.004036     0.35  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zstd', 'default', True, True, True, True, True, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.065447         0.004214     0.06  internal.index.large_lookup.track_time('mozilla-central-2018-08-01', 'zstd', 'default', True, True, True, True, True, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
   
    +         3.7e-05          7.1e-05     1.92  internal.index.no_lookup.track_time('mozilla-try-2019-02-18', 'zlib', 'default', True, True, True, True, True) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         3.8e-05          7.3e-05     1.92  internal.index.no_lookup.track_time('mozilla-try-2019-02-18', 'zstd', 'default', True, True, True, True, True) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.236488         0.000174     0.00  internal.index.small_lookup.track_time('mozilla-try-2019-02-18', 'zlib', 'default', True, True, True, True, True, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +           5e-05         0.000132     2.64  internal.index.small_lookup.track_time('mozilla-try-2019-02-18', 'zlib', 'default', True, True, True, True, True, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.023564          8.6e-05     0.00  internal.index.small_lookup.track_time('mozilla-try-2019-02-18', 'zlib', 'default', True, True, True, True, True, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.315712         0.000178     0.00  internal.index.small_lookup.track_time('mozilla-try-2019-02-18', 'zlib', 'default', True, True, True, True, True, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.315888         0.000141     0.00  internal.index.small_lookup.track_time('mozilla-try-2019-02-18', 'zlib', 'default', True, True, True, True, True, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         4.6e-05          8.4e-05     1.83  internal.index.small_lookup.track_time('mozilla-try-2019-02-18', 'zlib', 'default', True, True, True, True, True, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.232883          0.00018     0.00  internal.index.small_lookup.track_time('mozilla-try-2019-02-18', 'zstd', 'default', True, True, True, True, True, '(-10:) + :9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +           5e-05         0.000132     2.64  internal.index.small_lookup.track_time('mozilla-try-2019-02-18', 'zstd', 'default', True, True, True, True, True, '-10:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.023521          8.4e-05     0.00  internal.index.small_lookup.track_time('mozilla-try-2019-02-18', 'zstd', 'default', True, True, True, True, True, '0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.313372         0.000179     0.00  internal.index.small_lookup.track_time('mozilla-try-2019-02-18', 'zstd', 'default', True, True, True, True, True, ':9 + (-10:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -         0.31645         0.000143     0.00  internal.index.small_lookup.track_time('mozilla-try-2019-02-18', 'zstd', 'default', True, True, True, True, True, ':9') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +         5.2e-05          8.5e-05     1.63  internal.index.small_lookup.track_time('mozilla-try-2019-02-18', 'zstd', 'default', True, True, True, True, True, 'tip') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.242474         0.005356     0.02  internal.index.large_lookup.track_time('mozilla-try-2019-02-18', 'zlib', 'default', True, True, True, True, True, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.242892         0.005314     0.02  internal.index.large_lookup.track_time('mozilla-try-2019-02-18', 'zlib', 'default', True, True, True, True, True, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +        0.004802         0.005326     1.11  internal.index.large_lookup.track_time('mozilla-try-2019-02-18', 'zlib', 'default', True, True, True, True, True, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.028061         0.005319     0.19  internal.index.large_lookup.track_time('mozilla-try-2019-02-18', 'zlib', 'default', True, True, True, True, True, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.318157         0.005362     0.02  internal.index.large_lookup.track_time('mozilla-try-2019-02-18', 'zlib', 'default', True, True, True, True, True, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.238735         0.005358     0.02  internal.index.large_lookup.track_time('mozilla-try-2019-02-18', 'zstd', 'default', True, True, True, True, True, '(-10000:) + (:99)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.235539         0.005308     0.02  internal.index.large_lookup.track_time('mozilla-try-2019-02-18', 'zstd', 'default', True, True, True, True, True, '(-10000:) + 0') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    +        0.004622         0.005532     1.20  internal.index.large_lookup.track_time('mozilla-try-2019-02-18', 'zstd', 'default', True, True, True, True, True, '-10000:') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.028222         0.005337     0.19  internal.index.large_lookup.track_time('mozilla-try-2019-02-18', 'zstd', 'default', True, True, True, True, True, '0 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
    -        0.317942         0.005593     0.02  internal.index.large_lookup.track_time('mozilla-try-2019-02-18', 'zstd', 'default', True, True, True, True, True, ':99 + (-10000:)') [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

To: marmoute, #hg-reviewers
Cc: gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
This revision now requires changes to proceed.
durin42 added a comment.
durin42 requested changes to this revision.


  In D7894#120247 <https://phab.mercurial-scm.org/D7894#120247>, @marmoute wrote:
 
  > More throughful benchmarking has arrived.
  > This is a large win all over the board. The win for the larger repository very significant.
  >
  >   +         3.9e-05          7.1e-05     1.82  internal.index.no_lookup.track_time('mercurial-2018-08-01', 'zlib', 'default', True, True, True, True, False) [citrea/virtualenv-py2.7-pyyaml-HGMODULEPOLICYrust+c-HGWITHRUSTEXTcpython]
 
  [...]
 
  >  
 
  I literally have no idea what this wall of text is trying to tell me. Some column headers would be a baseline to making this readable, structuring it as a table somehow would be even better, and for actual archaeology it would have been put in some summarized form in the commit message.
 
  I've reviewed up to here and I'm basically happy, but I'll hold this one until the commit message has at least some useful summary of where we're going performance wise.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

To: marmoute, #hg-reviewers, durin42
Cc: durin42, gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
Herald added a reviewer: indygreg.
marmoute updated this revision to Diff 20165.

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7894?vs=20018&id=20165

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

AFFECTED FILES
  mercurial/configitems.py
  mercurial/debugcommands.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -84,3 +84,37 @@
   $ hg debugnodemap --check
   revision in index:   5002
   revision in nodemap: 5002
+
+Test code path without mmap
+---------------------------
+
+  $ echo bar > bar
+  $ hg add bar
+  $ hg ci -m 'bar' --config experimental.exp-persistent-nodemap.mmap=no
+
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=yes
+  revision in index:   5003
+  revision in nodemap: 5003
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=no
+  revision in index:   5003
+  revision in nodemap: 5003
+
+
+#if pure
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 123328
+  data-unused: 384
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=123328, sha256=10d26e9776b6596af0f89143a54eba8cc581e929c38242a02a7b0760698c6c70 (glob)
+
+#else
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 122944
+  data-unused: 0
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=122944, sha256=755976b22b64ab680401b45395953504e64e7fa8c31ac570f58dee21e15f9bc0 (glob)
+#endif
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -8,6 +8,7 @@
 
 from __future__ import absolute_import
 
+import errno
 import os
 import re
 import struct
@@ -45,11 +46,18 @@
     docket.data_unused = data_unused
 
     filename = _rawdata_filepath(revlog, docket)
-    data = revlog.opener.tryread(filename)
+    use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
+    try:
+        with revlog.opener(filename) as fd:
+            if use_mmap:
+                data = util.buffer(util.mmapread(fd, data_length))
+            else:
+                data = fd.read(data_length)
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
     if len(data) < data_length:
         return None
-    elif len(data) > data_length:
-        data = data[:data_length]
     return docket, data
 
 
@@ -81,6 +89,8 @@
 
     can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
     ondisk_docket = revlog._nodemap_docket
+    feed_data = util.safehasattr(revlog.index, "update_nodemap_data")
+    use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
 
     data = None
     # first attemp an incremental update of the data
@@ -97,12 +107,18 @@
             datafile = _rawdata_filepath(revlog, target_docket)
             # EXP-TODO: if this is a cache, this should use a cache vfs, not a
             # store vfs
+            new_length = target_docket.data_length + len(data)
             with revlog.opener(datafile, b'r+') as fd:
                 fd.seek(target_docket.data_length)
                 fd.write(data)
-                fd.seek(0)
-                new_data = fd.read(target_docket.data_length + len(data))
-            target_docket.data_length += len(data)
+                if feed_data:
+                    if use_mmap:
+                        fd.seek(0)
+                        new_data = fd.read(new_length)
+                    else:
+                        fd.flush()
+                        new_data = util.buffer(util.mmapread(fd, new_length))
+            target_docket.data_length = new_length
             target_docket.data_unused += data_changed_count
 
     if data is None:
@@ -115,9 +131,14 @@
             data = persistent_data(revlog.index)
         # EXP-TODO: if this is a cache, this should use a cache vfs, not a
         # store vfs
-        new_data = data
-        with revlog.opener(datafile, b'w') as fd:
+        with revlog.opener(datafile, b'w+') as fd:
             fd.write(data)
+            if feed_data:
+                if use_mmap:
+                    new_data = data
+                else:
+                    fd.flush()
+                    new_data = util.buffer(util.mmapread(fd, len(data)))
         target_docket.data_length = len(data)
     target_docket.tip_rev = revlog.tiprev()
     # EXP-TODO: if this is a cache, this should use a cache vfs, not a
@@ -125,7 +146,7 @@
     with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp:
         fp.write(target_docket.serialize())
     revlog._nodemap_docket = target_docket
-    if util.safehasattr(revlog.index, "update_nodemap_data"):
+    if feed_data:
         revlog.index.update_nodemap_data(target_docket, new_data)
 
     # EXP-TODO: if the transaction abort, we should remove the new data and
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -545,9 +545,6 @@
         indexdata = b''
         self._initempty = True
         try:
-            nodemap_data = nodemaputil.persisted_data(self)
-            if nodemap_data is not None:
-                self._nodemap_docket = nodemap_data[0]
             with self._indexfp() as f:
                 if (
                     mmapindexthreshold is not None
@@ -639,6 +636,7 @@
             if use_nodemap:
                 nodemap_data = nodemaputil.persisted_data(self)
                 if nodemap_data is not None:
+                    self._nodemap_docket = nodemap_data[0]
                     index.update_nodemap_data(*nodemap_data)
         except (ValueError, IndexError):
             raise error.RevlogError(
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -934,6 +934,8 @@
         options[b'rust.index'] = True
     if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
         options[b'exp-persistent-nodemap'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap.mmap'):
+        options[b'exp-persistent-nodemap.mmap'] = True
     if ui.configbool(b'devel', b'persistent-nodemap'):
         options[b'devel-force-nodemap'] = True
 
diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py
--- a/mercurial/debugcommands.py
+++ b/mercurial/debugcommands.py
@@ -2122,7 +2122,7 @@
         nm_data = nodemap.persisted_data(cl)
         if nm_data is not None:
             docket, data = nm_data
-            ui.write(data)
+            ui.write(data[:])
     elif opts['check']:
         unfi = repo.unfiltered()
         cl = unfi.changelog
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -666,6 +666,9 @@
     b'experimental', b'exp-persistent-nodemap', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap.mmap', default=True,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(



To: marmoute, #hg-reviewers, durin42, indygreg
Cc: durin42, gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
This revision now requires changes to proceed.
durin42 added a comment.
durin42 requested changes to this revision.


  (Marking this as wanting changes per discussions about adding some headline numbers in the log message so I stop looking at it until that happens.)

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

To: marmoute, #hg-reviewers, durin42, indygreg
Cc: durin42, gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
marmoute added a comment.


  phabricator seems to be very confused about the performance number formatting…
 
  It is easier to read here: https://foss.heptapod.net/octobus/mercurial-devel/commit/fb04cf42b051ef84bce081f9d10f5ff75c3e0445
 
  and can be pulled using `hg pull --rev "nodemap-python-side" https://foss.heptapod.net/octobus/mercurial-devel/`
 
  I can also confirm that phabricator will silently devour anything added after the `Differential Revision: ` line

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

To: marmoute, #hg-reviewers, durin42, indygreg
Cc: durin42, gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

D7894: nodemap: introduce an option to use mmap to read the nodemap mapping

marmoute (Pierre-Yves David)
In reply to this post by marmoute (Pierre-Yves David)
Closed by commit rHGf7459da77f23: nodemap: introduce an option to use mmap to read the nodemap mapping (authored by marmoute).
This revision was automatically updated to reflect the committed changes.
This revision was not accepted when it landed; it landed in state "Needs Revision".

REPOSITORY
  rHG Mercurial

CHANGES SINCE LAST UPDATE
  https://phab.mercurial-scm.org/D7894?vs=20165&id=20240

CHANGES SINCE LAST ACTION
  https://phab.mercurial-scm.org/D7894/new/

REVISION DETAIL
  https://phab.mercurial-scm.org/D7894

AFFECTED FILES
  mercurial/configitems.py
  mercurial/debugcommands.py
  mercurial/localrepo.py
  mercurial/revlog.py
  mercurial/revlogutils/nodemap.py
  tests/test-persistent-nodemap.t

CHANGE DETAILS

diff --git a/tests/test-persistent-nodemap.t b/tests/test-persistent-nodemap.t
--- a/tests/test-persistent-nodemap.t
+++ b/tests/test-persistent-nodemap.t
@@ -84,3 +84,37 @@
   $ hg debugnodemap --check
   revision in index:   5002
   revision in nodemap: 5002
+
+Test code path without mmap
+---------------------------
+
+  $ echo bar > bar
+  $ hg add bar
+  $ hg ci -m 'bar' --config experimental.exp-persistent-nodemap.mmap=no
+
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=yes
+  revision in index:   5003
+  revision in nodemap: 5003
+  $ hg debugnodemap --check --config experimental.exp-persistent-nodemap.mmap=no
+  revision in index:   5003
+  revision in nodemap: 5003
+
+
+#if pure
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 123328
+  data-unused: 384
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=123328, sha256=10d26e9776b6596af0f89143a54eba8cc581e929c38242a02a7b0760698c6c70 (glob)
+
+#else
+  $ hg debugnodemap --metadata
+  uid: ???????????????? (glob)
+  tip-rev: 5002
+  data-length: 122944
+  data-unused: 0
+  $ f --sha256 .hg/store/00changelog-*.nd --size
+  .hg/store/00changelog-????????????????.nd: size=122944, sha256=755976b22b64ab680401b45395953504e64e7fa8c31ac570f58dee21e15f9bc0 (glob)
+#endif
diff --git a/mercurial/revlogutils/nodemap.py b/mercurial/revlogutils/nodemap.py
--- a/mercurial/revlogutils/nodemap.py
+++ b/mercurial/revlogutils/nodemap.py
@@ -8,6 +8,7 @@
 
 from __future__ import absolute_import
 
+import errno
 import os
 import re
 import struct
@@ -45,11 +46,18 @@
     docket.data_unused = data_unused
 
     filename = _rawdata_filepath(revlog, docket)
-    data = revlog.opener.tryread(filename)
+    use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
+    try:
+        with revlog.opener(filename) as fd:
+            if use_mmap:
+                data = util.buffer(util.mmapread(fd, data_length))
+            else:
+                data = fd.read(data_length)
+    except OSError as e:
+        if e.errno != errno.ENOENT:
+            raise
     if len(data) < data_length:
         return None
-    elif len(data) > data_length:
-        data = data[:data_length]
     return docket, data
 
 
@@ -81,6 +89,8 @@
 
     can_incremental = util.safehasattr(revlog.index, "nodemap_data_incremental")
     ondisk_docket = revlog._nodemap_docket
+    feed_data = util.safehasattr(revlog.index, "update_nodemap_data")
+    use_mmap = revlog.opener.options.get("exp-persistent-nodemap.mmap")
 
     data = None
     # first attemp an incremental update of the data
@@ -97,12 +107,18 @@
             datafile = _rawdata_filepath(revlog, target_docket)
             # EXP-TODO: if this is a cache, this should use a cache vfs, not a
             # store vfs
+            new_length = target_docket.data_length + len(data)
             with revlog.opener(datafile, b'r+') as fd:
                 fd.seek(target_docket.data_length)
                 fd.write(data)
-                fd.seek(0)
-                new_data = fd.read(target_docket.data_length + len(data))
-            target_docket.data_length += len(data)
+                if feed_data:
+                    if use_mmap:
+                        fd.seek(0)
+                        new_data = fd.read(new_length)
+                    else:
+                        fd.flush()
+                        new_data = util.buffer(util.mmapread(fd, new_length))
+            target_docket.data_length = new_length
             target_docket.data_unused += data_changed_count
 
     if data is None:
@@ -115,9 +131,14 @@
             data = persistent_data(revlog.index)
         # EXP-TODO: if this is a cache, this should use a cache vfs, not a
         # store vfs
-        new_data = data
-        with revlog.opener(datafile, b'w') as fd:
+        with revlog.opener(datafile, b'w+') as fd:
             fd.write(data)
+            if feed_data:
+                if use_mmap:
+                    new_data = data
+                else:
+                    fd.flush()
+                    new_data = util.buffer(util.mmapread(fd, len(data)))
         target_docket.data_length = len(data)
     target_docket.tip_rev = revlog.tiprev()
     # EXP-TODO: if this is a cache, this should use a cache vfs, not a
@@ -125,7 +146,7 @@
     with revlog.opener(revlog.nodemap_file, b'w', atomictemp=True) as fp:
         fp.write(target_docket.serialize())
     revlog._nodemap_docket = target_docket
-    if util.safehasattr(revlog.index, "update_nodemap_data"):
+    if feed_data:
         revlog.index.update_nodemap_data(target_docket, new_data)
 
     # EXP-TODO: if the transaction abort, we should remove the new data and
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -545,9 +545,6 @@
         indexdata = b''
         self._initempty = True
         try:
-            nodemap_data = nodemaputil.persisted_data(self)
-            if nodemap_data is not None:
-                self._nodemap_docket = nodemap_data[0]
             with self._indexfp() as f:
                 if (
                     mmapindexthreshold is not None
@@ -639,6 +636,7 @@
             if use_nodemap:
                 nodemap_data = nodemaputil.persisted_data(self)
                 if nodemap_data is not None:
+                    self._nodemap_docket = nodemap_data[0]
                     index.update_nodemap_data(*nodemap_data)
         except (ValueError, IndexError):
             raise error.RevlogError(
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -934,6 +934,8 @@
         options[b'rust.index'] = True
     if ui.configbool(b'experimental', b'exp-persistent-nodemap'):
         options[b'exp-persistent-nodemap'] = True
+    if ui.configbool(b'experimental', b'exp-persistent-nodemap.mmap'):
+        options[b'exp-persistent-nodemap.mmap'] = True
     if ui.configbool(b'devel', b'persistent-nodemap'):
         options[b'devel-force-nodemap'] = True
 
diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py
--- a/mercurial/debugcommands.py
+++ b/mercurial/debugcommands.py
@@ -2122,7 +2122,7 @@
         nm_data = nodemap.persisted_data(cl)
         if nm_data is not None:
             docket, data = nm_data
-            ui.write(data)
+            ui.write(data[:])
     elif opts['check']:
         unfi = repo.unfiltered()
         cl = unfi.changelog
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -666,6 +666,9 @@
     b'experimental', b'exp-persistent-nodemap', default=False,
 )
 coreconfigitem(
+    b'experimental', b'exp-persistent-nodemap.mmap', default=True,
+)
+coreconfigitem(
     b'experimental', b'server.filesdata.recommended-batch-size', default=50000,
 )
 coreconfigitem(



To: marmoute, #hg-reviewers, durin42, indygreg
Cc: durin42, gracinet, martinvonz, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel