[PATCH 1 of 4 V2] changegroup: refactor emitrevision to use a `deltamode` argument

Previous Topic Next Topic
 
classic Classic list List threaded Threaded
5 messages Options
Reply | Threaded
Open this post in threaded view
|

[PATCH 1 of 4 V2] changegroup: refactor emitrevision to use a `deltamode` argument

Boris Feld
# HG changeset patch
# User Boris Feld <[hidden email]>
# Date 1539115321 -7200
#      Tue Oct 09 22:02:01 2018 +0200
# Node ID cc7eeac5f0d94e27c527524040f2cd1814c8102b
# Parent  2c0aa02ecd5a05ae76b6345962ee3a0ef773bd8a
# EXP-Topic slim-bundle
# Available At https://bitbucket.org/octobus/mercurial-devel/
#              hg pull https://bitbucket.org/octobus/mercurial-devel/ -r cc7eeac5f0d9
changegroup: refactor emitrevision to use a `deltamode` argument

This new argument gathers the semantic of `sendfulltext` and `deltaprevious`
in a single value. We are about to introduce a new type of constraints.
Avoiding yet another argument sounds like a plus.

diff --git a/mercurial/repository.py b/mercurial/repository.py
--- a/mercurial/repository.py
+++ b/mercurial/repository.py
@@ -37,6 +37,10 @@ REVISION_FLAG_EXTSTORED = 1 << 13
 REVISION_FLAGS_KNOWN = (
     REVISION_FLAG_CENSORED | REVISION_FLAG_ELLIPSIS | REVISION_FLAG_EXTSTORED)
 
+CG_DELTAMODE_STD = b'default'
+CG_DELTAMODE_PREV = b'previous'
+CG_DELTAMODE_FULL = b'fulltext'
+
 class ipeerconnection(interfaceutil.Interface):
     """Represents a "connection" to a repository.
 
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -2229,6 +2229,12 @@ class revlog(object):
         if nodesorder is None and not self._generaldelta:
             nodesorder = 'storage'
 
+        deltamode = repository.CG_DELTAMODE_STD
+        if deltaprevious:
+            deltamode = repository.CG_DELTAMODE_PREV
+        elif not self._storedeltachains:
+            deltamode = repository.CG_DELTAMODE_FULL
+
         return storageutil.emitrevisions(
             self, nodes, nodesorder, revlogrevisiondelta,
             deltaparentfn=self.deltaparent,
@@ -2236,10 +2242,9 @@ class revlog(object):
             rawsizefn=self.rawsize,
             revdifffn=self.revdiff,
             flagsfn=self.flags,
-            sendfulltext=not self._storedeltachains,
+            deltamode=deltamode,
             revisiondata=revisiondata,
-            assumehaveparentrevisions=assumehaveparentrevisions,
-            deltaprevious=deltaprevious)
+            assumehaveparentrevisions=assumehaveparentrevisions)
 
     DELTAREUSEALWAYS = 'always'
     DELTAREUSESAMEREVS = 'samerevs'
diff --git a/mercurial/utils/storageutil.py b/mercurial/utils/storageutil.py
--- a/mercurial/utils/storageutil.py
+++ b/mercurial/utils/storageutil.py
@@ -21,6 +21,7 @@ from .. import (
     error,
     mdiff,
     pycompat,
+    repository,
 )
 
 _nullhash = hashlib.sha1(nullid)
@@ -268,9 +269,8 @@ def resolvestripinfo(minlinkrev, tiprev,
 
 def emitrevisions(store, nodes, nodesorder, resultcls, deltaparentfn=None,
                   candeltafn=None, rawsizefn=None, revdifffn=None, flagsfn=None,
-                  sendfulltext=False,
-                  revisiondata=False, assumehaveparentrevisions=False,
-                  deltaprevious=False):
+                  deltamode=repository.CG_DELTAMODE_STD,
+                  revisiondata=False, assumehaveparentrevisions=False):
     """Generic implementation of ifiledata.emitrevisions().
 
     Emitting revision data is subtly complex. This function attempts to
@@ -321,14 +321,17 @@ def emitrevisions(store, nodes, nodesord
        Callable receiving a revision number and returns the integer flags
        value for it. If not defined, flags value will be 0.
 
-    ``sendfulltext``
+    ``deltamode``
+       constaint on delta to be sent:
+       * CG_DELTAMODE_STD  - normal mode, try to reuse storage deltas,
+       * CG_DELTAMODE_PREV - only delta against "prev",
+       * CG_DELTAMODE_FULL - only issue full snapshot.
+
        Whether to send fulltext revisions instead of deltas, if allowed.
 
     ``nodesorder``
     ``revisiondata``
     ``assumehaveparentrevisions``
-    ``deltaprevious``
-       See ``ifiledata.emitrevisions()`` interface documentation.
     """
 
     fnode = store.node
@@ -344,7 +347,7 @@ def emitrevisions(store, nodes, nodesord
 
     prevrev = None
 
-    if deltaprevious or assumehaveparentrevisions:
+    if deltamode == repository.CG_DELTAMODE_PREV or assumehaveparentrevisions:
         prevrev = store.parentrevs(revs[0])[0]
 
     # Set of revs available to delta against.
@@ -363,11 +366,11 @@ def emitrevisions(store, nodes, nodesord
             deltaparentrev = nullrev
 
         # Forced delta against previous mode.
-        if deltaprevious:
+        if deltamode == repository.CG_DELTAMODE_PREV:
             baserev = prevrev
 
         # We're instructed to send fulltext. Honor that.
-        elif sendfulltext:
+        elif deltamode == repository.CG_DELTAMODE_FULL:
             baserev = nullrev
 
         # There is a delta in storage. We try to use that because it
@@ -426,7 +429,8 @@ def emitrevisions(store, nodes, nodesord
                         baserevisionsize = len(store.revision(baserev,
                                                               raw=True))
 
-            elif baserev == nullrev and not deltaprevious:
+            elif (baserev == nullrev
+                    and deltamode != repository.CG_DELTAMODE_PREV):
                 revision = store.revision(node, raw=True)
                 available.add(rev)
             else:
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

[PATCH 2 of 4 V2] storage: also use `deltamode argument` for ifiledata

Boris Feld
# HG changeset patch
# User Boris Feld <[hidden email]>
# Date 1539120395 -7200
#      Tue Oct 09 23:26:35 2018 +0200
# Node ID 64d8829dc24e8fc02f228295bb3b330c696c7d34
# Parent  cc7eeac5f0d94e27c527524040f2cd1814c8102b
# EXP-Topic slim-bundle
# Available At https://bitbucket.org/octobus/mercurial-devel/
#              hg pull https://bitbucket.org/octobus/mercurial-devel/ -r 64d8829dc24e
storage: also use `deltamode argument` for ifiledata

Now that lower level uses such argument, we can propagate the change to higher
layers.

diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
--- a/mercurial/changegroup.py
+++ b/mercurial/changegroup.py
@@ -697,12 +697,16 @@ def deltagroup(repo, store, nodes, ischa
         progress = repo.ui.makeprogress(topic, unit=_('chunks'),
                                         total=len(nodes))
 
+    deltamode = repository.CG_DELTAMODE_STD
+    if forcedeltaparentprev:
+        deltamode = repository.CG_DELTAMODE_PREV
+
     revisions = store.emitrevisions(
         nodes,
         nodesorder=nodesorder,
         revisiondata=True,
         assumehaveparentrevisions=not ellipses,
-        deltaprevious=forcedeltaparentprev)
+        deltamode=deltamode)
 
     for i, revision in enumerate(revisions):
         if progress:
diff --git a/mercurial/filelog.py b/mercurial/filelog.py
--- a/mercurial/filelog.py
+++ b/mercurial/filelog.py
@@ -77,11 +77,11 @@ class filelog(object):
 
     def emitrevisions(self, nodes, nodesorder=None,
                       revisiondata=False, assumehaveparentrevisions=False,
-                      deltaprevious=False):
+                      deltamode=repository.CG_DELTAMODE_STD):
         return self._revlog.emitrevisions(
             nodes, nodesorder=nodesorder, revisiondata=revisiondata,
             assumehaveparentrevisions=assumehaveparentrevisions,
-            deltaprevious=deltaprevious)
+            deltamode=deltamode)
 
     def addrevision(self, revisiondata, transaction, linkrev, p1, p2,
                     node=None, flags=revlog.REVIDX_DEFAULT_FLAGS,
diff --git a/mercurial/manifest.py b/mercurial/manifest.py
--- a/mercurial/manifest.py
+++ b/mercurial/manifest.py
@@ -1575,11 +1575,11 @@ class manifestrevlog(object):
 
     def emitrevisions(self, nodes, nodesorder=None,
                       revisiondata=False, assumehaveparentrevisions=False,
-                      deltaprevious=False):
+                      deltamode=repository.CG_DELTAMODE_STD):
         return self._revlog.emitrevisions(
             nodes, nodesorder=nodesorder, revisiondata=revisiondata,
             assumehaveparentrevisions=assumehaveparentrevisions,
-            deltaprevious=deltaprevious)
+            deltamode=deltamode)
 
     def addgroup(self, deltas, linkmapper, transaction, addrevisioncb=None):
         return self._revlog.addgroup(deltas, linkmapper, transaction,
diff --git a/mercurial/repository.py b/mercurial/repository.py
--- a/mercurial/repository.py
+++ b/mercurial/repository.py
@@ -606,7 +606,7 @@ class ifiledata(interfaceutil.Interface)
                       nodesorder=None,
                       revisiondata=False,
                       assumehaveparentrevisions=False,
-                      deltaprevious=False):
+                      deltamode=CG_DELTAMODE_STD):
         """Produce ``irevisiondelta`` for revisions.
 
         Given an iterable of nodes, emits objects conforming to the
@@ -649,10 +649,10 @@ class ifiledata(interfaceutil.Interface)
         The ``linknode`` attribute on the returned ``irevisiondelta`` may not
         be set and it is the caller's responsibility to resolve it, if needed.
 
-        If ``deltaprevious`` is True and revision data is requested, all
-        revision data should be emitted as deltas against the revision
-        emitted just prior. The initial revision should be a delta against
-        its 1st parent.
+        If ``deltamode`` is CG_DELTAMODE_PREV and revision data is requested,
+        all revision data should be emitted as deltas against the revision
+        emitted just prior. The initial revision should be a delta against its
+        1st parent.
         """
 
 class ifilemutation(interfaceutil.Interface):
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -2221,7 +2221,8 @@ class revlog(object):
         return res
 
     def emitrevisions(self, nodes, nodesorder=None, revisiondata=False,
-                      assumehaveparentrevisions=False, deltaprevious=False):
+                      assumehaveparentrevisions=False,
+                      deltamode=repository.CG_DELTAMODE_STD):
         if nodesorder not in ('nodes', 'storage', None):
             raise error.ProgrammingError('unhandled value for nodesorder: %s' %
                                          nodesorder)
@@ -2229,10 +2230,8 @@ class revlog(object):
         if nodesorder is None and not self._generaldelta:
             nodesorder = 'storage'
 
-        deltamode = repository.CG_DELTAMODE_STD
-        if deltaprevious:
-            deltamode = repository.CG_DELTAMODE_PREV
-        elif not self._storedeltachains:
+        if (not self._storedeltachains and
+                deltamode != repository.CG_DELTAMODE_PREV):
             deltamode = repository.CG_DELTAMODE_FULL
 
         return storageutil.emitrevisions(
diff --git a/mercurial/testing/storage.py b/mercurial/testing/storage.py
--- a/mercurial/testing/storage.py
+++ b/mercurial/testing/storage.py
@@ -727,7 +727,8 @@ class ifiledatatests(basetestcase):
 
         # forceprevious=True forces a delta against the previous revision.
         # Special case for initial revision.
-        gen = f.emitrevisions([node0], revisiondata=True, deltaprevious=True)
+        gen = f.emitrevisions([node0], revisiondata=True,
+                              deltamode=repository.CG_DELTAMODE_PREV)
 
         rev = next(gen)
         self.assertEqual(rev.node, node0)
@@ -744,7 +745,7 @@ class ifiledatatests(basetestcase):
             next(gen)
 
         gen = f.emitrevisions([node0, node2], revisiondata=True,
-                              deltaprevious=True)
+                              deltamode=repository.CG_DELTAMODE_PREV)
 
         rev = next(gen)
         self.assertEqual(rev.node, node0)
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

[PATCH 3 of 4 V2] changegroup: allow to force delta to be against p1

Boris Feld
In reply to this post by Boris Feld
# HG changeset patch
# User Boris Feld <[hidden email]>
# Date 1539123662 -7200
#      Wed Oct 10 00:21:02 2018 +0200
# Node ID 2a54c05a0e8901bb171721024843531f95382616
# Parent  64d8829dc24e8fc02f228295bb3b330c696c7d34
# EXP-Topic slim-bundle
# Available At https://bitbucket.org/octobus/mercurial-devel/
#              hg pull https://bitbucket.org/octobus/mercurial-devel/ -r 2a54c05a0e89
changegroup: allow to force delta to be against p1

This new developer option is useful to general more "generic" bundle. Without
this option, a bundle generated from the repository use deltas similar to the
one stored in the specific repository it was generated from. This makes
performance testing a bit tricky.

Using deltas similar to the final result means all delta stored in the bundle
can be applied to the target repository without any further processing (except
for the rare case of a full snapshot). The application of such bundles
(almost) never exercises the (slower) path of searching for a new valid delta.
This result in unrealistic and too favorable timing and profile.

Instead, we introduce an option to make sure all revisions are stored as a
delta against p1. It might not be the best generation option, but it
guarantees that the content will be "generic", not favoring a specific target.

diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
--- a/mercurial/changegroup.py
+++ b/mercurial/changegroup.py
@@ -697,9 +697,16 @@ def deltagroup(repo, store, nodes, ischa
         progress = repo.ui.makeprogress(topic, unit=_('chunks'),
                                         total=len(nodes))
 
+    configtarget = repo.ui.config('devel', 'bundle.delta')
+    if configtarget not in ('', 'p1'):
+        msg = _("""config "devel.bundle.delta" as unknown value: %s""")
+        repo.ui.warn(msg % configtarget)
+
     deltamode = repository.CG_DELTAMODE_STD
     if forcedeltaparentprev:
         deltamode = repository.CG_DELTAMODE_PREV
+    elif configtarget == 'p1':
+        deltamode = repository.CG_DELTAMODE_P1
 
     revisions = store.emitrevisions(
         nodes,
diff --git a/mercurial/configitems.py b/mercurial/configitems.py
--- a/mercurial/configitems.py
+++ b/mercurial/configitems.py
@@ -329,6 +329,9 @@ coreconfigitem('devel', 'all-warnings',
 coreconfigitem('devel', 'bundle2.debug',
     default=False,
 )
+coreconfigitem('devel', 'bundle.delta',
+    default='',
+)
 coreconfigitem('devel', 'cache-vfs',
     default=None,
 )
diff --git a/mercurial/repository.py b/mercurial/repository.py
--- a/mercurial/repository.py
+++ b/mercurial/repository.py
@@ -40,6 +40,7 @@ REVISION_FLAGS_KNOWN = (
 CG_DELTAMODE_STD = b'default'
 CG_DELTAMODE_PREV = b'previous'
 CG_DELTAMODE_FULL = b'fulltext'
+CG_DELTAMODE_P1 = b'p1'
 
 class ipeerconnection(interfaceutil.Interface):
     """Represents a "connection" to a repository.
diff --git a/mercurial/utils/storageutil.py b/mercurial/utils/storageutil.py
--- a/mercurial/utils/storageutil.py
+++ b/mercurial/utils/storageutil.py
@@ -372,6 +372,9 @@ def emitrevisions(store, nodes, nodesord
         # We're instructed to send fulltext. Honor that.
         elif deltamode == repository.CG_DELTAMODE_FULL:
             baserev = nullrev
+        # We're instructed to use p1. Honor that
+        elif deltamode == repository.CG_DELTAMODE_P1:
+            baserev = p1rev
 
         # There is a delta in storage. We try to use that because it
         # amounts to effectively copying data from storage and is
diff --git a/tests/test-bundle.t b/tests/test-bundle.t
--- a/tests/test-bundle.t
+++ b/tests/test-bundle.t
@@ -900,3 +900,8 @@ the warning shouldn't be emitted
   $ hg update -R ../update2bundled.hg -r 0
   0 files updated, 0 files merged, 2 files removed, 0 files unresolved
 #endif
+
+Test the option that create slim bundle
+
+  $ hg bundle -a --config devel.bundle.delta=p1 ./slim.hg
+  3 changesets found
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

[PATCH 4 of 4 V2] changegroup: add a option to create bundle with full snapshot only

Boris Feld
In reply to this post by Boris Feld
# HG changeset patch
# User Boris Feld <[hidden email]>
# Date 1539858666 -7200
#      Thu Oct 18 12:31:06 2018 +0200
# Node ID 4c805ec9e478424e631ed8bb7ba6fd65714cfd15
# Parent  2a54c05a0e8901bb171721024843531f95382616
# EXP-Topic slim-bundle
# Available At https://bitbucket.org/octobus/mercurial-devel/
#              hg pull https://bitbucket.org/octobus/mercurial-devel/ -r 4c805ec9e478
changegroup: add a option to create bundle with full snapshot only

This is easy to implement now and can be useful for benchmarking.

diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
--- a/mercurial/changegroup.py
+++ b/mercurial/changegroup.py
@@ -698,7 +698,7 @@ def deltagroup(repo, store, nodes, ischa
                                         total=len(nodes))
 
     configtarget = repo.ui.config('devel', 'bundle.delta')
-    if configtarget not in ('', 'p1'):
+    if configtarget not in ('', 'p1', 'full'):
         msg = _("""config "devel.bundle.delta" as unknown value: %s""")
         repo.ui.warn(msg % configtarget)
 
@@ -707,6 +707,8 @@ def deltagroup(repo, store, nodes, ischa
         deltamode = repository.CG_DELTAMODE_PREV
     elif configtarget == 'p1':
         deltamode = repository.CG_DELTAMODE_P1
+    elif configtarget == 'full':
+        deltamode = repository.CG_DELTAMODE_FULL
 
     revisions = store.emitrevisions(
         nodes,
diff --git a/tests/test-bundle.t b/tests/test-bundle.t
--- a/tests/test-bundle.t
+++ b/tests/test-bundle.t
@@ -905,3 +905,7 @@ Test the option that create slim bundle
 
   $ hg bundle -a --config devel.bundle.delta=p1 ./slim.hg
   3 changesets found
+
+Test the option that create and no-delta's bundle
+  $ hg bundle -a --config devel.bundle.delta=full ./full.hg
+  3 changesets found
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel
Reply | Threaded
Open this post in threaded view
|

Re: [PATCH 4 of 4 V2] changegroup: add a option to create bundle with full snapshot only

Augie Fackler-2
seems fine, queued

> On Oct 18, 2018, at 06:56, Boris Feld <[hidden email]> wrote:
>
> # HG changeset patch
> # User Boris Feld <[hidden email]>
> # Date 1539858666 -7200
> #      Thu Oct 18 12:31:06 2018 +0200
> # Node ID 4c805ec9e478424e631ed8bb7ba6fd65714cfd15
> # Parent  2a54c05a0e8901bb171721024843531f95382616
> # EXP-Topic slim-bundle
> # Available At https://bitbucket.org/octobus/mercurial-devel/
> #              hg pull https://bitbucket.org/octobus/mercurial-devel/ -r 4c805ec9e478
> changegroup: add a option to create bundle with full snapshot only
>
> This is easy to implement now and can be useful for benchmarking.
>
> diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
> --- a/mercurial/changegroup.py
> +++ b/mercurial/changegroup.py
> @@ -698,7 +698,7 @@ def deltagroup(repo, store, nodes, ischa
>                                         total=len(nodes))
>
>     configtarget = repo.ui.config('devel', 'bundle.delta')
> -    if configtarget not in ('', 'p1'):
> +    if configtarget not in ('', 'p1', 'full'):
>         msg = _("""config "devel.bundle.delta" as unknown value: %s""")
>         repo.ui.warn(msg % configtarget)
>
> @@ -707,6 +707,8 @@ def deltagroup(repo, store, nodes, ischa
>         deltamode = repository.CG_DELTAMODE_PREV
>     elif configtarget == 'p1':
>         deltamode = repository.CG_DELTAMODE_P1
> +    elif configtarget == 'full':
> +        deltamode = repository.CG_DELTAMODE_FULL
>
>     revisions = store.emitrevisions(
>         nodes,
> diff --git a/tests/test-bundle.t b/tests/test-bundle.t
> --- a/tests/test-bundle.t
> +++ b/tests/test-bundle.t
> @@ -905,3 +905,7 @@ Test the option that create slim bundle
>
>   $ hg bundle -a --config devel.bundle.delta=p1 ./slim.hg
>   3 changesets found
> +
> +Test the option that create and no-delta's bundle
> +  $ hg bundle -a --config devel.bundle.delta=full ./full.hg
> +  3 changesets found
> _______________________________________________
> Mercurial-devel mailing list
> [hidden email]
> https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel

_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel