D10026: changegroup: add v4 changegroup for revlog v2 exchange

classic Classic list List threaded Threaded
1 message Options
Reply | Threaded
Open this post in threaded view
|

D10026: changegroup: add v4 changegroup for revlog v2 exchange

marmoute (Pierre-Yves David)
Alphare created this revision.
Herald added a reviewer: indygreg.
Herald added a reviewer: hg-reviewers.
Herald added a subscriber: mercurial-patches.

REPOSITORY
  rHG Mercurial

BRANCH
  default

REVISION DETAIL
  https://phab.mercurial-scm.org/D10026

AFFECTED FILES
  hgext/remotefilelog/remotefilelog.py
  hgext/remotefilelog/shallowbundle.py
  hgext/sqlitestore.py
  mercurial/bundlerepo.py
  mercurial/changegroup.py
  mercurial/debugcommands.py
  mercurial/exchange.py
  mercurial/exchangev2.py
  mercurial/localrepo.py
  mercurial/revlog.py
  tests/test-check-interfaces.py
  tests/test-revlog-raw.py

CHANGE DETAILS

diff --git a/tests/test-revlog-raw.py b/tests/test-revlog-raw.py
--- a/tests/test-revlog-raw.py
+++ b/tests/test-revlog-raw.py
@@ -147,6 +147,7 @@
                 b'flags': rlog.flags(r),
                 b'deltabase': rlog.node(deltaparent),
                 b'delta': rlog.revdiff(deltaparent, r),
+                b'sidedata': rlog.sidedata(r),
             }
 
         def deltaiter(self):
@@ -159,10 +160,11 @@
                 deltabase = chunkdata[b'deltabase']
                 delta = chunkdata[b'delta']
                 flags = chunkdata[b'flags']
+                sidedata = chunkdata[b'sidedata']
 
                 chain = node
 
-                yield (node, p1, p2, cs, deltabase, delta, flags)
+                yield (node, p1, p2, cs, deltabase, delta, flags, sidedata)
 
     def linkmap(lnode):
         return rlog.rev(lnode)
diff --git a/tests/test-check-interfaces.py b/tests/test-check-interfaces.py
--- a/tests/test-check-interfaces.py
+++ b/tests/test-check-interfaces.py
@@ -272,6 +272,7 @@
         flags=b'',
         baserevisionsize=None,
         revision=b'',
+        sidedata=b'',
         delta=None,
     )
     checkzobject(rd)
diff --git a/mercurial/revlog.py b/mercurial/revlog.py
--- a/mercurial/revlog.py
+++ b/mercurial/revlog.py
@@ -2512,7 +2512,7 @@
             deltacomputer = deltautil.deltacomputer(self)
             # loop through our set of deltas
             for data in deltas:
-                node, p1, p2, linknode, deltabase, delta, flags = data
+                node, p1, p2, linknode, deltabase, delta, flags, sidedata = data
                 link = linkmapper(linknode)
                 flags = flags or REVIDX_DEFAULT_FLAGS
 
diff --git a/mercurial/localrepo.py b/mercurial/localrepo.py
--- a/mercurial/localrepo.py
+++ b/mercurial/localrepo.py
@@ -313,7 +313,13 @@
         )
 
     def getbundle(
-        self, source, heads=None, common=None, bundlecaps=None, **kwargs
+        self,
+        source,
+        heads=None,
+        common=None,
+        bundlecaps=None,
+        remote_sidedata=None,
+        **kwargs
     ):
         chunks = exchange.getbundlechunks(
             self._repo,
@@ -321,6 +327,7 @@
             heads=heads,
             common=common,
             bundlecaps=bundlecaps,
+            remote_sidedata=remote_sidedata,
             **kwargs
         )[1]
         cb = util.chunkbuffer(chunks)
diff --git a/mercurial/exchangev2.py b/mercurial/exchangev2.py
--- a/mercurial/exchangev2.py
+++ b/mercurial/exchangev2.py
@@ -416,6 +416,8 @@
                 mdiff.trivialdiffheader(len(data)) + data,
                 # Flags not yet supported.
                 0,
+                # Sidedata not yet supported
+                {},
             )
 
     cl.addgroup(
@@ -495,6 +497,8 @@
                 delta,
                 # Flags not yet supported.
                 0,
+                # Sidedata not yet supported.
+                {},
             )
 
             progress.increment()
@@ -620,6 +624,8 @@
                 delta,
                 # Flags not yet supported.
                 0,
+                # Sidedata not yet supported.
+                {},
             )
 
             progress.increment()
@@ -718,6 +724,8 @@
                 delta,
                 # Flags not yet supported.
                 0,
+                # Sidedata not yet supported.
+                {},
             )
 
             progress.increment()
diff --git a/mercurial/exchange.py b/mercurial/exchange.py
--- a/mercurial/exchange.py
+++ b/mercurial/exchange.py
@@ -2249,7 +2249,13 @@
 
 
 def getbundlechunks(
-    repo, source, heads=None, common=None, bundlecaps=None, **kwargs
+    repo,
+    source,
+    heads=None,
+    common=None,
+    bundlecaps=None,
+    remote_sidedata=None,
+    **kwargs
 ):
     """Return chunks constituting a bundle's raw data.
 
@@ -2279,7 +2285,12 @@
         return (
             info,
             changegroup.makestream(
-                repo, outgoing, b'01', source, bundlecaps=bundlecaps
+                repo,
+                outgoing,
+                b'01',
+                source,
+                bundlecaps=bundlecaps,
+                remote_sidedata=remote_sidedata,
             ),
         )
 
@@ -2303,6 +2314,7 @@
             source,
             bundlecaps=bundlecaps,
             b2caps=b2caps,
+            remote_sidedata=remote_sidedata,
             **pycompat.strkwargs(kwargs)
         )
 
@@ -2325,6 +2337,7 @@
     b2caps=None,
     heads=None,
     common=None,
+    remote_sidedata=None,
     **kwargs
 ):
     """add a changegroup part to the requested bundle"""
@@ -2355,7 +2368,13 @@
         matcher = None
 
     cgstream = changegroup.makestream(
-        repo, outgoing, version, source, bundlecaps=bundlecaps, matcher=matcher
+        repo,
+        outgoing,
+        version,
+        source,
+        bundlecaps=bundlecaps,
+        matcher=matcher,
+        remote_sidedata=remote_sidedata,
     )
 
     part = bundler.newpart(b'changegroup', data=cgstream)
diff --git a/mercurial/debugcommands.py b/mercurial/debugcommands.py
--- a/mercurial/debugcommands.py
+++ b/mercurial/debugcommands.py
@@ -346,7 +346,7 @@
         def showchunks(named):
             ui.write(b"\n%s%s\n" % (indent_string, named))
             for deltadata in gen.deltaiter():
-                node, p1, p2, cs, deltabase, delta, flags = deltadata
+                node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
                 ui.write(
                     b"%s%s %s %s %s %s %d\n"
                     % (
@@ -372,7 +372,7 @@
             raise error.Abort(_(b'use debugbundle2 for this file'))
         gen.changelogheader()
         for deltadata in gen.deltaiter():
-            node, p1, p2, cs, deltabase, delta, flags = deltadata
+            node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
             ui.write(b"%s%s\n" % (indent_string, hex(node)))
 
 
diff --git a/mercurial/changegroup.py b/mercurial/changegroup.py
--- a/mercurial/changegroup.py
+++ b/mercurial/changegroup.py
@@ -32,10 +32,12 @@
 )
 
 from .interfaces import repository
+from .revlogutils import sidedata as sidedatamod
 
 _CHANGEGROUPV1_DELTA_HEADER = struct.Struct(b"20s20s20s20s")
 _CHANGEGROUPV2_DELTA_HEADER = struct.Struct(b"20s20s20s20s20s")
 _CHANGEGROUPV3_DELTA_HEADER = struct.Struct(b">20s20s20s20s20sH")
+_CHANGEGROUPV4_DELTA_HEADER = struct.Struct(b">20s20s20s20s20sHi")
 
 LFS_REQUIREMENT = b'lfs'
 
@@ -202,7 +204,9 @@
         header = self.deltaheader.unpack(headerdata)
         delta = readexactly(self._stream, l - self.deltaheadersize)
         node, p1, p2, deltabase, cs, flags = self._deltaheader(header, prevnode)
-        return (node, p1, p2, cs, deltabase, delta, flags)
+        # cg4 forward-compat
+        sidedata = {}
+        return (node, p1, p2, cs, deltabase, delta, flags, sidedata)
 
     def getchunks(self):
         """returns all the chunks contains in the bundle
@@ -552,6 +556,35 @@
                 raise error.Abort(_(b"received dir revlog group is empty"))
 
 
+class cg4unpacker(cg3unpacker):
+    """Unpacker for cg4 streams.
+
+    cg4 streams add support for exchanging sidedata.
+    """
+
+    version = b'04'
+
+    def deltachunk(self, prevnode):
+        res = super(cg4unpacker, self).deltachunk(prevnode)
+        if not res:
+            return res
+
+        (node, p1, p2, cs, deltabase, delta, flags, _sidedata) = res
+
+        sidedata_length_raw = readexactly(self._stream, 4)
+        (sidedata_length,) = struct.unpack(b">l", sidedata_length_raw)
+
+        sidedata = {}
+
+        assert sidedata_length == 0 or sidedata_length >= 4, sidedata_length
+        if sidedata_length > 0:
+            sidedata_length = sidedata_length - 4
+            sidedata_raw = readexactly(self._stream, sidedata_length)
+            sidedata = sidedatamod.deserialize_sidedata(sidedata_raw)
+
+        return node, p1, p2, cs, deltabase, delta, flags, sidedata
+
+
 class headerlessfixup(object):
     def __init__(self, fh, h):
         self._h = h
@@ -861,6 +894,7 @@
         shallow=False,
         ellipsisroots=None,
         fullnodes=None,
+        remote_sidedata=None,
     ):
         """Given a source repo, construct a bundler.
 
@@ -893,6 +927,8 @@
         nodes. We store this rather than the set of nodes that should be
         ellipsis because for very large histories we expect this to be
         significantly smaller.
+
+        remote_sidedata is the set of sidedata categories wanted by the remote.
         """
         assert oldmatcher
         assert matcher
@@ -988,7 +1024,7 @@
 
         for tree, deltas in it:
             if tree:
-                assert self.version == b'03'
+                assert self.version in (b'03', b'04')
                 chunk = _fileheader(tree)
                 size += len(chunk)
                 yield chunk
@@ -1394,6 +1430,7 @@
     shallow=False,
     ellipsisroots=None,
     fullnodes=None,
+    remote_sidedata=None,
 ):
     builddeltaheader = lambda d: _CHANGEGROUPV1_DELTA_HEADER.pack(
         d.node, d.p1node, d.p2node, d.linknode
@@ -1424,6 +1461,7 @@
     shallow=False,
     ellipsisroots=None,
     fullnodes=None,
+    remote_sidedata=None,
 ):
     builddeltaheader = lambda d: _CHANGEGROUPV2_DELTA_HEADER.pack(
         d.node, d.p1node, d.p2node, d.basenode, d.linknode
@@ -1453,6 +1491,7 @@
     shallow=False,
     ellipsisroots=None,
     fullnodes=None,
+    remote_sidedata=None,
 ):
     builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
         d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags
@@ -1473,12 +1512,45 @@
     )
 
 
+def _makecg4packer(
+    repo,
+    oldmatcher,
+    matcher,
+    bundlecaps,
+    ellipses=False,
+    shallow=False,
+    ellipsisroots=None,
+    fullnodes=None,
+    remote_sidedata=None,
+):
+    builddeltaheader = lambda d: _CHANGEGROUPV3_DELTA_HEADER.pack(
+        d.node, d.p1node, d.p2node, d.basenode, d.linknode, d.flags
+    )
+
+    return cgpacker(
+        repo,
+        oldmatcher,
+        matcher,
+        b'04',
+        builddeltaheader=builddeltaheader,
+        manifestsend=closechunk(),
+        bundlecaps=bundlecaps,
+        ellipses=ellipses,
+        shallow=shallow,
+        ellipsisroots=ellipsisroots,
+        fullnodes=fullnodes,
+        remote_sidedata=remote_sidedata,
+    )
+
+
 _packermap = {
     b'01': (_makecg1packer, cg1unpacker),
     # cg2 adds support for exchanging generaldelta
     b'02': (_makecg2packer, cg2unpacker),
     # cg3 adds support for exchanging revlog flags and treemanifests
     b'03': (_makecg3packer, cg3unpacker),
+    # ch4 adds support for exchanging sidedata
+    b'04': (_makecg4packer, cg4unpacker),
 }
 
 
@@ -1498,11 +1570,9 @@
         #
         # (or even to push subset of history)
         needv03 = True
-    if b'exp-sidedata-flag' in repo.requirements:
-        needv03 = True
-        # don't attempt to use 01/02 until we do sidedata cleaning
-        versions.discard(b'01')
-        versions.discard(b'02')
+    has_revlogv2 = requirements.REVLOGV2_REQUIREMENT in repo.requirements
+    if not has_revlogv2:
+        versions.discard(b'04')
     if not needv03:
         versions.discard(b'03')
     return versions
@@ -1565,6 +1635,7 @@
     shallow=False,
     ellipsisroots=None,
     fullnodes=None,
+    remote_sidedata=None,
 ):
     assert version in supportedoutgoingversions(repo)
 
@@ -1601,6 +1672,7 @@
         shallow=shallow,
         ellipsisroots=ellipsisroots,
         fullnodes=fullnodes,
+        remote_sidedata=remote_sidedata,
     )
 
 
@@ -1644,8 +1716,15 @@
     fastpath=False,
     bundlecaps=None,
     matcher=None,
+    remote_sidedata=None,
 ):
-    bundler = getbundler(version, repo, bundlecaps=bundlecaps, matcher=matcher)
+    bundler = getbundler(
+        version,
+        repo,
+        bundlecaps=bundlecaps,
+        matcher=matcher,
+        remote_sidedata=remote_sidedata,
+    )
 
     repo = repo.unfiltered()
     commonrevs = outgoing.common
diff --git a/mercurial/bundlerepo.py b/mercurial/bundlerepo.py
--- a/mercurial/bundlerepo.py
+++ b/mercurial/bundlerepo.py
@@ -61,7 +61,7 @@
         self.repotiprev = n - 1
         self.bundlerevs = set()  # used by 'bundle()' revset expression
         for deltadata in cgunpacker.deltaiter():
-            node, p1, p2, cs, deltabase, delta, flags = deltadata
+            node, p1, p2, cs, deltabase, delta, flags, sidedata = deltadata
 
             size = len(delta)
             start = cgunpacker.tell() - size
diff --git a/hgext/sqlitestore.py b/hgext/sqlitestore.py
--- a/hgext/sqlitestore.py
+++ b/hgext/sqlitestore.py
@@ -681,7 +681,16 @@
     ):
         empty = True
 
-        for node, p1, p2, linknode, deltabase, delta, wireflags in deltas:
+        for (
+            node,
+            p1,
+            p2,
+            linknode,
+            deltabase,
+            delta,
+            wireflags,
+            sidedata,
+        ) in deltas:
             storeflags = 0
 
             if wireflags & repository.REVISION_FLAG_CENSORED:
diff --git a/hgext/remotefilelog/shallowbundle.py b/hgext/remotefilelog/shallowbundle.py
--- a/hgext/remotefilelog/shallowbundle.py
+++ b/hgext/remotefilelog/shallowbundle.py
@@ -67,7 +67,7 @@
             shallowcg1packer, self, nodelist, rlog, lookup, units=units
         )
 
-    def generatefiles(self, changedfiles, *args):
+    def generatefiles(self, changedfiles, *args, **kwargs):
         try:
             linknodes, commonrevs, source = args
         except ValueError:
@@ -92,7 +92,9 @@
                     [f for f in changedfiles if not repo.shallowmatch(f)]
                 )
 
-        return super(shallowcg1packer, self).generatefiles(changedfiles, *args)
+        return super(shallowcg1packer, self).generatefiles(
+            changedfiles, *args, **kwargs
+        )
 
     def shouldaddfilegroups(self, source):
         repo = self._repo
@@ -176,9 +178,11 @@
         repo.shallowmatch = original
 
 
-def addchangegroupfiles(orig, repo, source, revmap, trp, expectedfiles, *args):
+def addchangegroupfiles(
+    orig, repo, source, revmap, trp, expectedfiles, *args, **kwargs
+):
     if not shallowutil.isenabled(repo):
-        return orig(repo, source, revmap, trp, expectedfiles, *args)
+        return orig(repo, source, revmap, trp, expectedfiles, *args, **kwargs)
 
     newfiles = 0
     visited = set()
@@ -272,7 +276,7 @@
 
         revisiondata = revisiondatas[(f, node)]
         # revisiondata: (node, p1, p2, cs, deltabase, delta, flags)
-        node, p1, p2, linknode, deltabase, delta, flags = revisiondata
+        node, p1, p2, linknode, deltabase, delta, flags, sidedata = revisiondata
 
         if not available(f, node, f, deltabase):
             continue
diff --git a/hgext/remotefilelog/remotefilelog.py b/hgext/remotefilelog/remotefilelog.py
--- a/hgext/remotefilelog/remotefilelog.py
+++ b/hgext/remotefilelog/remotefilelog.py
@@ -306,6 +306,7 @@
         assumehaveparentrevisions=False,
         deltaprevious=False,
         deltamode=None,
+        sidedata_helpers=None,
     ):
         # we don't use any of these parameters here
         del nodesorder, revisiondata, assumehaveparentrevisions, deltaprevious
@@ -333,6 +334,8 @@
                 baserevisionsize=None,
                 revision=revision,
                 delta=delta,
+                # Sidedata is not supported yet
+                sidedata=None,
             )
 
     def revdiff(self, node1, node2):



To: Alphare, indygreg, #hg-reviewers
Cc: mercurial-patches, mercurial-devel
_______________________________________________
Mercurial-devel mailing list
[hidden email]
https://www.mercurial-scm.org/mailman/listinfo/mercurial-devel