mercurial/crew

changeset 8810:ac92775b3b80

Add patch.eol to ignore EOLs when patching (issue1019)

The intent is to fix many issues involving patching when win32ext is enabled.
With win32ext, the working directory and repository files EOLs are not the same
which means that patches made on a non-win32ext host do not apply cleanly
because of EOLs discrepancies. A theorically correct approach would be
transform either the patched file or the patch content with the
encoding/decoding filters used by win32ext. This solution is tricky to
implement and invasive, instead we prefer to address the win32ext case, by
offering a way to ignore input EOLs when patching and rewriting them when
saving the patched result.
author Patrick Mezard <pmezard@gmail.com>
date Mon Jun 15 00:03:26 2009 +0200 (7 months ago)
parents 6fce36336e42
children 8b35b08724eb
files doc/hgrc.5.txt hgext/keyword.py mercurial/commands.py mercurial/patch.py tests/test-import-eol tests/test-import-eol.out
line diff
     1.1 --- a/doc/hgrc.5.txt
     1.2 +++ b/doc/hgrc.5.txt
     1.3 @@ -607,6 +607,17 @@
     1.4      Optional. It's the hostname that the sender can use to identify
     1.5      itself to the MTA.
     1.6  
     1.7 +[[patch]]
     1.8 +patch::
     1.9 +  Settings used when applying patches, for instance through the 'import'
    1.10 +  command or with Mercurial Queues extension.
    1.11 +  eol;;
    1.12 +    When set to 'strict' patch content and patched files end of lines
    1.13 +    are preserved. When set to 'lf' or 'crlf', both files end of lines
    1.14 +    are ignored when patching and the result line endings are
    1.15 +    normalized to either LF (Unix) or CRLF (Windows).
    1.16 +    Default: strict.
    1.17 +
    1.18  [[paths]]
    1.19  paths::
    1.20    Assigns symbolic names to repositories. The left side is the
     2.1 --- a/hgext/keyword.py
     2.2 +++ b/hgext/keyword.py
     2.3 @@ -485,10 +485,10 @@
     2.4                  release(lock, wlock)
     2.5  
     2.6      # monkeypatches
     2.7 -    def kwpatchfile_init(orig, self, ui, fname, opener, missing=False):
     2.8 +    def kwpatchfile_init(orig, self, ui, fname, opener, missing=False, eol=None):
     2.9          '''Monkeypatch/wrap patch.patchfile.__init__ to avoid
    2.10          rejects or conflicts due to expanded keywords in working dir.'''
    2.11 -        orig(self, ui, fname, opener, missing)
    2.12 +        orig(self, ui, fname, opener, missing, eol)
    2.13          # shrink keywords read from working dir
    2.14          self.lines = kwt.shrinklines(self.fname, self.lines)
    2.15  
     3.1 --- a/mercurial/commands.py
     3.2 +++ b/mercurial/commands.py
     3.3 @@ -1764,7 +1764,7 @@
     3.4                  files = {}
     3.5                  try:
     3.6                      patch.patch(tmpname, ui, strip=strip, cwd=repo.root,
     3.7 -                                files=files)
     3.8 +                                files=files, eolmode=None)
     3.9                  finally:
    3.10                      files = patch.updatedir(ui, repo, files, similarity=sim/100.)
    3.11                  if not opts.get('no_commit'):
     4.1 --- a/mercurial/patch.py
     4.2 +++ b/mercurial/patch.py
     4.3 @@ -228,13 +228,42 @@
     4.4  
     4.5      return (dopatch, gitpatches)
     4.6  
     4.7 +class linereader:
     4.8 +    # simple class to allow pushing lines back into the input stream
     4.9 +    def __init__(self, fp, textmode=False):
    4.10 +        self.fp = fp
    4.11 +        self.buf = []
    4.12 +        self.textmode = textmode
    4.13 +
    4.14 +    def push(self, line):
    4.15 +        if line is not None:
    4.16 +            self.buf.append(line)
    4.17 +
    4.18 +    def readline(self):
    4.19 +        if self.buf:
    4.20 +            l = self.buf[0]
    4.21 +            del self.buf[0]
    4.22 +            return l
    4.23 +        l = self.fp.readline()
    4.24 +        if self.textmode and l.endswith('\r\n'):
    4.25 +            l = l[:-2] + '\n'
    4.26 +        return l
    4.27 +
    4.28 +    def __iter__(self):
    4.29 +        while 1:
    4.30 +            l = self.readline()
    4.31 +            if not l:
    4.32 +                break
    4.33 +            yield l
    4.34 +
    4.35  # @@ -start,len +start,len @@ or @@ -start +start @@ if len is 1
    4.36  unidesc = re.compile('@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))? @@')
    4.37  contextdesc = re.compile('(---|\*\*\*) (\d+)(,(\d+))? (---|\*\*\*)')
    4.38  
    4.39  class patchfile(object):
    4.40 -    def __init__(self, ui, fname, opener, missing=False):
    4.41 +    def __init__(self, ui, fname, opener, missing=False, eol=None):
    4.42          self.fname = fname
    4.43 +        self.eol = eol
    4.44          self.opener = opener
    4.45          self.ui = ui
    4.46          self.lines = []
    4.47 @@ -260,14 +289,20 @@
    4.48      def readlines(self, fname):
    4.49          fp = self.opener(fname, 'r')
    4.50          try:
    4.51 -            return fp.readlines()
    4.52 +            return list(linereader(fp, self.eol is not None))
    4.53          finally:
    4.54              fp.close()
    4.55  
    4.56      def writelines(self, fname, lines):
    4.57          fp = self.opener(fname, 'w')
    4.58          try:
    4.59 -            fp.writelines(lines)
    4.60 +            if self.eol and self.eol != '\n':
    4.61 +                for l in lines:
    4.62 +                    if l and l[-1] == '\n':
    4.63 +                        l = l[:1] + self.eol
    4.64 +                    fp.write(l)
    4.65 +            else:
    4.66 +                fp.writelines(lines)
    4.67          finally:
    4.68              fp.close()
    4.69  
    4.70 @@ -782,28 +817,6 @@
    4.71  
    4.72      return fname, missing
    4.73  
    4.74 -class linereader(object):
    4.75 -    # simple class to allow pushing lines back into the input stream
    4.76 -    def __init__(self, fp):
    4.77 -        self.fp = fp
    4.78 -        self.buf = []
    4.79 -
    4.80 -    def push(self, line):
    4.81 -        if line is not None:
    4.82 -            self.buf.append(line)
    4.83 -
    4.84 -    def readline(self):
    4.85 -        if self.buf:
    4.86 -            return self.buf.pop(0)
    4.87 -        return self.fp.readline()
    4.88 -
    4.89 -    def __iter__(self):
    4.90 -        while 1:
    4.91 -            l = self.readline()
    4.92 -            if not l:
    4.93 -                break
    4.94 -            yield l
    4.95 -
    4.96  def scangitpatch(lr, firstline):
    4.97      """
    4.98      Git patches can emit:
    4.99 @@ -824,19 +837,21 @@
   4.100          fp = lr.fp
   4.101      except IOError:
   4.102          fp = cStringIO.StringIO(lr.fp.read())
   4.103 -    gitlr = linereader(fp)
   4.104 +    gitlr = linereader(fp, lr.textmode)
   4.105      gitlr.push(firstline)
   4.106      (dopatch, gitpatches) = readgitpatch(gitlr)
   4.107      fp.seek(pos)
   4.108      return dopatch, gitpatches
   4.109  
   4.110 -def iterhunks(ui, fp, sourcefile=None):
   4.111 +def iterhunks(ui, fp, sourcefile=None, textmode=False):
   4.112      """Read a patch and yield the following events:
   4.113      - ("file", afile, bfile, firsthunk): select a new target file.
   4.114      - ("hunk", hunk): a new hunk is ready to be applied, follows a
   4.115      "file" event.
   4.116      - ("git", gitchanges): current diff is in git format, gitchanges
   4.117      maps filenames to gitpatch records. Unique event.
   4.118 +
   4.119 +    If textmode is True, input line-endings are normalized to LF.
   4.120      """
   4.121      changed = {}
   4.122      current_hunk = None
   4.123 @@ -850,7 +865,7 @@
   4.124      # our states
   4.125      BFILE = 1
   4.126      context = None
   4.127 -    lr = linereader(fp)
   4.128 +    lr = linereader(fp, textmode)
   4.129      dopatch = True
   4.130      # gitworkdone is True if a git operation (copy, rename, ...) was
   4.131      # performed already for the current file. Useful when the file
   4.132 @@ -954,17 +969,25 @@
   4.133      if hunknum == 0 and dopatch and not gitworkdone:
   4.134          raise NoHunks
   4.135  
   4.136 -def applydiff(ui, fp, changed, strip=1, sourcefile=None, reverse=False):
   4.137 -    """reads a patch from fp and tries to apply it.  The dict 'changed' is
   4.138 -       filled in with all of the filenames changed by the patch.  Returns 0
   4.139 -       for a clean patch, -1 if any rejects were found and 1 if there was
   4.140 -       any fuzz."""
   4.141 +def applydiff(ui, fp, changed, strip=1, sourcefile=None, reverse=False,
   4.142 +              eol=None):
   4.143 +    """
   4.144 +    Reads a patch from fp and tries to apply it. 
   4.145  
   4.146 +    The dict 'changed' is filled in with all of the filenames changed
   4.147 +    by the patch. Returns 0 for a clean patch, -1 if any rejects were
   4.148 +    found and 1 if there was any fuzz.
   4.149 +
   4.150 +    If 'eol' is None, the patch content and patched file are read in
   4.151 +    binary mode. Otherwise, line endings are ignored when patching then
   4.152 +    normalized to 'eol' (usually '\n' or \r\n').
   4.153 +    """
   4.154      rejects = 0
   4.155      err = 0
   4.156      current_file = None
   4.157      gitpatches = None
   4.158      opener = util.opener(os.getcwd())
   4.159 +    textmode = eol is not None
   4.160  
   4.161      def closefile():
   4.162          if not current_file:
   4.163 @@ -972,7 +995,7 @@
   4.164          current_file.close()
   4.165          return len(current_file.rej)
   4.166  
   4.167 -    for state, values in iterhunks(ui, fp, sourcefile):
   4.168 +    for state, values in iterhunks(ui, fp, sourcefile, textmode):
   4.169          if state == 'hunk':
   4.170              if not current_file:
   4.171                  continue
   4.172 @@ -987,11 +1010,11 @@
   4.173              afile, bfile, first_hunk = values
   4.174              try:
   4.175                  if sourcefile:
   4.176 -                    current_file = patchfile(ui, sourcefile, opener)
   4.177 +                    current_file = patchfile(ui, sourcefile, opener, eol=eol)
   4.178                  else:
   4.179                      current_file, missing = selectfile(afile, bfile, first_hunk,
   4.180                                              strip, reverse)
   4.181 -                    current_file = patchfile(ui, current_file, opener, missing)
   4.182 +                    current_file = patchfile(ui, current_file, opener, missing, eol)
   4.183              except PatchError, err:
   4.184                  ui.warn(str(err) + '\n')
   4.185                  current_file, current_hunk = None, None
   4.186 @@ -1104,9 +1127,17 @@
   4.187                           util.explain_exit(code)[0])
   4.188      return fuzz
   4.189  
   4.190 -def internalpatch(patchobj, ui, strip, cwd, files={}):
   4.191 +def internalpatch(patchobj, ui, strip, cwd, files={}, eolmode='strict'):
   4.192      """use builtin patch to apply <patchobj> to the working directory.
   4.193      returns whether patch was applied with fuzz factor."""
   4.194 +
   4.195 +    if eolmode is None:
   4.196 +        eolmode = ui.config('patch', 'eol', 'strict')
   4.197 +    try:
   4.198 +        eol = {'strict': None, 'crlf': '\r\n', 'lf': '\n'}[eolmode.lower()]
   4.199 +    except KeyError:
   4.200 +        raise util.Abort(_('Unsupported line endings type: %s') % eolmode)
   4.201 +            
   4.202      try:
   4.203          fp = file(patchobj, 'rb')
   4.204      except TypeError:
   4.205 @@ -1115,7 +1146,7 @@
   4.206          curdir = os.getcwd()
   4.207          os.chdir(cwd)
   4.208      try:
   4.209 -        ret = applydiff(ui, fp, files, strip=strip)
   4.210 +        ret = applydiff(ui, fp, files, strip=strip, eol=eol)
   4.211      finally:
   4.212          if cwd:
   4.213              os.chdir(curdir)
   4.214 @@ -1123,9 +1154,18 @@
   4.215          raise PatchError
   4.216      return ret > 0
   4.217  
   4.218 -def patch(patchname, ui, strip=1, cwd=None, files={}):
   4.219 -    """apply <patchname> to the working directory.
   4.220 -    returns whether patch was applied with fuzz factor."""
   4.221 +def patch(patchname, ui, strip=1, cwd=None, files={}, eolmode='strict'):
   4.222 +    """Apply <patchname> to the working directory.
   4.223 +
   4.224 +    'eolmode' specifies how end of lines should be handled. It can be:
   4.225 +    - 'strict': inputs are read in binary mode, EOLs are preserved
   4.226 +    - 'crlf': EOLs are ignored when patching and reset to CRLF
   4.227 +    - 'lf': EOLs are ignored when patching and reset to LF
   4.228 +    - None: get it from user settings, default to 'strict'
   4.229 +    'eolmode' is ignored when using an external patcher program.
   4.230 +
   4.231 +    Returns whether patch was applied with fuzz factor.
   4.232 +    """
   4.233      patcher = ui.config('ui', 'patch')
   4.234      args = []
   4.235      try:
   4.236 @@ -1134,7 +1174,7 @@
   4.237                                   files)
   4.238          else:
   4.239              try:
   4.240 -                return internalpatch(patchname, ui, strip, cwd, files)
   4.241 +                return internalpatch(patchname, ui, strip, cwd, files, eolmode)
   4.242              except NoHunks:
   4.243                  patcher = util.find_exe('gpatch') or util.find_exe('patch') or 'patch'
   4.244                  ui.debug(_('no valid hunks found; trying with %r instead\n') %
     5.1 new file mode 100755
     5.2 --- /dev/null
     5.3 +++ b/tests/test-import-eol
     5.4 @@ -0,0 +1,53 @@
     5.5 +#!/bin/sh
     5.6 +
     5.7 +cat > makepatch.py <<EOF
     5.8 +f = file('eol.diff', 'wb')
     5.9 +w = f.write
    5.10 +w('test message\n')
    5.11 +w('diff --git a/a b/a\n')
    5.12 +w('--- a/a\n')
    5.13 +w('+++ b/a\n')
    5.14 +w('@@ -1,5 +1,5 @@\n')
    5.15 +w(' a\n')
    5.16 +w('-b\r\n')
    5.17 +w('+y\r\n')
    5.18 +w(' c\r\n')
    5.19 +w(' d\n')
    5.20 +w('-e\n')
    5.21 +w('\ No newline at end of file\n')
    5.22 +w('+z\r\n')
    5.23 +w('\ No newline at end of file\r\n')
    5.24 +EOF
    5.25 +
    5.26 +hg init repo
    5.27 +cd repo
    5.28 +echo '\.diff' > .hgignore
    5.29 +
    5.30 +# Test different --eol values
    5.31 +python -c 'file("a", "wb").write("a\nb\nc\nd\ne")'
    5.32 +hg ci -Am adda
    5.33 +python ../makepatch.py
    5.34 +echo % invalid eol
    5.35 +hg --config patch.eol='LFCR' import eol.diff
    5.36 +hg revert -a
    5.37 +echo % force LF
    5.38 +hg --traceback --config patch.eol='LF' import eol.diff
    5.39 +python -c 'print repr(file("a","rb").read())'
    5.40 +hg st
    5.41 +echo % force CRLF
    5.42 +hg up -C 0
    5.43 +hg --traceback --config patch.eol='CRLF' import eol.diff
    5.44 +python -c 'print repr(file("a","rb").read())'
    5.45 +hg st
    5.46 +
    5.47 +# Test --eol and binary patches
    5.48 +python -c 'file("b", "wb").write("a\x00\nb")'
    5.49 +hg ci -Am addb
    5.50 +python -c 'file("b", "wb").write("a\x00\nc")'
    5.51 +hg diff --git > bin.diff
    5.52 +hg revert --no-backup b
    5.53 +echo % binary patch with --eol
    5.54 +hg import --config patch.eol='CRLF' -m changeb bin.diff
    5.55 +python -c 'print repr(file("b","rb").read())'
    5.56 +hg st
    5.57 +cd ..
     6.1 new file mode 100644
     6.2 --- /dev/null
     6.3 +++ b/tests/test-import-eol.out
     6.4 @@ -0,0 +1,16 @@
     6.5 +adding .hgignore
     6.6 +adding a
     6.7 +% invalid eol
     6.8 +applying eol.diff
     6.9 +abort: Unsupported line endings type: LFCR
    6.10 +% force LF
    6.11 +applying eol.diff
    6.12 +'a\ny\nc\nd\nz'
    6.13 +% force CRLF
    6.14 +1 files updated, 0 files merged, 0 files removed, 0 files unresolved
    6.15 +applying eol.diff
    6.16 +'a\r\ny\r\nc\r\nd\r\nz'
    6.17 +adding b
    6.18 +% binary patch with --eol
    6.19 +applying bin.diff
    6.20 +'a\x00\nc'

Contact: Thomas Arendsen Hein <hg@intevation.org> - Intevation GmbH