areguera / rpms / mailman

Forked from rpms/mailman 5 years ago
Clone

Blame SOURCES/mailman-2.1.12-dmarc.patch

ab2ccd
diff --git a/Mailman/Bouncers/SimpleMatch.py b/Mailman/Bouncers/SimpleMatch.py
ab2ccd
index a6a952a..0901b50 100644
ab2ccd
--- a/Mailman/Bouncers/SimpleMatch.py
ab2ccd
+++ b/Mailman/Bouncers/SimpleMatch.py
ab2ccd
@@ -42,7 +42,7 @@ PATTERNS = [
ab2ccd
     # sz-sb.de, corridor.com, nfg.nl
ab2ccd
     (_c('the following addresses had'),
ab2ccd
      _c('transcript of session follows'),
ab2ccd
-     _c(r'<(?P<fulladdr>[^>]*)>|\(expanded from: [^>)]*)>?\)')),
ab2ccd
+     _c(r'^ *(\(expanded from: )?[^\s@]+@[^\s@>]+?)>?\)?\s*$')),
ab2ccd
     # robanal.demon.co.uk
ab2ccd
     (_c('this message was created automatically by mail delivery software'),
ab2ccd
      _c('original message follows'),
ab2ccd
diff --git a/Mailman/Bouncers/Yahoo.py b/Mailman/Bouncers/Yahoo.py
ab2ccd
index b3edf4f..08ede54 100644
ab2ccd
--- a/Mailman/Bouncers/Yahoo.py
ab2ccd
+++ b/Mailman/Bouncers/Yahoo.py
ab2ccd
@@ -20,9 +20,15 @@ import re
ab2ccd
 import email
ab2ccd
 from email.Utils import parseaddr
ab2ccd
 
ab2ccd
-tcre = re.compile(r'message\s+from\s+yahoo\.\S+', re.IGNORECASE)
ab2ccd
+tcre = (re.compile(r'message\s+from\s+yahoo\.\S+', re.IGNORECASE),
ab2ccd
+        re.compile(r'Sorry, we were unable to deliver your message to '
ab2ccd
+                   r'the following address(\(es\))?\.',
ab2ccd
+                   re.IGNORECASE),
ab2ccd
+        )
ab2ccd
 acre = re.compile(r'<(?P<addr>[^>]*)>:')
ab2ccd
-ecre = re.compile(r'--- Original message follows')
ab2ccd
+ecre = (re.compile(r'--- Original message follows'),
ab2ccd
+        re.compile(r'--- Below this line is a copy of the message'),
ab2ccd
+        )
ab2ccd
 
ab2ccd
 
ab2ccd
 
ab2ccd
@@ -36,18 +42,26 @@ def process(msg):
ab2ccd
     # simple state machine
ab2ccd
     #     0 == nothing seen
ab2ccd
     #     1 == tag line seen
ab2ccd
+    #     2 == end line seen
ab2ccd
     state = 0
ab2ccd
     for line in email.Iterators.body_line_iterator(msg):
ab2ccd
         line = line.strip()
ab2ccd
-        if state == 0 and tcre.match(line):
ab2ccd
-            state = 1
ab2ccd
+        if state == 0:
ab2ccd
+            for cre in tcre:
ab2ccd
+                if cre.match(line):
ab2ccd
+                    state = 1
ab2ccd
+                    break
ab2ccd
         elif state == 1:
ab2ccd
             mo = acre.match(line)
ab2ccd
             if mo:
ab2ccd
                 addrs.append(mo.group('addr'))
ab2ccd
                 continue
ab2ccd
-            mo = ecre.match(line)
ab2ccd
-            if mo:
ab2ccd
-                # we're at the end of the error response
ab2ccd
-                break
ab2ccd
+            for cre in ecre:
ab2ccd
+                mo = cre.match(line)
ab2ccd
+                if mo:
ab2ccd
+                    # we're at the end of the error response
ab2ccd
+                    state = 2
ab2ccd
+                    break
ab2ccd
+        elif state == 2:
ab2ccd
+            break
ab2ccd
     return addrs
ab2ccd
diff --git a/Mailman/Defaults.py.in b/Mailman/Defaults.py.in
ab2ccd
old mode 100644
ab2ccd
new mode 100755
ab2ccd
index 4fe63db..8e42f54
ab2ccd
--- a/Mailman/Defaults.py.in
ab2ccd
+++ b/Mailman/Defaults.py.in
ab2ccd
@@ -505,6 +505,7 @@ GLOBAL_PIPELINE = [
ab2ccd
     # (outgoing) path, finally leaving the message in the outgoing queue.
ab2ccd
     'AfterDelivery',
ab2ccd
     'Acknowledge',
ab2ccd
+    'WrapMessage',
ab2ccd
     'ToOutgoing',
ab2ccd
     ]
ab2ccd
 
ab2ccd
@@ -914,6 +915,29 @@ DEFAULT_DEFAULT_MEMBER_MODERATION = No
ab2ccd
 # moderators?
ab2ccd
 DEFAULT_FORWARD_AUTO_DISCARDS = Yes
ab2ccd
 
ab2ccd
+# Shall dmarc_moderation_action be applied to messages From: domains with
ab2ccd
+# a DMARC policy of quarantine as well as reject?  This sets the default for
ab2ccd
+# the list setting that controls it.
ab2ccd
+DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION = Yes
ab2ccd
+
ab2ccd
+# Default action for posts whose From: address domain has a DMARC policy of
ab2ccd
+# reject or quarantine.  See DEFAULT_FROM_IS_LIST below.  Whatever is set as
ab2ccd
+# the default here precludes the list owner from setting a lower value.
ab2ccd
+# 0 = Accept
ab2ccd
+# 1 = Munge From
ab2ccd
+# 2 = Wrap Message
ab2ccd
+# 3 = Reject
ab2ccd
+# 4 = Discard
ab2ccd
+DEFAULT_DMARC_MODERATION_ACTION = 0
ab2ccd
+
ab2ccd
+# Parameters for DMARC DNS lookups. If you are seeing 'DNSException:
ab2ccd
+# Unable to query DMARC policy ...' entries in your error log, you may need
ab2ccd
+# to adjust these.
ab2ccd
+# The time to wait for a response from a name server before timeout.
ab2ccd
+DMARC_RESOLVER_TIMEOUT = seconds(3)
ab2ccd
+# The total time to spend trying to get an answer to the question.
ab2ccd
+DMARC_RESOLVER_LIFETIME = seconds(5)
ab2ccd
+
ab2ccd
 # What shold happen to non-member posts which are do not match explicit
ab2ccd
 # non-member actions?
ab2ccd
 # 0 = Accept
ab2ccd
@@ -950,6 +974,25 @@ DEFAULT_SEND_WELCOME_MSG = Yes
ab2ccd
 # Send goodbye messages to unsubscribed members?
ab2ccd
 DEFAULT_SEND_GOODBYE_MSG = Yes
ab2ccd
 
ab2ccd
+# Some list posts and mail to the -owner address may contain DomainKey or
ab2ccd
+# DomainKeys Identified Mail (DKIM) signature headers <http://www.dkim.org/>.
ab2ccd
+# Various list transformations to the message such as adding a list header or
ab2ccd
+# footer or scrubbing attachments or even reply-to munging can break these
ab2ccd
+# signatures.  It is generally felt that these signatures have value, even if
ab2ccd
+# broken and even if the outgoing message is resigned.  However, some sites
ab2ccd
+# may wish to remove these headers by setting this to Yes.
ab2ccd
+REMOVE_DKIM_HEADERS = No
ab2ccd
+
ab2ccd
+# The following is a three way setting.  It sets the default for the list's
ab2ccd
+# from_is_list policy which is applied to all posts except those for which a
ab2ccd
+# dmarc_moderation_action other than accept applies.
ab2ccd
+# 0 -> Do not rewrite the From: or wrap the message.
ab2ccd
+# 1 -> Rewrite the From: header of posts replacing the posters address with
ab2ccd
+#      that of the list.  Also see REMOVE_DKIM_HEADERS above.
ab2ccd
+# 2 -> Do not modify the From: of the message, but wrap the message in an outer
ab2ccd
+#      message From the list address.
ab2ccd
+DEFAULT_FROM_IS_LIST = 0
ab2ccd
+
ab2ccd
 # Wipe sender information, and make it look like the list-admin
ab2ccd
 # address sends all messages
ab2ccd
 DEFAULT_ANONYMOUS_LIST = No
ab2ccd
diff --git a/Mailman/Gui/General.py b/Mailman/Gui/General.py
ab2ccd
index 8271a30..05dc9ba 100644
ab2ccd
--- a/Mailman/Gui/General.py
ab2ccd
+++ b/Mailman/Gui/General.py
ab2ccd
@@ -153,6 +153,72 @@ class General(GUIBase):
ab2ccd
                             (listname %%05d) -> (listname 00123)
ab2ccd
              """)),
ab2ccd
 
ab2ccd
+            ('from_is_list', mm_cfg.Radio,
ab2ccd
+             (_('No'), _('Munge From'), _('Wrap Message')), 0,
ab2ccd
+             _("""Replace the From: header address with the list's posting
ab2ccd
+             address to mitigate issues stemming from the original From:
ab2ccd
+             domain's DMARC or similar policies."""),
ab2ccd
+             _("""Several protocols now in wide use attempt to ensure that use
ab2ccd
+             of the domain in the author's address (ie, in the From: header
ab2ccd
+             field) is authorized by that domain.  These protocols may be
ab2ccd
+             incompatible with common list features such as footers, causing
ab2ccd
+             participating email services to bounce list traffic merely
ab2ccd
+             because of the address in the From: field.  This has resulted
ab2ccd
+             in members being unsubscribed despite being perfectly able to
ab2ccd
+             receive mail.
ab2ccd
+             

ab2ccd
+             The following actions are applied to all list messages when
ab2ccd
+             selected here.  To apply these actions only to messages where the
ab2ccd
+             domain in the From: header is determined to use such a protocol,
ab2ccd
+             see the 
ab2ccd
+             href="?VARHELP=privacy/sender/dmarc_moderation_action">
ab2ccd
+             dmarc_moderation_action settings under Privacy options...
ab2ccd
+             -> Sender filters.
ab2ccd
+             

Settings:

ab2ccd
+             
ab2ccd
+             
No
ab2ccd
+             
Do nothing special. This is appropriate for anonymous lists.
ab2ccd
+             It is appropriate for dedicated announcement lists, unless the
ab2ccd
+             From: address of authorized posters might be in a domain with a
ab2ccd
+             DMARC or similar policy. It is also appropriate if you choose to
ab2ccd
+             use dmarc_moderation_action other than Accept for this list.
ab2ccd
+             
Munge From
ab2ccd
+             
This action replaces the poster's address in the From: header
ab2ccd
+             with the list's posting address and adds the poster's address to
ab2ccd
+             the addresses in the original Reply-To: header.
ab2ccd
+             
Wrap Message
ab2ccd
+             
Just wrap the message in an outer message with the From:
ab2ccd
+             header containing the list's posting address and with the original
ab2ccd
+             From: address added to the addresses in the original Reply-To:
ab2ccd
+             header and with Content-Type: message/rfc822.  This is effectively
ab2ccd
+             a one message MIME format digest.
ab2ccd
+             
ab2ccd
+             

The transformations for anonymous_list are applied before

ab2ccd
+             any of these actions. It is not useful to apply actions other
ab2ccd
+             than No to an anonymous list, and if you do so, the result may
ab2ccd
+             be surprising.
ab2ccd
+             

The Reply-To: header munging actions below interact with these

ab2ccd
+             actions as follows:
ab2ccd
+             

first_strip_reply_to = Yes will remove all the incoming

ab2ccd
+             Reply-To: addresses but will still add the poster's address to
ab2ccd
+             Reply-To: for all three settings of reply_goes_to_list which
ab2ccd
+             respectively will result in just the poster's address, the
ab2ccd
+             poster's address and the list posting address or the poster's
ab2ccd
+             address and the explicit reply_to_address in the outgoing
ab2ccd
+             Reply-To: header. If first_strip_reply_to = No the poster's
ab2ccd
+             address in the original From: header, if not already included in
ab2ccd
+             the Reply-To:, will be added to any existing Reply-To:
ab2ccd
+             address(es).
ab2ccd
+             

These actions, whether selected here or via

ab2ccd
+             href="?VARHELP=privacy/sender/dmarc_moderation_action">
ab2ccd
+             dmarc_moderation_action, do not apply to messages in digests
ab2ccd
+             or archives or sent to usenet via the Mail<->News gateways.
ab2ccd
+             

If

ab2ccd
+             href="?VARHELP=privacy/sender/dmarc_moderation_action">
ab2ccd
+             dmarc_moderation_action applies to this message with an
ab2ccd
+             action other than Accept, that action rather than this is
ab2ccd
+             applied""")),
ab2ccd
+
ab2ccd
             ('anonymous_list', mm_cfg.Radio, (_('No'), _('Yes')), 0,
ab2ccd
              _("""Hide the sender of a message, replacing it with the list
ab2ccd
              address (Removes From, Sender and Reply-To fields)""")),
ab2ccd
diff --git a/Mailman/Gui/NonDigest.py b/Mailman/Gui/NonDigest.py
ab2ccd
old mode 100644
ab2ccd
new mode 100755
ab2ccd
diff --git a/Mailman/Gui/Privacy.py b/Mailman/Gui/Privacy.py
ab2ccd
index 75eff2b..5d717bb 100644
ab2ccd
--- a/Mailman/Gui/Privacy.py
ab2ccd
+++ b/Mailman/Gui/Privacy.py
ab2ccd
@@ -158,6 +158,11 @@ class Privacy(GUIBase):
ab2ccd
             ]
ab2ccd
 
ab2ccd
         adminurl = mlist.GetScriptURL('admin', absolute=1)
ab2ccd
+    
ab2ccd
+        if mlist.dmarc_quarantine_moderation_action:
ab2ccd
+            quarantine = _('/Quarantine')
ab2ccd
+        else:
ab2ccd
+            quarantine = ''
ab2ccd
         sender_rtn = [
ab2ccd
             _("""When a message is posted to the list, a series of
ab2ccd
             moderation steps are taken to decide whether a moderator must
ab2ccd
@@ -235,6 +240,59 @@ class Privacy(GUIBase):
ab2ccd
              >rejection notice to
ab2ccd
              be sent to moderated members who post to this list.""")),
ab2ccd
 
ab2ccd
+            ('dmarc_moderation_action', mm_cfg.Radio,
ab2ccd
+             (_('Accept'), _('Munge From'), _('Wrap Message'), _('Reject'),
ab2ccd
+                 _('Discard')), 0,
ab2ccd
+             _("""Action to take when anyone posts to the
ab2ccd
+             list from a domain with a DMARC Reject%(quarantine)s Policy."""),
ab2ccd
+
ab2ccd
+             _("""
  • Munge From -- applies the
ab2ccd
+             href="?VARHELP=general/from_is_list">from_is_list Munge From
ab2ccd
+             transformation to these messages.
ab2ccd
+
ab2ccd
+             

  • Wrap Message -- applies the
  • ab2ccd
    +             href="?VARHELP=general/from_is_list">from_is_list Wrap
    ab2ccd
    +             Message transformation to these messages.
    ab2ccd
    +
    ab2ccd
    +             

  • Reject -- this automatically rejects the message by
  • ab2ccd
    +             sending a bounce notice to the post's author.  The text of the
    ab2ccd
    +             bounce notice can be 
    ab2ccd
    +             href="?VARHELP=privacy/sender/dmarc_moderation_notice"
    ab2ccd
    +             >configured by you.
    ab2ccd
    +
    ab2ccd
    +             

  • Discard -- this simply discards the message, with
  • ab2ccd
    +             no notice sent to the post's author.
    ab2ccd
    +             
    ab2ccd
    +
    ab2ccd
    +             

    This setting takes precedence over the

    ab2ccd
    +             href="?VARHELP=general/from_is_list"> from_is_list setting
    ab2ccd
    +             if the message is From: an affected domain and the setting is
    ab2ccd
    +             other than Accept.""")),
    ab2ccd
    +
    ab2ccd
    +            ('dmarc_quarantine_moderation_action', mm_cfg.Radio,
    ab2ccd
    +             (_('No'), _('Yes')), 0,
    ab2ccd
    +             _("""Shall the above dmarc_moderation_action apply to messages
    ab2ccd
    +               From: domains with DMARC p=quarantine as well as p=reject"""),
    ab2ccd
    +
    ab2ccd
    +             _("""
    • No -- this applies dmarc_moderation_action to
    ab2ccd
    +               only those posts From: a domain with DMARC p=reject.  This is
    ab2ccd
    +               appropriate if you are concerned about bounced messages, but
    ab2ccd
    +               want to apply dmarc_moderation_action to as few messages as
    ab2ccd
    +               possible.
    ab2ccd
    +               

  • Yes -- this applies dmarc_moderation_action to
  • ab2ccd
    +               posts From: a domain with DMARC p=reject or p=quarantine.
    ab2ccd
    +               

    If a message is From: a domain with DMARC p=quarantine

    ab2ccd
    +               and dmarc_moderation_action is not applied (this set to No)
    ab2ccd
    +               the message will likely not bounce, but will be delivered to
    ab2ccd
    +               recipients' spam folders or other hard to find places.""")),
    ab2ccd
    +
    ab2ccd
    +            ('dmarc_moderation_notice', mm_cfg.Text, (10, WIDTH), 1,
    ab2ccd
    +             _("""Text to include in any
    ab2ccd
    +             
    ab2ccd
    +             >rejection notice to
    ab2ccd
    +             be sent to anyone who posts to this list from a domain
    ab2ccd
    +             with a DMARC Reject%(quarantine)s Policy.""")),
    ab2ccd
    +
    ab2ccd
                 _('Non-member filters'),
    ab2ccd
     
    ab2ccd
                 ('accept_these_nonmembers', mm_cfg.EmailListEx, (10, WIDTH), 1,
    ab2ccd
    @@ -399,7 +457,7 @@ class Privacy(GUIBase):
    ab2ccd
                  case, each rule is matched in turn, with processing stopped after
    ab2ccd
                  the first match.
    ab2ccd
     
    ab2ccd
    -             Note that headers are collected from all the attachments 
    ab2ccd
    +             Note that headers are collected from all the attachments
    ab2ccd
                  (except for the mailman administrivia message) and
    ab2ccd
                  matched against the regular expressions. With this feature,
    ab2ccd
                  you can effectively sort out messages with dangerous file
    ab2ccd
    @@ -442,6 +500,11 @@ class Privacy(GUIBase):
    ab2ccd
             # an option.
    ab2ccd
             if property == 'subscribe_policy' and not mm_cfg.ALLOW_OPEN_SUBSCRIBE:
    ab2ccd
                 val += 1
    ab2ccd
    +        if (property == 'dmarc_moderation_action' and
    ab2ccd
    +                val < mm_cfg.DEFAULT_DMARC_MODERATION_ACTION):
    ab2ccd
    +            doc.addError(_("""dmarc_moderation_action must be >= the configured
    ab2ccd
    +                           default value."""))
    ab2ccd
    +            val = mm_cfg.DEFAULT_DMARC_MODERATION_ACTION
    ab2ccd
             setattr(mlist, property, val)
    ab2ccd
     
    ab2ccd
         # We need to handle the header_filter_rules widgets specially, but
    ab2ccd
    diff --git a/Mailman/Handlers/AvoidDuplicates.py b/Mailman/Handlers/AvoidDuplicates.py
    ab2ccd
    index 038034c..549d8e7 100644
    ab2ccd
    --- a/Mailman/Handlers/AvoidDuplicates.py
    ab2ccd
    +++ b/Mailman/Handlers/AvoidDuplicates.py
    ab2ccd
    @@ -24,6 +24,7 @@ warning header, or pass it through, depending on the user's preferences.
    ab2ccd
     
    ab2ccd
     from email.Utils import getaddresses, formataddr
    ab2ccd
     from Mailman import mm_cfg
    ab2ccd
    +from Mailman.Handlers.CookHeaders import change_header
    ab2ccd
     
    ab2ccd
     COMMASPACE = ', '
    ab2ccd
     
    ab2ccd
    @@ -95,6 +96,10 @@ def process(mlist, msg, msgdata):
    ab2ccd
         # Set the new list of recipients
    ab2ccd
         msgdata['recips'] = newrecips
    ab2ccd
         # RFC 2822 specifies zero or one CC header
    ab2ccd
    -    del msg['cc']
    ab2ccd
         if ccaddrs:
    ab2ccd
    -        msg['Cc'] = COMMASPACE.join([formataddr(i) for i in ccaddrs.values()])
    ab2ccd
    +        change_header('Cc',
    ab2ccd
    +        COMMASPACE.join([formataddr(i) for i in ccaddrs.values()]),
    ab2ccd
    +        mlist, msg, msgdata)
    ab2ccd
    +    else:
    ab2ccd
    +        del msg['cc']
    ab2ccd
    +
    ab2ccd
    diff --git a/Mailman/Handlers/CookHeaders.py b/Mailman/Handlers/CookHeaders.py
    ab2ccd
    old mode 100644
    ab2ccd
    new mode 100755
    ab2ccd
    index 8e7e668..c556967
    ab2ccd
    --- a/Mailman/Handlers/CookHeaders.py
    ab2ccd
    +++ b/Mailman/Handlers/CookHeaders.py
    ab2ccd
    @@ -64,13 +64,23 @@ def uheader(mlist, s, header_name=None, continuation_ws='\t', maxlinelen=None):
    ab2ccd
             charset = 'us-ascii'
    ab2ccd
         return Header(s, charset, maxlinelen, header_name, continuation_ws)
    ab2ccd
     
    ab2ccd
    +def change_header(name, value, mlist, msg, msgdata, delete=True, repl=True):
    ab2ccd
    +    if ((msgdata.get('from_is_list') == 2 or
    ab2ccd
    +        (msgdata.get('from_is_list') == 0 and mlist.from_is_list == 2)) and 
    ab2ccd
    +        not msgdata.get('_fasttrack')
    ab2ccd
    +       ) or name.lower() in ('from', 'reply-to'):
    ab2ccd
    +        msgdata.setdefault('add_header', {})[name] = value
    ab2ccd
    +    elif repl or not msg.has_key(name):
    ab2ccd
    +        if delete:
    ab2ccd
    +            del msg[name]
    ab2ccd
    +        msg[name] = value
    ab2ccd
    +
    ab2ccd
     
    ab2ccd
     
    ab2ccd
     def process(mlist, msg, msgdata):
    ab2ccd
         # Set the "X-Ack: no" header if noack flag is set.
    ab2ccd
         if msgdata.get('noack'):
    ab2ccd
    -        del msg['x-ack']
    ab2ccd
    -        msg['X-Ack'] = 'no'
    ab2ccd
    +        change_header('X-Ack', 'no', mlist, msg, msgdata)
    ab2ccd
         # Because we're going to modify various important headers in the email
    ab2ccd
         # message, we want to save some of the information in the msgdata
    ab2ccd
         # dictionary for later.  Specifically, the sender header will get waxed,
    ab2ccd
    @@ -87,7 +97,8 @@ def process(mlist, msg, msgdata):
    ab2ccd
                 pass
    ab2ccd
         # Mark message so we know we've been here, but leave any existing
    ab2ccd
         # X-BeenThere's intact.
    ab2ccd
    -    msg['X-BeenThere'] = mlist.GetListEmail()
    ab2ccd
    +    change_header('X-BeenThere', mlist.GetListEmail(),
    ab2ccd
    +                  mlist, msg, msgdata, delete=False)
    ab2ccd
         # Add Precedence: and other useful headers.  None of these are standard
    ab2ccd
         # and finding information on some of them are fairly difficult.  Some are
    ab2ccd
         # just common practice, and we'll add more here as they become necessary.
    ab2ccd
    @@ -101,12 +112,31 @@ def process(mlist, msg, msgdata):
    ab2ccd
         # known exploits in a particular version of Mailman and we know a site is
    ab2ccd
         # using such an old version, they may be vulnerable.  It's too easy to
    ab2ccd
         # edit the code to add a configuration variable to handle this.
    ab2ccd
    -    if not msg.has_key('x-mailman-version'):
    ab2ccd
    -        msg['X-Mailman-Version'] = mm_cfg.VERSION
    ab2ccd
    +    change_header('X-Mailman-Version', mm_cfg.VERSION,
    ab2ccd
    +                  mlist, msg, msgdata, repl=False)
    ab2ccd
         # We set "Precedence: list" because this is the recommendation from the
    ab2ccd
         # sendmail docs, the most authoritative source of this header's semantics.
    ab2ccd
    -    if not msg.has_key('precedence'):
    ab2ccd
    -        msg['Precedence'] = 'list'
    ab2ccd
    +    change_header('Precedence', 'list',
    ab2ccd
    +                  mlist, msg, msgdata, repl=False)
    ab2ccd
    +    # Do we change the from so the list takes ownership of the email
    ab2ccd
    +    if (msgdata.get('from_is_list') or mlist.from_is_list) and not fasttrack:
    ab2ccd
    +        realname, email = parseaddr(msg['from'])
    ab2ccd
    +        if not realname:
    ab2ccd
    +            if mlist.isMember(email):
    ab2ccd
    +                realname = mlist.getMemberName(email) or email
    ab2ccd
    +            else:
    ab2ccd
    +                realname = email
    ab2ccd
    +        # Remove domain from realname if it looks like an email address
    ab2ccd
    +        realname = re.sub(r'@([^ .]+\.)+[^ .]+$', '---', realname)
    ab2ccd
    +        # Remember the original From: here for adding to Reply-To: below.
    ab2ccd
    +        o_from = parseaddr(msg['from'])
    ab2ccd
    +        change_header('From',
    ab2ccd
    +                      formataddr(('%s via %s' % (realname, mlist.real_name),
    ab2ccd
    +                                 mlist.GetListEmail())),
    ab2ccd
    +                      mlist, msg, msgdata)
    ab2ccd
    +    else:
    ab2ccd
    +        # Use this as a flag
    ab2ccd
    +        o_from = None
    ab2ccd
         # Reply-To: munging.  Do not do this if the message is "fast tracked",
    ab2ccd
         # meaning it is internally crafted and delivered to a specific user.  BAW:
    ab2ccd
         # Yuck, I really hate this feature but I've caved under the sheer pressure
    ab2ccd
    @@ -136,18 +166,23 @@ def process(mlist, msg, msgdata):
    ab2ccd
                 orig = msg.get_all('reply-to', [])
    ab2ccd
                 for pair in getaddresses(orig):
    ab2ccd
                     add(pair)
    ab2ccd
    +        # We also need to put the old From: in Reply-To: in all cases.
    ab2ccd
    +        if o_from:
    ab2ccd
    +            add(o_from)
    ab2ccd
             # Set Reply-To: header to point back to this list.  Add this last
    ab2ccd
             # because some folks think that some MUAs make it easier to delete
    ab2ccd
             # addresses from the right than from the left.
    ab2ccd
             if mlist.reply_goes_to_list == 1:
    ab2ccd
                 i18ndesc = uheader(mlist, mlist.description, 'Reply-To')
    ab2ccd
                 add((str(i18ndesc), mlist.GetListEmail()))
    ab2ccd
    -        del msg['reply-to']
    ab2ccd
             # Don't put Reply-To: back if there's nothing to add!
    ab2ccd
             if new:
    ab2ccd
                 # Preserve order
    ab2ccd
    -            msg['Reply-To'] = COMMASPACE.join(
    ab2ccd
    -                [formataddr(pair) for pair in new])
    ab2ccd
    +            change_header('Reply-To', 
    ab2ccd
    +                          COMMASPACE.join([formataddr(pair) for pair in new]),
    ab2ccd
    +                          mlist, msg, msgdata)
    ab2ccd
    +        else:
    ab2ccd
    +            del msg['reply-to']
    ab2ccd
             # The To field normally contains the list posting address.  However
    ab2ccd
             # when messages are fully personalized, that header will get
    ab2ccd
             # overwritten with the address of the recipient.  We need to get the
    ab2ccd
    @@ -158,18 +193,31 @@ def process(mlist, msg, msgdata):
    ab2ccd
             # above code?
    ab2ccd
             # Also skip Cc if this is an anonymous list as list posting address
    ab2ccd
             # is already in From and Reply-To in this case.
    ab2ccd
    -        if mlist.personalize == 2 and mlist.reply_goes_to_list <> 1 \
    ab2ccd
    -           and not mlist.anonymous_list:
    ab2ccd
    +        # We do add the Cc in cases where From: header munging is being done
    ab2ccd
    +        # because even though the list address is in From:, the Reply-To:
    ab2ccd
    +        # poster will override it. Brain dead MUAs may then address the list
    ab2ccd
    +        # twice on a 'reply all', but reasonable MUAs should do the right
    ab2ccd
    +        # thing.
    ab2ccd
    +        if (mlist.personalize == 2 and mlist.reply_goes_to_list <> 1 and
    ab2ccd
    +            not mlist.anonymous_list):
    ab2ccd
                 # Watch out for existing Cc headers, merge, and remove dups.  Note
    ab2ccd
                 # that RFC 2822 says only zero or one Cc header is allowed.
    ab2ccd
                 new = []
    ab2ccd
                 d = {}
    ab2ccd
    -            for pair in getaddresses(msg.get_all('cc', [])):
    ab2ccd
    -                add(pair)
    ab2ccd
    +            # AvoidDuplicates may have set a new Cc: in msgdata.add_header,
    ab2ccd
    +            # so check that.
    ab2ccd
    +            if (msgdata.has_key('add_header') and
    ab2ccd
    +                    msgdata['add_header'].has_key('Cc')):
    ab2ccd
    +                for pair in getaddresses([msgdata['add_header']['Cc']]):
    ab2ccd
    +                    add(pair)
    ab2ccd
    +            else:
    ab2ccd
    +                for pair in getaddresses(msg.get_all('cc', [])):
    ab2ccd
    +                    add(pair)
    ab2ccd
                 i18ndesc = uheader(mlist, mlist.description, 'Cc')
    ab2ccd
                 add((str(i18ndesc), mlist.GetListEmail()))
    ab2ccd
    -            del msg['Cc']
    ab2ccd
    -            msg['Cc'] = COMMASPACE.join([formataddr(pair) for pair in new])
    ab2ccd
    +            change_header('Cc',
    ab2ccd
    +                          COMMASPACE.join([formataddr(pair) for pair in new]),
    ab2ccd
    +                          mlist, msg, msgdata)
    ab2ccd
         # Add list-specific headers as defined in RFC 2369 and RFC 2919, but only
    ab2ccd
         # if the message is being crafted for a specific list (e.g. not for the
    ab2ccd
         # password reminders).
    ab2ccd
    @@ -191,8 +239,7 @@ def process(mlist, msg, msgdata):
    ab2ccd
             # without desc we need to ensure the MUST brackets
    ab2ccd
             listid_h = '<%s>' % listid
    ab2ccd
         # We always add a List-ID: header.
    ab2ccd
    -    del msg['list-id']
    ab2ccd
    -    msg['List-Id'] = listid_h
    ab2ccd
    +    change_header('List-Id', listid_h, mlist, msg, msgdata)
    ab2ccd
         # For internally crafted messages, we also add a (nonstandard),
    ab2ccd
         # "X-List-Administrivia: yes" header.  For all others (i.e. those coming
    ab2ccd
         # from list posts), we add a bunch of other RFC 2369 headers.
    ab2ccd
    @@ -219,13 +266,12 @@ def process(mlist, msg, msgdata):
    ab2ccd
         # First we delete any pre-existing headers because the RFC permits only
    ab2ccd
         # one copy of each, and we want to be sure it's ours.
    ab2ccd
         for h, v in headers.items():
    ab2ccd
    -        del msg[h]
    ab2ccd
             # Wrap these lines if they are too long.  78 character width probably
    ab2ccd
             # shouldn't be hardcoded, but is at least text-MUA friendly.  The
    ab2ccd
             # adding of 2 is for the colon-space separator.
    ab2ccd
             if len(h) + 2 + len(v) > 78:
    ab2ccd
                 v = CONTINUATION.join(v.split(', '))
    ab2ccd
    -        msg[h] = v
    ab2ccd
    +        change_header(h, v, mlist, msg, msgdata)
    ab2ccd
     
    ab2ccd
     
    ab2ccd
     
    ab2ccd
    @@ -302,8 +348,7 @@ def prefix_subject(mlist, msg, msgdata):
    ab2ccd
                         h = u' '.join([prefix, subject])
    ab2ccd
                 h = h.encode('us-ascii')
    ab2ccd
                 h = uheader(mlist, h, 'Subject', continuation_ws=ws)
    ab2ccd
    -            del msg['subject']
    ab2ccd
    -            msg['Subject'] = h
    ab2ccd
    +            change_header('Subject', h, mlist, msg, msgdata)
    ab2ccd
                 ss = u' '.join([recolon, subject])
    ab2ccd
                 ss = ss.encode('us-ascii')
    ab2ccd
                 ss = uheader(mlist, ss, 'Subject', continuation_ws=ws)
    ab2ccd
    @@ -321,8 +366,7 @@ def prefix_subject(mlist, msg, msgdata):
    ab2ccd
         # TK: Subject is concatenated and unicode string.
    ab2ccd
         subject = subject.encode(cset, 'replace')
    ab2ccd
         h.append(subject, cset)
    ab2ccd
    -    del msg['subject']
    ab2ccd
    -    msg['Subject'] = h
    ab2ccd
    +    change_header('Subject', h, mlist, msg, msgdata)
    ab2ccd
         ss = uheader(mlist, recolon, 'Subject', continuation_ws=ws)
    ab2ccd
         ss.append(subject, cset)
    ab2ccd
         msgdata['stripped_subject'] = ss
    ab2ccd
    diff --git a/Mailman/Handlers/Moderate.py b/Mailman/Handlers/Moderate.py
    ab2ccd
    index a362d96..2f1f38f 100644
    ab2ccd
    --- a/Mailman/Handlers/Moderate.py
    ab2ccd
    +++ b/Mailman/Handlers/Moderate.py
    ab2ccd
    @@ -21,6 +21,7 @@
    ab2ccd
     import re
    ab2ccd
     from email.MIMEMessage import MIMEMessage
    ab2ccd
     from email.MIMEText import MIMEText
    ab2ccd
    +from email.Utils import parseaddr
    ab2ccd
     
    ab2ccd
     from Mailman import mm_cfg
    ab2ccd
     from Mailman import Utils
    ab2ccd
    @@ -47,9 +48,34 @@ class ModeratedMemberPost(Hold.ModeratedPost):
    ab2ccd
     
    ab2ccd
     
    ab2ccd
     def process(mlist, msg, msgdata):
    ab2ccd
    -    if msgdata.get('approved') or msgdata.get('fromusenet'):
    ab2ccd
    +    if msgdata.get('approved'):
    ab2ccd
             return
    ab2ccd
    -    # First of all, is the poster a member or not?
    ab2ccd
    +    # Before anything else, check DMARC if necessary.
    ab2ccd
    +    msgdata['from_is_list'] = 0
    ab2ccd
    +    dn, addr = parseaddr(msg.get('from'))
    ab2ccd
    +    if addr and mlist.dmarc_moderation_action > 0:
    ab2ccd
    +        if Utils.IsDMARCProhibited(mlist, addr):
    ab2ccd
    +            # Note that for dmarc_moderation_action, 0 = Accept, 
    ab2ccd
    +            #    1 = Munge, 2 = Wrap, 3 = Reject, 4 = Discard
    ab2ccd
    +            if mlist.dmarc_moderation_action == 1:
    ab2ccd
    +                msgdata['from_is_list'] = 1
    ab2ccd
    +            elif mlist.dmarc_moderation_action == 2:
    ab2ccd
    +                msgdata['from_is_list'] = 2
    ab2ccd
    +            elif mlist.dmarc_moderation_action == 3:
    ab2ccd
    +                # Reject
    ab2ccd
    +                text = mlist.dmarc_moderation_notice
    ab2ccd
    +                if text:
    ab2ccd
    +                    text = Utils.wrap(text)
    ab2ccd
    +                else:
    ab2ccd
    +                    text = Utils.wrap(_(
    ab2ccd
    +"""You are not allowed to post to this mailing list From: a domain which
    ab2ccd
    +publishes a DMARC policy of reject or quarantine, and your message has been
    ab2ccd
    +automatically rejected.  If you think that your messages are being rejected in
    ab2ccd
    +error, contact the mailing list owner at %(listowner)s."""))
    ab2ccd
    +                raise Errors.RejectMessage, text
    ab2ccd
    +            elif mlist.dmarc_moderation_action == 4:
    ab2ccd
    +                raise Errors.DiscardMessage
    ab2ccd
    +    # Then, is the poster a member or not?
    ab2ccd
         for sender in msg.get_senders():
    ab2ccd
             if mlist.isMember(sender):
    ab2ccd
                 break
    ab2ccd
    @@ -105,7 +131,7 @@ def process(mlist, msg, msgdata):
    ab2ccd
         # moderation configuration variables.  Handle by way of generic non-member
    ab2ccd
         # action.
    ab2ccd
         assert 0 <= mlist.generic_nonmember_action <= 4
    ab2ccd
    -    if mlist.generic_nonmember_action == 0:
    ab2ccd
    +    if mlist.generic_nonmember_action == 0 or msgdata.get('fromusenet'):
    ab2ccd
             # Accept
    ab2ccd
             return
    ab2ccd
         elif mlist.generic_nonmember_action == 1:
    ab2ccd
    diff --git a/Mailman/Handlers/Tagger.py b/Mailman/Handlers/Tagger.py
    ab2ccd
    index 0d3ce49..2117290 100644
    ab2ccd
    --- a/Mailman/Handlers/Tagger.py
    ab2ccd
    +++ b/Mailman/Handlers/Tagger.py
    ab2ccd
    @@ -24,6 +24,7 @@ import email.Iterators
    ab2ccd
     
    ab2ccd
     from Mailman import Utils
    ab2ccd
     from Mailman.Logging.Syslog import syslog
    ab2ccd
    +from Mailman.Handlers.CookHeaders import change_header
    ab2ccd
     
    ab2ccd
     CRNL = '\r\n'
    ab2ccd
     EMPTYSTRING = ''
    ab2ccd
    @@ -60,8 +61,9 @@ def process(mlist, msg, msgdata):
    ab2ccd
                     break
    ab2ccd
         if hits:
    ab2ccd
             msgdata['topichits'] = hits.keys()
    ab2ccd
    -        msg['X-Topics'] = NLTAB.join(hits.keys())
    ab2ccd
    -    
    ab2ccd
    +        change_header('X-Topics', NLTAB.join(hits.keys()),
    ab2ccd
    +                      mlist, msg, msgdata, delete=False)
    ab2ccd
    +
    ab2ccd
     
    ab2ccd
     
    ab2ccd
     def scanbody(msg, numlines=None):
    ab2ccd
    diff --git a/Mailman/Handlers/WrapMessage.py b/Mailman/Handlers/WrapMessage.py
    ab2ccd
    new file mode 100644
    ab2ccd
    index 0000000..9678f6f
    ab2ccd
    --- /dev/null
    ab2ccd
    +++ b/Mailman/Handlers/WrapMessage.py
    ab2ccd
    @@ -0,0 +1,72 @@
    ab2ccd
    +# Copyright (C) 2013-2014 by the Free Software Foundation, Inc.
    ab2ccd
    +#
    ab2ccd
    +# This program is free software; you can redistribute it and/or
    ab2ccd
    +# modify it under the terms of the GNU General Public License
    ab2ccd
    +# as published by the Free Software Foundation; either version 2
    ab2ccd
    +# of the License, or (at your option) any later version.
    ab2ccd
    +#
    ab2ccd
    +# This program is distributed in the hope that it will be useful,
    ab2ccd
    +# but WITHOUT ANY WARRANTY; without even the implied warranty of
    ab2ccd
    +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    ab2ccd
    +# GNU General Public License for more details.
    ab2ccd
    +#
    ab2ccd
    +# You should have received a copy of the GNU General Public License
    ab2ccd
    +# along with this program; if not, write to the Free Software
    ab2ccd
    +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
    ab2ccd
    +# USA.
    ab2ccd
    +
    ab2ccd
    +"""Wrap the message in an outer message/rfc822 part and transfer/add
    ab2ccd
    +some headers from the original.
    ab2ccd
    +
    ab2ccd
    +Also, in the case of Munge From, replace the From: and Reply-To: in the
    ab2ccd
    +original message.
    ab2ccd
    +"""
    ab2ccd
    +
    ab2ccd
    +import copy
    ab2ccd
    +
    ab2ccd
    +from Mailman import mm_cfg
    ab2ccd
    +from Mailman.Utils import unique_message_id
    ab2ccd
    +from Mailman.Message import Message
    ab2ccd
    +
    ab2ccd
    +# Headers from the original that we want to keep in the wrapper.
    ab2ccd
    +KEEPERS = ('to',
    ab2ccd
    +           'in-reply-to',
    ab2ccd
    +           'references',
    ab2ccd
    +           'x-mailman-approved-at',
    ab2ccd
    +          )
    ab2ccd
    +
    ab2ccd
    +
    ab2ccd
    +
    ab2ccd
    +def process(mlist, msg, msgdata):
    ab2ccd
    +    # This is the negation of we're wrapping because dmarc_moderation_action
    ab2ccd
    +    # is wrap this message or from_is_list applies and is wrap.
    ab2ccd
    +    if not (msgdata.get('from_is_list') == 2 or
    ab2ccd
    +            (mlist.from_is_list == 2 and msgdata.get('from_is_list') == 0)):
    ab2ccd
    +        # Now see if we need to add a From: and/or Reply-To: without wrapping.
    ab2ccd
    +        a_h = msgdata.get('add_header')
    ab2ccd
    +        if a_h:
    ab2ccd
    +            if a_h.get('From'):
    ab2ccd
    +                del msg['from']
    ab2ccd
    +                msg['From'] = a_h.get('From')
    ab2ccd
    +            if a_h.get('Reply-To'):
    ab2ccd
    +                del msg['reply-to']
    ab2ccd
    +                msg['Reply-To'] = a_h.get('Reply-To')
    ab2ccd
    +        return
    ab2ccd
    +
    ab2ccd
    +    # There are various headers in msg that we don't want, so we basically
    ab2ccd
    +    # make a copy of the msg, then delete almost everything and set/copy
    ab2ccd
    +    # what we want.
    ab2ccd
    +    omsg = copy.deepcopy(msg)
    ab2ccd
    +    for key in msg.keys():
    ab2ccd
    +        if key.lower() not in KEEPERS:
    ab2ccd
    +            del msg[key]
    ab2ccd
    +    msg['MIME-Version'] = '1.0'
    ab2ccd
    +    msg['Content-Type'] = 'message/rfc822'
    ab2ccd
    +    msg['Content-Disposition'] = 'inline'
    ab2ccd
    +    msg['Message-ID'] = unique_message_id(mlist)
    ab2ccd
    +    # Add the headers from CookHeaders.
    ab2ccd
    +    for k, v in msgdata['add_header'].items():
    ab2ccd
    +        msg[k] = v
    ab2ccd
    +    # And set the payload.
    ab2ccd
    +    msg.set_payload(omsg.as_string())
    ab2ccd
    +
    ab2ccd
    diff --git a/Mailman/MailList.py b/Mailman/MailList.py
    ab2ccd
    old mode 100644
    ab2ccd
    new mode 100755
    ab2ccd
    index 6083fb1..f948b69
    ab2ccd
    --- a/Mailman/MailList.py
    ab2ccd
    +++ b/Mailman/MailList.py
    ab2ccd
    @@ -346,6 +346,7 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
    ab2ccd
             self.bounce_matching_headers = \
    ab2ccd
                     mm_cfg.DEFAULT_BOUNCE_MATCHING_HEADERS
    ab2ccd
             self.header_filter_rules = []
    ab2ccd
    +        self.from_is_list = mm_cfg.DEFAULT_FROM_IS_LIST
    ab2ccd
             self.anonymous_list = mm_cfg.DEFAULT_ANONYMOUS_LIST
    ab2ccd
             internalname = self.internal_name()
    ab2ccd
             self.real_name = internalname[0].upper() + internalname[1:]
    ab2ccd
    @@ -386,6 +387,10 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
    ab2ccd
             # 2==Discard
    ab2ccd
             self.member_moderation_action = 0
    ab2ccd
             self.member_moderation_notice = ''
    ab2ccd
    +        self.dmarc_moderation_action = mm_cfg.DEFAULT_DMARC_MODERATION_ACTION
    ab2ccd
    +        self.dmarc_quarantine_moderation_action = (
    ab2ccd
    +            mm_cfg.DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION)
    ab2ccd
    +        self.dmarc_moderation_notice = ''
    ab2ccd
             self.accept_these_nonmembers = []
    ab2ccd
             self.hold_these_nonmembers = []
    ab2ccd
             self.reject_these_nonmembers = []
    ab2ccd
    @@ -712,7 +717,14 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
    ab2ccd
         def CheckVersion(self, stored_state):
    ab2ccd
             """Auto-update schema if necessary."""
    ab2ccd
             if self.data_version >= mm_cfg.DATA_FILE_VERSION:
    ab2ccd
    -            return
    ab2ccd
    +            # Some lists could have been created by newer Mailman version than
    ab2ccd
    +            # this one. We are adding just few variables, so check for these
    ab2ccd
    +            # variables explicitely.
    ab2ccd
    +            if (hasattr(self, "from_is_list")
    ab2ccd
    +                and hasattr(self, "dmarc_moderation_action")
    ab2ccd
    +                and hasattr(self, "dmarc_moderation_notice")
    ab2ccd
    +                and hasattr(self, "dmarc_quarantine_moderation_action")):
    ab2ccd
    +                return
    ab2ccd
             # Initialize any new variables
    ab2ccd
             self.InitVars()
    ab2ccd
             # Then reload the database (but don't recurse).  Force a reload even
    ab2ccd
    @@ -1025,7 +1030,8 @@ class MailList(HTMLFormatter, Deliverer, ListAdmin,
    ab2ccd
             # And send an acknowledgement to the user...
    ab2ccd
             if userack:
    ab2ccd
                 self.SendUnsubscribeAck(emailaddr, userlang)
    ab2ccd
    -        # ...and to the administrator
    ab2ccd
    +        # ...and to the administrator in the correct language.  (LP: #1308655)
    ab2ccd
    +        i18n.set_language(self.preferred_language)
    ab2ccd
             if admin_notif:
    ab2ccd
                 realname = self.real_name
    ab2ccd
                 subject = _('%(realname)s unsubscribe notification')
    ab2ccd
    diff --git a/Mailman/Message.py b/Mailman/Message.py
    ab2ccd
    index 84e4aa2..13e7ff2 100644
    ab2ccd
    --- a/Mailman/Message.py
    ab2ccd
    +++ b/Mailman/Message.py
    ab2ccd
    @@ -61,6 +61,43 @@ class Generator(email.Generator.Generator):
    ab2ccd
     
    ab2ccd
     
    ab2ccd
     
    ab2ccd
    +class Generator(email.Generator.Generator):
    ab2ccd
    +    """Generates output from a Message object tree, keeping signatures.
    ab2ccd
    +
    ab2ccd
    +       Headers will by default _not_ be folded in attachments.
    ab2ccd
    +    """
    ab2ccd
    +    def __init__(self, outfp, mangle_from_=True,
    ab2ccd
    +                 maxheaderlen=78, children_maxheaderlen=0):
    ab2ccd
    +        email.Generator.Generator.__init__(self, outfp,
    ab2ccd
    +                mangle_from_=mangle_from_, maxheaderlen=maxheaderlen)
    ab2ccd
    +        self.__children_maxheaderlen = children_maxheaderlen
    ab2ccd
    +
    ab2ccd
    +    def clone(self, fp):
    ab2ccd
    +        """Clone this generator with maxheaderlen set for children"""
    ab2ccd
    +        return self.__class__(fp, self._mangle_from_,
    ab2ccd
    +                self.__children_maxheaderlen, self.__children_maxheaderlen)
    ab2ccd
    +
    ab2ccd
    +    # This is the _handle_message method with the fix for bug 7970.
    ab2ccd
    +    def _handle_message(self, msg):
    ab2ccd
    +        s = StringIO()
    ab2ccd
    +        g = self.clone(s)
    ab2ccd
    +        # The payload of a message/rfc822 part should be a multipart sequence
    ab2ccd
    +        # of length 1.  The zeroth element of the list should be the Message
    ab2ccd
    +        # object for the subpart.  Extract that object, stringify it, and
    ab2ccd
    +        # write it out.
    ab2ccd
    +        # Except, it turns out, when it's a string instead, which happens when
    ab2ccd
    +        # and only when HeaderParser is used on a message of mime type
    ab2ccd
    +        # message/rfc822.  Such messages are generated by, for example,
    ab2ccd
    +        # Groupwise when forwarding unadorned messages.  (Issue 7970.)  So
    ab2ccd
    +        # in that case we just emit the string body.
    ab2ccd
    +        payload = msg.get_payload()
    ab2ccd
    +        if isinstance(payload, list):
    ab2ccd
    +            g.flatten(msg.get_payload(0), unixfrom=False)
    ab2ccd
    +            payload = s.getvalue()
    ab2ccd
    +        self._fp.write(payload)
    ab2ccd
    +
    ab2ccd
    +
    ab2ccd
    +
    ab2ccd
     class Message(email.Message.Message):
    ab2ccd
         def __init__(self):
    ab2ccd
             # We need a version number so that we can optimize __setstate__()
    ab2ccd
    @@ -243,6 +280,20 @@ class Message(email.Message.Message):
    ab2ccd
             return fp.getvalue()
    ab2ccd
     
    ab2ccd
     
    ab2ccd
    +    def as_string(self, unixfrom=False, mangle_from_=True):
    ab2ccd
    +        """Return entire formatted message as a string using
    ab2ccd
    +        Mailman.Message.Generator.
    ab2ccd
    +
    ab2ccd
    +        Operates like email.Message.Message.as_string, only
    ab2ccd
    +	using Mailman's Message.Generator class. Only the top headers will
    ab2ccd
    +        get folded.
    ab2ccd
    +        """
    ab2ccd
    +        fp = StringIO()
    ab2ccd
    +        g = Generator(fp, mangle_from_=mangle_from_)
    ab2ccd
    +        g.flatten(self, unixfrom=unixfrom)
    ab2ccd
    +        return fp.getvalue()
    ab2ccd
    +
    ab2ccd
    +
    ab2ccd
     
    ab2ccd
     class UserNotification(Message):
    ab2ccd
         """Class for internally crafted messages."""
    ab2ccd
    diff --git a/Mailman/Utils.py b/Mailman/Utils.py
    ab2ccd
    index c8275df..8021942 100644
    ab2ccd
    --- a/Mailman/Utils.py
    ab2ccd
    +++ b/Mailman/Utils.py
    ab2ccd
    @@ -71,6 +71,14 @@ except NameError:
    ab2ccd
         True = 1
    ab2ccd
         False = 0
    ab2ccd
     
    ab2ccd
    +try:
    ab2ccd
    +    import dns.resolver
    ab2ccd
    +    import dns.rdatatype
    ab2ccd
    +    from dns.exception import DNSException
    ab2ccd
    +    dns_resolver = True
    ab2ccd
    +except ImportError:
    ab2ccd
    +    dns_resolver = False
    ab2ccd
    +
    ab2ccd
     EMPTYSTRING = ''
    ab2ccd
     UEMPTYSTRING = u''
    ab2ccd
     NL = '\n'
    ab2ccd
    @@ -1047,3 +1055,91 @@ def suspiciousHTML(html):
    ab2ccd
         else:
    ab2ccd
             return False
    ab2ccd
     
    ab2ccd
    +
    ab2ccd
    +
    ab2ccd
    +
    ab2ccd
    +# This takes an email address, and returns True if DMARC policy is p=reject
    ab2ccd
    +# or possibly quarantine.
    ab2ccd
    +def IsDMARCProhibited(mlist, email):
    ab2ccd
    +    if not dns_resolver:
    ab2ccd
    +         return False
    ab2ccd
    +
    ab2ccd
    +    email = email.lower()
    ab2ccd
    +    at_sign = email.find('@')
    ab2ccd
    +    if at_sign < 1:
    ab2ccd
    +        return False
    ab2ccd
    +    dmarc_domain = '_dmarc.' + email[at_sign+1:]
    ab2ccd
    +
    ab2ccd
    +    try:
    ab2ccd
    +        resolver = dns.resolver.Resolver()
    ab2ccd
    +        resolver.timeout = float(mm_cfg.DMARC_RESOLVER_TIMEOUT)
    ab2ccd
    +        resolver.lifetime = float(mm_cfg.DMARC_RESOLVER_LIFETIME)
    ab2ccd
    +        txt_recs = resolver.query(dmarc_domain, dns.rdatatype.TXT)
    ab2ccd
    +    except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer):
    ab2ccd
    +        return False
    ab2ccd
    +    except DNSException, e:
    ab2ccd
    +        syslog('error',
    ab2ccd
    +               'DNSException: Unable to query DMARC policy for %s (%s). %s',
    ab2ccd
    +              email, dmarc_domain, e.__class__)
    ab2ccd
    +        return False
    ab2ccd
    +    else:
    ab2ccd
    +# people are already being dumb, don't trust them to provide honest DNS
    ab2ccd
    +# where the answer section only contains what was asked for, nor to include
    ab2ccd
    +# CNAMEs before the values they point to.
    ab2ccd
    +        full_record = ""
    ab2ccd
    +        results_by_name = {}
    ab2ccd
    +        cnames = {}
    ab2ccd
    +        want_names = set([dmarc_domain + '.'])
    ab2ccd
    +        for txt_rec in txt_recs.response.answer:
    ab2ccd
    +            if txt_rec.rdtype == dns.rdatatype.CNAME:
    ab2ccd
    +                cnames[txt_rec.name.to_text()] = (
    ab2ccd
    +                    txt_rec.items[0].target.to_text())
    ab2ccd
    +            if txt_rec.rdtype != dns.rdatatype.TXT:
    ab2ccd
    +                continue
    ab2ccd
    +            results_by_name.setdefault(txt_rec.name.to_text(), []).append(
    ab2ccd
    +                "".join(txt_rec.items[0].strings))
    ab2ccd
    +        expands = list(want_names)
    ab2ccd
    +        seen = set(expands)
    ab2ccd
    +        while expands:
    ab2ccd
    +            item = expands.pop(0)
    ab2ccd
    +            if item in cnames:
    ab2ccd
    +                if cnames[item] in seen:
    ab2ccd
    +                    continue # cname loop
    ab2ccd
    +                expands.append(cnames[item])
    ab2ccd
    +                seen.add(cnames[item])
    ab2ccd
    +                want_names.add(cnames[item])
    ab2ccd
    +                want_names.discard(item)
    ab2ccd
    +
    ab2ccd
    +        if len(want_names) != 1:
    ab2ccd
    +            syslog('error',
    ab2ccd
    +                   """multiple DMARC entries in results for %s,
    ab2ccd
    +                   processing each to be strict""",
    ab2ccd
    +                   dmarc_domain)
    ab2ccd
    +        for name in want_names:
    ab2ccd
    +            if name not in results_by_name:
    ab2ccd
    +                continue
    ab2ccd
    +            dmarcs = filter(lambda n: n.startswith('v=DMARC1;'),
    ab2ccd
    +                            results_by_name[name])
    ab2ccd
    +            if len(dmarcs) == 0:
    ab2ccd
    +                return False
    ab2ccd
    +            if len(dmarcs) > 1:
    ab2ccd
    +                syslog('error',
    ab2ccd
    +                       """RRset of TXT records for %s has %d v=DMARC1 entries;
    ab2ccd
    +                       testing them all""",
    ab2ccd
    +                        dmarc_domain, len(dmarc))
    ab2ccd
    +            for entry in dmarcs:
    ab2ccd
    +                if re.search(r'\bp=reject\b', entry, re.IGNORECASE):
    ab2ccd
    +                    syslog('vette',
    ab2ccd
    +                        'DMARC lookup for %s (%s) found p=reject in %s = %s',
    ab2ccd
    +                        email, dmarc_domain, name, entry)
    ab2ccd
    +                    return True
    ab2ccd
    +
    ab2ccd
    +                if (mlist.dmarc_quarantine_moderation_action and
    ab2ccd
    +                    re.search(r'\bp=quarantine\b', entry, re.IGNORECASE)):
    ab2ccd
    +                    syslog('vette',
    ab2ccd
    +                      'DMARC lookup for %s (%s) found p=quarantine in %s = %s',
    ab2ccd
    +                            email, dmarc_domain, name, entry)
    ab2ccd
    +                    return True
    ab2ccd
    +
    ab2ccd
    +    return False
    ab2ccd
    +
    ab2ccd
    diff --git a/Mailman/Version.py b/Mailman/Version.py
    ab2ccd
    index 05e6500..af4a2df 100644
    ab2ccd
    --- a/Mailman/Version.py
    ab2ccd
    +++ b/Mailman/Version.py
    ab2ccd
    @@ -37,7 +37,7 @@ HEX_VERSION = ((MAJOR_REV << 24) | (MINOR_REV << 16) | (MICRO_REV << 8) |
    ab2ccd
                    (REL_LEVEL << 4)  | (REL_SERIAL << 0))
    ab2ccd
     
    ab2ccd
     # config.pck schema version number
    ab2ccd
    -DATA_FILE_VERSION = 100
    ab2ccd
    +DATA_FILE_VERSION = 101
    ab2ccd
     
    ab2ccd
     # qfile/*.db schema version number
    ab2ccd
     QFILE_SCHEMA_VERSION = 3
    ab2ccd
    diff --git a/Mailman/versions.py b/Mailman/versions.py
    ab2ccd
    old mode 100644
    ab2ccd
    new mode 100755
    ab2ccd
    index 81fafd5..138e770
    ab2ccd
    --- a/Mailman/versions.py
    ab2ccd
    +++ b/Mailman/versions.py
    ab2ccd
    @@ -313,6 +313,9 @@ def UpdateOldVars(l, stored_state):
    ab2ccd
                 pass
    ab2ccd
             else:
    ab2ccd
                 l.digest_members[k] = 0
    ab2ccd
    +    # from_is_list was called author_is_list in 2.1.16rc2 (only).
    ab2ccd
    +    PreferStored('author_is_list', 'from_is_list',
    ab2ccd
    +                 mm_cfg.DEFAULT_FROM_IS_LIST)
    ab2ccd
     
    ab2ccd
     
    ab2ccd
     
    ab2ccd
    @@ -383,6 +386,11 @@ def NewVars(l):
    ab2ccd
         # the current GUI description model.  So, 0==Hold, 1==Reject, 2==Discard
    ab2ccd
         add_only_if_missing('member_moderation_action', 0)
    ab2ccd
         add_only_if_missing('member_moderation_notice', '')
    ab2ccd
    +    add_only_if_missing('dmarc_moderation_action', 
    ab2ccd
    +                       mm_cfg.DEFAULT_DMARC_MODERATION_ACTION)
    ab2ccd
    +    add_only_if_missing('dmarc_quarantine_moderation_action',
    ab2ccd
    +                       mm_cfg.DEFAULT_DMARC_QUARANTINE_MODERATION_ACTION)
    ab2ccd
    +    add_only_if_missing('dmarc_moderation_notice', '')
    ab2ccd
         add_only_if_missing('new_member_options',
    ab2ccd
                             mm_cfg.DEFAULT_NEW_MEMBER_OPTIONS)
    ab2ccd
         # Emergency moderation flag
    ab2ccd
    diff --git a/contrib/majordomo2mailman.pl b/contrib/majordomo2mailman.pl
    ab2ccd
    index c874862..770dc57 100644
    ab2ccd
    --- a/contrib/majordomo2mailman.pl
    ab2ccd
    +++ b/contrib/majordomo2mailman.pl
    ab2ccd
    @@ -480,6 +480,7 @@ sub init_defaultmmconf {
    ab2ccd
     		  'max_num_recipients', "10",
    ab2ccd
     		  'forbidden_posters', "[]",
    ab2ccd
     		  'bounce_matching_headers',  "\"\"\"\n\"\"\"\n",
    ab2ccd
    +		  'from_is_list', "0",
    ab2ccd
     		  'anonymous_list', "0",
    ab2ccd
     		  'nondigestable', "1",
    ab2ccd
     		  'digestable', "1",