Commit 61d65f97 authored by Lars Kruse's avatar Lars Kruse

Merge branch 'incoming-mails-via-lmtp' into release-2-3

parents 98c417ab 03312597
Pipeline #1263 failed with stage
in 1 minute and 37 seconds
......@@ -10,8 +10,8 @@ Package: stadtgestalten
Architecture: all
# python3-requests: for django-allauth
# python3-six: for django-mailbox
Depends: python3-pil, python3-requests, python3-six, ${misc:Depends},
${python3:Depends}, moreutils, python3-xapian
Depends: python3-html2text, python3-pil, python3-requests, python3-six,
python3-xapian ${misc:Depends}, ${python3:Depends}, moreutils
Recommends: ghostscript, python3-psycopg2, uwsgi, uwsgi-plugin-python3,
uwsgi-plugin-router-access, ${misc:Recommends}
Suggests: sqlite3, postgresql-client-common
......
# optional:
# MAILTO=admin@example.org
# process incoming mails
# The LMTP daemon (since grouprise 2.3) is supposed to handle incoming mail in the future.
# This requires changes of the SMTP server setup. We will start with single recipient groups, until
# all traffic is handled by LMTP.
*/3 * * * * root chronic sh -c 'stadtctl getmail 2>&1 | grep -vE "^(INFO:|WARNING:django_mailbox.models:.*content-transfer-encoding)"'
# calculate scores for groups and users
......
import logging
import django.conf
import django.db.models.signals
import django_mailbox.signals
from django.conf import settings
from django.db.models.functions import Lower
from django.dispatch import receiver
import core.models
from features.associations import models as associations
from features.content.models import Content
from features.conversations import models as conversations
from features.files import models as files
from features.gestalten import models as gestalten
from features.groups import models as groups
from . import models, notifications
from . import notifications
logger = logging.getLogger(__name__)
post_create = django.dispatch.Signal(providing_args=['instance'])
......@@ -24,137 +10,3 @@ post_create = django.dispatch.Signal(providing_args=['instance'])
@receiver(post_create)
def contribution_created(sender, instance, **kwargs):
notifications.ContributionCreated.send_all(instance)
def get_sender(message):
try:
gestalt = gestalten.Gestalt.objects.annotate(email=Lower('user__email')).get(
email=message.from_address[0].lower())
except gestalten.Gestalt.DoesNotExist:
try:
gestalt = gestalten.Gestalt.objects.annotate(
email=Lower('user__emailaddress__email')).get(
email=message.from_address[0].lower())
except gestalten.Gestalt.DoesNotExist:
gestalt = None
return gestalt
def is_autoresponse(msg):
email = msg.get_email_object()
# RFC 3834 (https://tools.ietf.org/html/rfc3834#section-5)
if email.get('Auto-Submitted') == 'no':
return False
elif email.get('Auto-Submitted'):
return True
# non-standard fields (https://tools.ietf.org/html/rfc3834#section-3.1.8)
if email.get('Precedence') == 'bulk':
return True
if email.get('X-AUTORESPONDER'):
return True
return False
@receiver(django_mailbox.signals.message_received)
def process_incoming_message(sender, message, **args):
token_beg = len(django.conf.settings.DEFAULT_REPLY_TO_EMAIL.split('{')[0])
token_end = len(django.conf.settings.DEFAULT_REPLY_TO_EMAIL.rsplit('}')[1])
DOMAIN = django.conf.settings.DEFAULT_REPLY_TO_EMAIL.split('@')[1]
def create_contribution(container, gestalt, message, in_reply_to=None):
text = message.text.strip()
if text == '':
text = message.html
t = models.Text.objects.create(text=text)
contribution = models.Contribution.objects.create(
author=gestalt,
container=container,
in_reply_to=in_reply_to,
contribution=t)
files.File.objects.create_from_message(message, attached_to=contribution)
return contribution
def process_reply(address):
token = address[token_beg:-token_end]
try:
in_reply_to_text = models.Contribution.objects.get_by_message_id(
message.get_email_object().get('In-Reply-To'))
except models.Contribution.DoesNotExist:
in_reply_to_text = None
key = core.models.PermissionToken.objects.get(secret_key=token)
sender = get_sender(message)
if key.gestalt != sender:
raise django.core.exceptions.PermissionDenied(
'Du darfst diese Benachrichtigung nicht unter dieser E-Mail-Adresse '
'beantworten. Du hast folgende Möglichkeiten:\n'
'* Melde Dich auf der Website an und beantworte die Nachricht dort.\n'
'* Antworte unter der E-Mail-Adresse, an die die Benachrichtigung '
'gesendet wurde.\n'
'* Füge die E-Mail-Adresse, unter der Du antworten möchtest, Deinem '
'Benutzerkonto hinzu.')
if type(key.target) == Content:
container = key.target
else:
container = key.target.container
contribution = create_contribution(
container, key.gestalt, message, in_reply_to=in_reply_to_text)
post_create.send(sender=None, instance=contribution)
def process_initial(address):
local, domain = address.split('@')
if domain != DOMAIN:
raise ValueError('Domain does not match.')
gestalt = get_sender(message)
group = groups.Group.objects.get(slug=local)
if gestalt and gestalt.user.has_perm(
'conversations.create_group_conversation_by_email', group):
conversation = conversations.Conversation.objects.create(subject=message.subject)
contribution = create_contribution(conversation, gestalt, message)
associations.Association.objects.create(
entity_type=group.content_type, entity_id=group.id,
container_type=conversation.content_type, container_id=conversation.id)
post_create.send(sender=None, instance=contribution)
else:
raise django.core.exceptions.PermissionDenied(
'Du darfst mit dieser Gruppe kein Gespräch per E-Mail beginnen. Bitte '
'verwende die Schaltfläche auf der Webseite.')
def process_message(address):
try:
process_reply(address)
except core.models.PermissionToken.DoesNotExist:
if address == settings.STADTGESTALTEN_BOT_EMAIL:
for to_address in message.to_addresses:
process_initial(to_address)
else:
process_initial(address)
# FIXME: use X-Stadtgestalten-to header (mailbox without domain)
delivered_to = message.get_email_object()['Delivered-To']
if not delivered_to:
logger.error('Could not process message {}: no Delivered-To header'.format(message.id))
return
for address in [delivered_to]:
address = address.lstrip('<')
address = address.rstrip('>')
if not is_autoresponse(message):
try:
process_message(address)
except ValueError as e:
logger.error('Could not process receiver {} in message {}. {}'.format(
address, message.id, e))
except (groups.Group.DoesNotExist, django.core.exceptions.PermissionDenied) as e:
logger.warning('Could not process receiver {} in message {}. {}'.format(
address, message.id, e))
django.core.mail.send_mail(
'Re: {}'.format(message.subject).replace('\n', ' ').replace('\r', ''),
'Konnte die Nachricht nicht verarbeiten. {}'.format(e),
from_email=django.conf.settings.DEFAULT_FROM_EMAIL,
recipient_list=[message.from_header],
fail_silently=True)
else:
logger.warning('Ignored message {} as autoresponse'.format(message.id))
import django
from django.core import mail
from django.urls import reverse
from django_mailbox import models as mailbox_models, signals as mailbox_signals
import features.articles.tests
from core import tests
from features.associations import models as associations
from features.gestalten import tests as gestalten
from features.memberships import test_mixins as memberships
from . import models
......@@ -23,70 +17,6 @@ class ContributionMixin(features.articles.tests.ArticleMixin):
self.contribution = self.create_contribution()
class ContentReplyByEmail(
memberships.AuthenticatedMemberMixin, tests.Test):
def test_content_reply_by_email(self):
# create article
self.client.post(
reverse('create-group-article', args=(self.group.slug,)),
{'title': 'Test', 'text': 'Test'})
a = self.assertExists(associations.Association, content__title='Test')
self.assertNotificationSent()
# generate reply message
reply_to = mail.outbox[0].extra_headers['Reply-To']
msg = mailbox_models.Message(
from_header=self.gestalt.user.email,
body='Delivered-To: {}\n\nText B'.format(reply_to))
# send signal like getmail would
mailbox_signals.message_received.send(self, message=msg)
self.assertExists(
models.Contribution, content=a.content.get(),
text__text='Text B')
class ConversationInitiateByEmail(memberships.MemberMixin, tests.Test):
def test_conversation_initiate_by_email(self):
# generate initial message
msg = mailbox_models.Message(
from_header=self.gestalt.user.email,
body='Delivered-To: {}@localhost\n\nText A'.format(self.group.slug))
# send signal like getmail would
mailbox_signals.message_received.send(self, message=msg)
self.assertExists(
models.Contribution, conversation__associations__group=self.group,
text__text='Text A')
def test_conversation_initiate_by_email_failing(self):
# generate initial message
msg = mailbox_models.Message(
from_header=self.gestalt.user.email,
body='Delivered-To: not-existing@localhost\n\nText A')
# send signal like getmail would
mailbox_signals.message_received.send(self, message=msg)
self.assertEqual(len(mail.outbox), 1)
class ConversationReplyByEmail(
gestalten.AuthenticatedMixin, gestalten.OtherGestaltMixin, tests.Test):
def test_texts_reply_by_email(self):
# send message to other_gestalt via web interface
self.client.post(
self.get_url('create-gestalt-conversation', self.other_gestalt.pk),
{'subject': 'Subject A', 'text': 'Text A'})
text_a = self.assertExists(models.Contribution, conversation__subject='Subject A')
self.assertNotificationSent()
# generate reply message
reply_to = mail.outbox[0].extra_headers['Reply-To']
msg = mailbox_models.Message(
from_header=self.other_gestalt.user.email,
body='Delivered-To: {}\n\nText B'.format(reply_to))
# send signal like getmail would
mailbox_signals.message_received.send(self, message=msg)
self.assertExists(
models.Contribution, conversation=text_a.conversation.get(),
text__text='Text B')
class Delete(ContributionMixin, django.test.TestCase):
def test_delete_contribution(self):
delete_url = reverse(
......
from os import path
import re
import io
import os
import tempfile
from typing import List
import django
from django.db import models
import django.core.files
from features.imports.signals import ParsedMailAttachment
from imagekit.models import ImageSpecField
from imagekit.processors import ResizeToFit, Transpose
......@@ -10,15 +14,44 @@ import core
from features.contributions import models as contributions
EXCLUDE_RE = r'Content-Type: application/pgp-signature'
IGNORE_CONTENT_TYPES = {'application/pgp-signature'}
def get_unique_storage_filename(name_template: str, base_dir: str,
default_prefix: str='attachment-') -> str:
""" determine a suitable name for a file to be stored in a directory
The file may not overwrite an existing file and it should keep its original extension.
"""
temp_kwargs = {'dir': base_dir, 'prefix': default_prefix, 'delete': False}
if name_template:
basename, extension = os.path.splitext(os.path.basename(name_template))
if extension:
temp_kwargs['suffix'] = '.' + extension
if basename:
temp_kwargs['prefix'] = basename + '-'
storage_file = tempfile.NamedTemporaryFile(**temp_kwargs)
storage_file.close()
return storage_file.name
class FileManager(models.Manager):
def create_from_message(self, message, attached_to):
for attachment in message.attachments.all():
if re.search(EXCLUDE_RE, attachment.headers):
def create_from_message_attachments(self, attachments: List[ParsedMailAttachment],
attached_to):
for attachment in attachments:
if attachment.content_type in IGNORE_CONTENT_TYPES:
continue
f = self.create(file=attachment.document, filename=attachment.get_filename())
if attachment.data is not None:
# create the file and reference it
filename = get_unique_storage_filename(attachment.filename,
File.file.field.storage.base_location)
f = self.create()
f.file.save(os.path.basename(filename),
django.core.files.File(io.BytesIO(attachment.data)))
else:
file_source = attachment.model_obj
f = self.create(file=file_source, filename=attachment.filename)
contributions.Contribution.objects.create(
container_id=attached_to.container_id,
container_type=attached_to.container_type,
......@@ -54,7 +87,7 @@ class File(core.models.Model):
@property
def display_name(self):
return self.filename or path.basename(self.file.name)
return self.filename or os.path.basename(self.file.name)
def __str__(self):
return self.display_name
......@@ -63,13 +96,13 @@ class File(core.models.Model):
return self.file.url
def is_image(self):
root, ext = path.splitext(self.file.name.lower())
root, ext = os.path.splitext(self.file.name.lower())
return ext in ('.svg', '.apng', '.png', '.gif', '.jpg', '.jpeg')
def is_video(self):
root, ext = path.splitext(self.file.name.lower())
root, ext = os.path.splitext(self.file.name.lower())
return ext in ('.mp4', '.ogg', '.webm')
def is_audio(self):
root, ext = path.splitext(self.file.name.lower())
root, ext = os.path.splitext(self.file.name.lower())
return ext in ('.opus', '.ogg', '.aac', '.mp3', '.flac', '.m4a')
import contextlib
import os
import shutil
import unittest
import tempfile
import django
import django_mailbox
import core
import core.tests
from features.associations import models as associations
from features.files.models import get_unique_storage_filename
from features.images import tests as images
from features.memberships import test_mixins as memberships
......@@ -93,6 +97,42 @@ class Gestalt(images.ImageMixin, memberships.AuthenticatedMemberMixin, core.test
self.assertOk(url=self.get_group_file_url())
class FilenameGenerator(unittest.TestCase):
def setUp(self):
self._base_dir = tempfile.mkdtemp()
# Manage a set of created filenames. Otherwise the 'tearDown' method preempts the context
# cleanup and thus fails to remove the temporary directory.
self._filenames = set()
super().setUp()
def tearDown(self):
for filename in self._filenames:
os.unlink(filename)
os.rmdir(self._base_dir)
@contextlib.contextmanager
def get_unique_filename(self, name_template, default_prefix=None):
filename = get_unique_storage_filename(name_template, self._base_dir,
default_prefix=default_prefix)
self._filenames.add(filename)
yield filename
os.unlink(filename)
self._filenames.remove(filename)
def test_unique_name_generator(self):
with self.get_unique_filename('foo.bar.baz', 'nom') as filename:
self.assertEqual(os.path.dirname(filename), self._base_dir)
self.assertTrue(os.path.basename(filename))
self.assertTrue(os.path.basename(filename).startswith('foo.bar-'), filename)
self.assertTrue(os.path.basename(filename).endswith('.baz'), filename)
with self.get_unique_filename(None, 'foo-') as filename:
self.assertEqual(os.path.dirname(filename), self._base_dir)
self.assertTrue(os.path.basename(filename))
self.assertTrue(os.path.basename(filename).startswith('foo-'), filename)
self.assertNotIn('.', os.path.basename(filename))
class TestUrls(core.tests.Test):
def test_files_404(self):
r = self.client.get(self.get_url('create-group-file', 'non-existent'))
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
aiosmtpd # debian: python3-aiosmtpd
bleach # debian: python3-bleach
django # debian: python3-django
django-allauth # debian (buster): python3-django-allauth
......@@ -11,10 +12,12 @@ django-mailer # NOT IN DEBIAN
djangorestframework # debian: python-djangorestframework
feedparser # debian: python3-feedparser
flake8 # debian: flake8
html2text # debian: python3-html2text
markdown # debian: python3-markdown
mdx_unimoji # NOT IN DEBIAN
pillow # debian: python-pil
pymdown-extensions # NOT IN DEBIAN
aiosmtplib # NOT IN DEBIAN
schulze # NOT IN DEBIAN
randomcolor # NOT IN DEBIAN
raven # debian (buster): python3-raven
......
......@@ -187,6 +187,7 @@ DEFAULT_FROM_EMAIL = 'noreply@localhost'
DEFAULT_REPLY_TO_EMAIL = 'stadtgestalten+{reply_key}@localhost'
FROM_EMAIL_WITH_SLUG = 'noreply+{slug}@localhost'
STADTGESTALTEN_BOT_EMAIL = 'stadtgestalten-bot@localhost'
GROUPRISE_POSTMASTER_EMAIL = 'postmaster@localhost'
EMAIL_BACKEND = 'mailer.backend.DbBackend'
MAILER_EMAIL_BACKEND = 'django.core.mail.backends.smtp.EmailBackend'
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment