add debian packaging

* split up makefile into separate modules
* implement debian packaging based on makefilet
* replace grunt with simple make targets
* change asset build paths
* change media and static root urls
* update docs
parent a35c9078
[bumpversion]
current_version = 1.7.3
commit = True
tag = True
[bumpversion:file:VERSION]
[bumpversion:file:setup.py]
[bumpversion:file:package.json]
[bumpversion:file:stadt/__init__.py]
search = ^VERSION = "{current_version}"
......@@ -10,7 +10,7 @@ charset = utf-8
[{*.js,*.json,*.vue}]
indent_size = 2
[Makefile]
[{make.d/*,Makefile,debian/rules}]
indent_style = tab
[package.json]
......
[flake8]
max-line-length = 99
exclude = */migrations, node_modules
exclude = */migrations, node_modules, stadt/settings/local.py,
build, .pybuild, debian/stadtgestalten, debian/tmp, debian/root
ignore = E121,E123,E126,E226,E24,E704,W503
__pycache__
.idea
backup
build
db.sqlite3*
media
node_modules
local_settings.py
static
whoosh_index
res/img/backdrops
stadt/ASSET_VERSION
stadt/CACHE_VERSION
stadt/settings/local.py
*.swp
*.pyc
*.log
# debian shizzle
makefilet
.pybuild/
debian/debhelper-build-stamp
debian/stadtgestalten/
debian/*.debhelper
debian/tmp
debian/*.substvars
debian/files
stadtgestalten.egg-info/
/root/
# Contribution Guidelines
## yay! hello there!
Nice to have you on board. Whenever you add new features try not to break any existing tests and add new ones whenever you can. You can use the make targets `lint`, `lint_js`, `lint_py` to lint your code and `test` for testing.
## Release Workflow
To create a new release take the following steps:
1. clean your workspace (the output of `git status` should be empty)
2. checkout the master branch and update it
3. run `make release-major`, `make release-minor` or `make release-patch`
4. push your updated master branch (`git push`) and tags (`git push --tags`
5. done
¹ you might also want to checkout `config --global push.followTags true` ;)
module.exports = function (grunt) {
const pkg = grunt.file.readJSON('package.json')
// static files
const rawBanner = grunt.file.read('res/templates/banner.txt')
const banner = grunt.template.process(rawBanner, {data: {package: pkg}})
// postcss config
const postcssAutoprefixer = require('autoprefixer')({browsers: ['last 5 versions', 'ie 11']})
const postcssBanner = require('postcss-banner')({banner: banner})
const postcssWring = require('csswring')
// Project configuration.
grunt.initConfig({
fontdump: {
google: {
options: {
web_directory: '../fonts'
},
files: {
'build/fonts/google/fonts.css': 'http://fonts.googleapis.com/css?family=' +
'Roboto Slab:300,400,700|' +
'Roboto:300,400,400italic,500,700'
}
}
},
less: {
options: {
strictMath: true,
strictUnits: true,
sourceMap: true,
outputSourceFiles: true,
compress: false
},
app: {
options: {
sourceMapFilename: 'build/css/app_unprefixed.css.map',
sourceMapURL: './app_unprefixed.css.map'
},
files: {
'build/css/app_unprefixed.css': 'res/less/app.less'
}
}
},
postcss: {
options: {
map: {
inline: false,
annotation: true,
prev: 'build/css/app.css.map'
},
processors: [
postcssAutoprefixer,
postcssWring,
postcssBanner
]
},
dist: {
src: 'build/css/app_unprefixed.css',
dest: 'stadt/static/css/app.css'
}
},
exec: {
webpack_dev: 'node_modules/.bin/webpack',
webpack_dist: 'NODE_ENV=production node_modules/.bin/webpack --bail'
},
copy: {
fonts: {
files: [
{cwd: 'build/fonts/google', src: '*.!(css)', dest: 'stadt/static/fonts', expand: true},
{cwd: 'node_modules/font-awesome/fonts', src: '*', dest: 'stadt/static/fonts', expand: true},
{cwd: 'res/fonts', src: '**', dest: 'stadt/static/fonts', expand: true}
]
},
images: {
files: [
{cwd: 'res/img', src: '**/*.!(svg)', dest: 'stadt/static/img', expand: true}
]
},
configs: {
files: [
{src: 'res/config/manifest.json', dest: 'stadt/static/config/manifest.json'}
]
}
},
svgmin: {
dist: {
files: [
{cwd: 'res/img', src: '**/*.svg', dest: 'stadt/static/img', expand: true}
]
}
},
watch: {
less: {
files: ['res/**/*.less'],
tasks: ['css']
}
}
})
// These plugins provide necessary tasks.
grunt.loadNpmTasks('grunt-contrib-watch')
grunt.loadNpmTasks('grunt-contrib-less')
grunt.loadNpmTasks('grunt-contrib-copy')
grunt.loadNpmTasks('grunt-fontdump')
grunt.loadNpmTasks('grunt-postcss')
grunt.loadNpmTasks('grunt-svgmin')
grunt.loadNpmTasks('grunt-exec')
// Default task.
grunt.registerTask('css', ['less', 'postcss'])
grunt.registerTask('js', ['exec:webpack_dist'])
grunt.registerTask('fonts', ['fontdump', 'copy:fonts'])
grunt.registerTask('images', ['copy:images', 'svgmin'])
grunt.registerTask('misc', ['copy:configs'])
grunt.registerTask('default', ['fonts', 'images', 'css', 'js', 'misc'])
}
RM ?= rm -f
YARN_BIN ?= yarn
NODEJS_BIN ?= $(shell which node nodejs | head -1)
GRUNT_BIN = node_modules/.bin/grunt
STANDARD_BIN = node_modules/.bin/standard
BUILD_PATH ?= build
BACKUP_PATH ?= backup
PYTHON_DIRS = content entities stadt features core utils
# uwsgi beobachtet diese Datei und schaltet bei ihrer Existenz in den Offline-Modus:
# if-exists = _OFFLINE_MARKER_UWSGI
# route = .* redirect:https://offline.stadtgestalten.org/
# endif =
OFFLINE_MARKER_FILE = _OFFLINE_MARKER_UWSGI
DJANGO_SETTINGS_MODULE ?= stadt.settings
DB_CONNECTION_BACKUP ?= $(shell (echo "from $(DJANGO_SETTINGS_MODULE) import *; d=DATABASES['default']; format_string = {'sqlite3': 'echo .backup $(DB_BACKUP_FILE) | sqlite3 {NAME}', 'postgresql': 'pg_dump --no-owner --no-privileges \"postgresql://{USER}:{PASSWORD}@{HOST}/{NAME}\" >$(DB_BACKUP_FILE)'}[d['ENGINE'].split('.')[-1]]; print(format_string.format(**DATABASES['default']))") | PYTHONPATH=. python)
DB_CONNECTION_RESTORE ?= $(shell (echo "from $(DJANGO_SETTINGS_MODULE) import *; d=DATABASES['default']; format_string = {'sqlite3': 'echo .restore $(DB_BACKUP_FILE) | sqlite3 {NAME}', 'postgresql': 'psql \"postgresql://{USER}:{PASSWORD}@{HOST}/{NAME}\" <$(DB_RESTORE_DATAFILE)'}[d['ENGINE'].split('.')[-1]]; print(format_string.format(**DATABASES['default']))") | PYTHONPATH=. python)
DB_BACKUP_FILE ?= $(BACKUP_PATH)/data-$(shell date +%Y%m%d%H%M).sql
# symlink magic for badly packaged dependencies using "node" explicitly
HELPER_BIN_PATH = $(BUILD_PATH)/helper-bin
HELPER_PATH_ENV = PATH=$(HELPER_BIN_PATH):$$PATH
NODEJS_SYMLINK = $(HELPER_BIN_PATH)/node
ASSET_VERSION_PATH = stadt/ASSET_VERSION
CACHE_VERSION_PATH = stadt/CACHE_VERSION
# in dieser Datei wird die Zeichenkette 'VERSION = "X.Y.Z"' erwartet
VERSION_FILE = package.json
# Auslesen der aktuellen Version und Hochzaehlen des gewaehlten Index (je nach Release-Stufe)
NEXT_RELEASE = $(shell PYTHONPATH=. python -c "import stadt.version; print(stadt.version.$(RELEASE_INCREMENT_FUNCTION)())")
GIT_RELEASE_TAG = v$(NEXT_RELEASE)
.PHONY: asset_version cache_version check-virtualenv clean database-backup \
database-restore default deploy deploy-git release-breaking \
release-feature release-patch reload static update-virtualenv test \
website-offline website-online lint
asset_version:
git log --oneline res | head -n 1 | cut -f 1 -d " " > $(ASSET_VERSION_PATH)
cache_version:
git log --oneline --pretty=format:"%ct" | head -n 1 > $(CACHE_VERSION_PATH)
database-backup:
@mkdir -p "$(BACKUP_PATH)"
$(DB_CONNECTION_BACKUP)
database-restore:
@if [ -z "$$DB_RESTORE_DATAFILE" ]; then \
echo >&2 "ERROR: You need to specify the source data file location (DB_RESTORE_DATAFILE=???)"; \
exit 1; fi
$(DB_CONNECTION_RESTORE)
grunt: check-js-deps
($(HELPER_PATH_ENV); export PATH; $(NODEJS_BIN) $(GRUNT_BIN))
$(NODEJS_SYMLINK):
mkdir -p "$(HELPER_BIN_PATH)"
@[ -n "$(NODEJS_BIN)" ] || { echo >&2 "Requirement 'nodejs' is missing for build"; exit 1; }
ln -s "$(NODEJS_BIN)" "$(NODEJS_SYMLINK)"
static: check-virtualenv
python manage.py collectstatic --no-input
reload:
@# trigger UWSGI-Reload
touch "$$(echo "$(DJANGO_SETTINGS_MODULE)" | tr '.' '/').py"
website-offline:
touch $(OFFLINE_MARKER_FILE)
website-online:
$(RM) $(OFFLINE_MARKER_FILE)
check-virtualenv:
@# this should fail if dependencies are missing or no virtualenv is active
python manage.py check
check-js-deps: $(NODEJS_SYMLINK)
($(HELPER_PATH_ENV); export PATH; $(YARN_BIN) install)
update-virtualenv: check-virtualenv
pip install -r requirements.txt
python manage.py migrate
deploy:
$(MAKE) asset_version
$(MAKE) cache_version
$(MAKE) test
$(MAKE) website-offline
$(MAKE) database-backup
$(MAKE) grunt
$(MAKE) update-virtualenv
$(MAKE) static
# in "website-online" ist ein "reload" enthalten
$(MAKE) website-online
deploy-git:
git pull
$(MAKE) deploy
# Position der zu veraendernden Zahl in der Release-Nummer (X.Y.Z)
release-breaking: RELEASE_INCREMENT_FUNCTION=get_next_breaking_version
release-feature: RELEASE_INCREMENT_FUNCTION=get_next_feature_version
release-patch: RELEASE_INCREMENT_FUNCTION=get_next_patch_version
release-breaking release-feature release-patch:
@if [ -n "$$(git status -s)" ]; then \
printf >&2 "\n%s\n\n" "*** ERROR: The working directory needs to be clean for a release. ***"; \
false; fi
@if [ -n "$$(git tag -l | while read v; do [ "$$v" != "$(GIT_RELEASE_TAG)" ] || echo FOUND; done)" ]; then \
printf >&2 "\n%s\n\n" "*** ERROR: There is already a git tag of the next version: $(NEXT_RELEASE). Use 'git tag -d $(GIT_RELEASE_TAG)' if know what you are doing."; \
false; fi
# we rely on the specific formatting of this line in 'package.json'
sed -i 's/"version": .*/"version": "$(NEXT_RELEASE)",/' "$(VERSION_FILE)"
git add "$(VERSION_FILE)"
git commit -m "Release $(NEXT_RELEASE)"
git tag -a "$(GIT_RELEASE_TAG)"
$(STANDARD_BIN):
$(MAKE) check-js-deps
lint: check-virtualenv $(STANDARD_BIN)
python -m flake8 $(PYTHON_DIRS)
($(HELPER_PATH_ENV); export PATH; $(YARN_BIN) run lint)
test: lint check-virtualenv $(STANDARD_BIN)
($(HELPER_PATH_ENV); export PATH; $(YARN_BIN) run test)
@# Auf doppelte Test-Methoden-Namen pruefen - diese koennen sich gegenseitig verdecken.
@# Dabei ignorieren wir das Verzeichnis ./.venv/ - es wird von der gitlab-Testumgebung
@# erzeugt und produziert Namenskollisionen mit Django-Tests.
@duplicate_function_names=$$(find -type f -name tests.py \
| grep -v "^\./\.venv/" \
| xargs grep -h "def test_" \
| sed 's/^ \+def //g' | cut -f 1 -d "(" \
| sort | uniq -d); \
if [ -n "$$duplicate_function_names" ]; then \
echo "[ERROR] non-unique test method names found:"; \
echo "$$duplicate_function_names" | sed 's/^/ /g'; \
exit 1; \
fi >&2
@# Die Umgebungsvariable "STADTGESTALTEN_IN_TEST" kann in "local_settings.py" geprueft
@# werden, um die Verwendung einer postgres/mysql-Datenbankverbindung ohne "create"-Rechte
@# zu verhindern. Mit sqlite klappen die Tests dann natuerlich.
STADTGESTALTEN_IN_TEST=1 python manage.py test
clean:
$(RM) -r node_modules
$(RM) -r bower_components
$(RM) -r static
$(RM) -r $(BUILD_PATH)
$(RM) $(OFFLINE_MARKER_FILE)
# load makefilet
include make.d/makefilet-download-ondemand.mk
# define default target
.PHONY: default-target
default-target: build
# include project makefiles
include make.d/virtualenv.mk
include make.d/app.mk
include make.d/assets.mk
include make.d/test.mk
include make.d/dist.mk
# Quick Setup
# Stadtgestalten
pip install -r requirements.txt
python manage.py migrate
python manage.py runserver
npm install && node_modules/.bin/grunt
Stadtgestalten is a platform destined to encourage and enable social action and solidarity in the context of your city. Bildet Banden!
Visit http://localhost:8000/
## Quick Setup
1. You will need [yarn](https://yarnpkg.com/lang/en/), [virtualenv](https://virtualenv.pypa.io/en/stable/), [node](https://nodejs.org/en/), [python3](https://www.python.org/), [flake8](http://flake8.pycqa.org/en/latest/), [pip](https://pip.pypa.io/en/stable/) and [make](https://www.gnu.org/software/make/) to get started. If you have all of those, you may proceed :). Otherwise see the Dependencies Section
2. Run `make app_setup` and wait until you see something like `Starting development server at http://127.0.0.1:8000/`
3. Visit http://127.0.0.1:8000/
# Release workflow
## Dependencies
1. clean your workspace (the output of `git status` should be empty)
2. checkout the master branch and update it
3. run `make release-patch`, `make release-feature` or `make release-breaking`
4. describe your changes in the `git tag` message
5. in case of problems: discard your last commit and stop reading here
6. push your updated master branch (`git push ???`) and push the tags (`git push --tags`)
7. deploy the updated master branch on the target host: `make deploy-git`
Depending on your distribution (we assume you’ll be using something like Linux here) the build dependencies of this project will be available via your package manager.
### Debian
For `virtualenv`, `python3`, `flake8` and `pip` use apt:
```sh
apt install make virtualenv python3 python3-flake8 python3-pip
```
`node` is available as `nodejs` and `nodejs-legacy` (please install both), but you’ll have to have Debian Stretch to get a node version that is going to work. The nodejs people also offer pre-packaged up to date builds [here](https://nodejs.org/en/download/package-manager/#debian-and-ubuntu-based-linux-distributions).
`yarn` is not yet available in Debian. Take a look at their [installation manual](https://yarnpkg.com/en/docs/install).
# Local settings
### Arch Linux
Fortunately all of the required packages are available via pacman.
```sh
pacman -Sy make nodejs yarn flake8 python python-virtualenv python-pip
```
Create a file `local_settings.py` and add all settings that you want to
override based on `stadt/settings.py`. The latter file imports all settings
from `local_settings.py` in case this file exists.
## Local Settings
# Database setup
Your local Django settings will be located in `stadt/settings/local.py`. Use `make app_local_settings` to create a default configuration.
## Database Setup
The preconfigured database is a local sqlite file.
For production deployment you should use a database server.
## PostgreSQL
### PostgreSQL
The following statement creates a suitable database including proper collation settings:
CREATE USER stadtgestalten with password 'PUT RANDOM NOISE';
CREATE DATABASE stadtgestalten WITH ENCODING 'UTF8' LC_COLLATE='de_DE.UTF8' LC_CTYPE='de_DE.UTF8' TEMPLATE=template0 OWNER stadtgestalten;
The above command requires the locale 'de_DE.UTF8' in the system of the database server.
# Production deployment
## UWSGI
The following uwsgi configuration is sufficient for running the software:
[uwsgi]
plugins = python3
chdir = /srv/stadtgestalten/stadt
file = wsgi.py
touch-reload = /srv/stadtgestalten/local_settings.py
touch-reload = settings.py
virtualenv = /srv/virtualenvs/stadtgestalten
pythonpath = /srv/stadtgestalten
socket = /var/run/uwsgi/app/stadtgestalten/socket
# anschalten fuer profiling
#env = PROFILING_DIRECTORY=/tmp/profiling-stadtgestalten/
# Switch to maintenance mode
plugins = router_redirect
# "touch-reload" for the offline-marker file is necessary, since "if-exists" is only processed
# during startup (or reload).
touch-reload = /srv/stadtgestalten/_OFFLINE_MARKER_UWSGI
if-exists = /srv/stadtgestalten/_OFFLINE_MARKER_UWSGI
route = .* redirect:https://offline.stadtgestalten.org/
endif =
The command above requires the locale 'de_DE.UTF8' in the system of the database server.
## Production deployment
We recommend to use the provided debian package. It already comes with a UWSGI config.
## Contributing
See [CONTRIBUTING.md](./CONTRIBUTING.md)
stadtgestalten (1.7.3-1) unstable; urgency=medium
* Initial release.
-- Stadtgestalten Maintainers <wir@stadtgestalten.org> Fri, 26 Apr 2017 06:44:00 +0200
Source: stadtgestalten
Section: python
Priority: optional
Maintainer: Stadtgestalten Maintainers <wir@stadtgestalten.org>
Build-Depends: debhelper (>= 9), dh-exec, dh-python, dh-virtualenv,
flake8, python3-all, python3-setuptools, yarn, nodejs, nodejs-legacy
Standards-Version: 3.9.6
Package: stadtgestalten
Architecture: all
Depends: ${misc:Depends}, ${python3:Depends}
Recommends: python3-psycopg2, uwsgi, uwsgi-plugin-python3,
uwsgi-plugin-router-access, ${misc:Recommends}
Suggests: sqlite3, postgresql-client-common
Description: Web platform that enables social action and solidarity
Stadtgestalten is a web-based platform providing tools for groups and
initiatives in a local context.
Copyright 2015 - 2017 Stadtgestalten Contributors
server {
server_name YOUR_HOSTNAME;
error_page 503 = @maintenance;
location / {
uwsgi_pass unix:/var/run/uwsgi/app/stadtgestalten/socket;
include uwsgi_params;
uwsgi_intercept_errors on;
}
location /media/ {
alias /var/lib/stadtgestalten/media/;
}
location /static/ {
alias /usr/share/stadtgestalten/static/;
}
location @maintenance {
root /usr/share/stadtgestalten/offline-website;
try_files /index.html =503;
}
}
# this is the local stadtgestalten configuration
# you may override any options that stadtgestalten provides in its default settings.py file.
# this file is imported once the default settings.py was loaded
from stadt.settings.default import *
#################################################
#
# PLEASE SET THE FOLLOWING OPTIONS
#
#################################################
#
# SECRET_KEY = 'YOUR_SECRET_KEY'
# ALLOWED_HOSTS = ['example.com', 'localhost']
#
# ADMINS = [
# ('Admins', 'administration@example.com'),
# ]
#
# ABOUT_GROUP_ID = 1
#
#################################################
# set debug mode to false
DEBUG = False
# increase session cookie time to 1 year
SESSION_COOKIE_AGE = 60 * 60 * 24 * 365
[uwsgi]
# used plugins
plugins = python3
plugins = router_redirect
# reload this config whenever these file change
touch-reload = /etc/stadtgestalten/maintenance_mode
touch-reload = /etc/stadtgestalten/settings.py
# project run configuration
chdir = /usr/share/stadtgestalten
pythonpath = /usr/share/stadtgestalten
module = stadt.wsgi:application
virtualenv = /usr/share/stadtgestalten/.virtualenv
uid = stadtgestalten
gid = nogroup
# basic process configuration
master = True
vacuum = True
# performance & scaling
workers = 2
threads = 2
# socket configuration
chown-socket = www-data:www-data
chmod-socket = 640
# logging
logto = /var/log/uwsgi/%n.log
# profiling
# env = PROFILING_DIRECTORY=/tmp/profiling-stadtgestalten/
# maintenance mode
if-exists = /etc/stadtgestalten/maintenance_mode
route = .* break:503
endif =
#!/bin/sh
set -eu
EXEC_USER=stadtgestalten
EXEC_BASE=/usr/share/stadtgestalten
if [ "$(id -nu)" = "$EXEC_USER" ]; then
. "$EXEC_BASE/.virtualenv/bin/activate"
python3 "$EXEC_BASE/manage.py" "$@"
elif [ "$(id -u)" = 0 ]; then
su -s "$0" "$EXEC_USER" -- "$@"
else
echo "please run stadtctl as root or '$EXEC_USER'" >&2
exit 1
fi
#!/usr/bin/make -f
export DH_VIRTUALENV_INSTALL_ROOT=/usr/share/stadtgestalten
%:
dh $@ --with python-virtualenv --upgrade-pip --python /usr/bin/python3 --use-system-packages \
--skip-install --install-suffix .virtualenv
override_dh_install:
@# dh-virtualenv disabled dh_auto_install by default, because it normally executes
@# `setup.py install` and considers that a full installation (which may likely be fine).
@# As we skip dh-virtualenvs install step in order to prepare the package for installation in
@# `/usr/share/stadtgestalten` we sill want to run auto_install to call `make install`
dh_auto_install
dh_install
override_dh_fixperms:
dh_fixperms
chmod 640 debian/stadtgestalten/etc/stadtgestalten/settings.py
extend-diff-ignore="/__pycache__/"
#!/usr/bin/dh-exec
debian/root/etc/uwsgi/apps-available/stadtgestalten.ini => /etc/uwsgi/apps-available/stadtgestalten.ini
debian/root/etc/stadtgestalten/settings.py => /etc/stadtgestalten/settings.py
debian/root/usr/bin/stadtctl => /usr/bin/stadtctl
debian/root/var/backups/stadtgestalten/README => /var/backups/stadtgestalten/README
etc/stadtgestalten/settings.py /usr/share/stadtgestalten/stadt/settings/local.py
#!/bin/sh
set -eu
PKG_USER="stadtgestalten"
DIR_ETC="/etc/$PKG_USER"
DIR_HOME="/var/lib/$PKG_USER"
if [ "$1" = "configure" ]; then
if ! getent passwd "$PKG_USER" >/dev/null; then
adduser --quiet --system --disabled-password \
--home "$DIR_HOME" "$PKG_USER"
fi
if [ -f "/etc/uwsgi/apps-enabled/$PKG_USER.ini" ]; then
if /usr/bin/stadtctl migrate --no-input >/dev/null; then
rm -f "/etc/$PKG_USER/maintenance_mode"
else
echo "error while executing $PKG_USER migrations. maintenance mode still active" >&2
fi
fi
chown "$PKG_USER:root" /var/backups/stadtgestalten
chown "$PKG_USER:root" "$DIR_ETC/settings.py"
fi
set +eu
# dh_installdeb will replace this with shell code automatically
# generated by other debhelper scripts.
#DEBHELPER#
exit 0