From a47890f366cc10b193a70462cbee9526ad7e7216 Mon Sep 17 00:00:00 2001 From: Dustin Pianalto Date: Sat, 14 Sep 2019 11:28:53 -0800 Subject: [PATCH] Lots of changes --- .gitignore | 2 + .vscode/settings.json | 3 + config/settings/base.py | 2 +- docker-compose.yml | 60 +++ entrypoint | 45 ++ geeksbot/__init__.py | 0 geeksbot/__main__.py | 168 +++++++ geeksbot/config/bot_config.json | 5 + geeksbot/config/restart | 1 + geeksbot/exts/admin.py | 76 +++ geeksbot/imports/checks.py | 0 geeksbot/imports/utils.py | 362 ++++++++++++++ requirements/base.txt | 13 - requirements/geeksbot.txt | 8 + requirements/production.txt | 7 - requirements/web.txt | 20 + services/Dockerfile-base | 30 ++ services/Dockerfile-geeksbot | 12 + services/Dockerfile-web | 36 ++ services/postgresql/postgres.conf | 690 ++++++++++++++++++++++++++ services/web/geeksbot.conf | 48 ++ services/web/gunicorn.conf | 10 + services/web/nginx.conf | 44 ++ services/web/supervisor_geeksbot.conf | 15 + services/web/supervisord.conf | 21 + 25 files changed, 1657 insertions(+), 21 deletions(-) create mode 100644 .vscode/settings.json create mode 100644 docker-compose.yml create mode 100644 entrypoint create mode 100644 geeksbot/__init__.py create mode 100644 geeksbot/__main__.py create mode 100644 geeksbot/config/bot_config.json create mode 100644 geeksbot/config/restart create mode 100644 geeksbot/exts/admin.py create mode 100644 geeksbot/imports/checks.py create mode 100644 geeksbot/imports/utils.py create mode 100644 requirements/geeksbot.txt create mode 100644 requirements/web.txt create mode 100644 services/Dockerfile-base create mode 100644 services/Dockerfile-geeksbot create mode 100644 services/Dockerfile-web create mode 100644 services/postgresql/postgres.conf create mode 100644 services/web/geeksbot.conf create mode 100644 services/web/gunicorn.conf create mode 100644 services/web/nginx.conf create mode 100644 services/web/supervisor_geeksbot.conf create mode 100644 services/web/supervisord.conf diff --git a/.gitignore b/.gitignore index 67c8aae..fa38240 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ __pycache__/ *.py[cod] *$py.class +ssl_certs + # C extensions *.so diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..e9d3c8e --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "/home/dustyp/.virtualenvs/geeksbot/bin/python" +} \ No newline at end of file diff --git a/config/settings/base.py b/config/settings/base.py index 4bff7ac..a03a820 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -11,7 +11,7 @@ APPS_DIR = ROOT_DIR.path("geeksbot_v2") env = environ.Env() -READ_DOT_ENV_FILE = env.bool("DJANGO_READ_DOT_ENV_FILE", default=False) +READ_DOT_ENV_FILE = env.bool("DJANGO_READ_DOT_ENV_FILE", default=True) if READ_DOT_ENV_FILE: # OS environment variables take precedence over variables from .env env.read_env(str(ROOT_DIR.path(".env"))) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..2ccf48c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,60 @@ +version: '3' + +services: + base: + build: + context: . + dockerfile: "${PWD}/services/Dockerfile-base" + image: base:latest + + db: + image: postgres + ports: + - "5432:5432" + volumes: + - "${PWD}/services/postgresql/postgres.conf:/etc/postgresql/postgresql.conf" + - "db:/var/lib/postgresql/data:rw" + env_file: ${PWD}/.env + redis: + image: redis:5.0.3 + ports: + - "6379:6379" + web: + build: + context: . + dockerfile: "${PWD}/services/Dockerfile-web" + env_file: ${PWD}/.env + ports: + - "80:80" + - "8000:8000" + - "443:443" + depends_on: + - db + - redis + - base + environment: + - DATABASE_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DB} + - REDIS_URL=redis://${REDIS_HOST}:${REDIS_PORT}/${REDIS_DB} + volumes: + - ${PWD}/config:/code/config + - ${PWD}/geeksbot_web:/code/geeksbot_web + - ${PWD}/geeksbot_v2:/code/geeksbot_v2 + geeksbot: + build: + context: . + dockerfile: "${PWD}/services/Dockerfile-geeksbot" + env_file: ${PWD}/.env + depends_on: + - db + - redis + - base + - web + volumes: + - ${PWD}/geeksbot:/code/geeksbot + - ~/.ssh/id_rsa:/root/.ssh/id_rsa + - ~/.ssh/id_rsa.pub:/root/.ssh/authorized_keys + - ~/.ssh/known_hosts:/root/.ssh/known_hosts + +volumes: + db: + external: true diff --git a/entrypoint b/entrypoint new file mode 100644 index 0000000..941bd8e --- /dev/null +++ b/entrypoint @@ -0,0 +1,45 @@ +#!/bin/sh + +set -o errexit +set -o pipefail +set -o nounset + + + + +if [ -z "${POSTGRES_USER}" ]; then + base_postgres_image_default_user='postgres' + export POSTGRES_USER="${base_postgres_image_default_user}" +fi + +postgres_ready() { +python << END +import sys + +import psycopg2 + +try: + psycopg2.connect( + dbname="${POSTGRES_DB}", + user="${POSTGRES_USER}", + password="${POSTGRES_PASSWORD}", + host="${POSTGRES_HOST}", + port="${POSTGRES_PORT}", + ) +except psycopg2.OperationalError: + sys.exit(-1) +sys.exit(0) + +END +} +until postgres_ready; do + >&2 echo 'Waiting for PostgreSQL to become available...' + sleep 1 +done +>&2 echo 'PostgreSQL is available' + +python manage.py collectstatic --noinput +python manage.py makemigrations +python manage.py migrate + +/usr/bin/supervisord -c /etc/supervisor/supervisord.conf diff --git a/geeksbot/__init__.py b/geeksbot/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/geeksbot/__main__.py b/geeksbot/__main__.py new file mode 100644 index 0000000..07dc3a6 --- /dev/null +++ b/geeksbot/__main__.py @@ -0,0 +1,168 @@ +import logging +import time +from datetime import datetime +import os + +log_format = '{asctime}.{msecs:03.0f}|{levelname:<8}|{name}::{message}' +date_format = '%Y.%m.%d %H.%M.%S' + +log_dir = '/tmp/logs/geeksbot' + +if not os.path.exists(log_dir): + os.makedirs(log_dir) + +log_file = '{0}/geeksbot_{1}.log'.format(log_dir, datetime.now().strftime('%Y%m%d_%H%M%S%f')) + +ch = logging.StreamHandler() +fh = logging.FileHandler(log_file) +ch.setLevel(logging.INFO) +fh.setLevel(logging.INFO) +formatter = logging.Formatter(log_format, datefmt=date_format, style='{') +ch.setFormatter(formatter) +fh.setFormatter(formatter) +logging.basicConfig(level=logging.INFO, handlers=[ch, fh]) +logger = logging.getLogger() +logger.info('Logging Setup Complete') + +time.sleep(1) + +logger.info('Starting Imports') + +start = datetime.utcnow() +# noinspection PyPackageRequirements +import discord +logger.info('Discord.py Imported') +# noinspection PyPackageRequirements +from discord.ext import commands +logger.info('commands Imported') +# noinspection PyPackageRequirements +from discord.ext.commands.view import StringView +logger.info('StringView Imported') +# noinspection PyPackageRequirements +from discord.ext.commands.context import Context +logger.info('Context Imported') +logger.info(f'Discord.py Import Complete - Took {(datetime.utcnow() - start).total_seconds()} seconds') +# noinspection PyRedeclaration +start = datetime.utcnow() +from concurrent import futures +logger.info('Concurrent futures Imported') +from multiprocessing import Pool +logger.info('Multiprocesing Pool Imported') +logger.info(f'Process Libs Import Complete - Took {(datetime.utcnow() - start).total_seconds()} seconds') +# noinspection PyRedeclaration +start = datetime.utcnow() +import re +logger.info('re Imported') +from typing import Dict +logger.info('Typing Dict Imported') +import json +logger.info('JSON Imported') +import aiohttp +logger.info('aiohttp Imported') +logger.info(f'Misc Libs Import Complete - Took {(datetime.utcnow() - start).total_seconds()} seconds') +# noinspection PyRedeclaration +# start = datetime.utcnow() +# from geeksbot.imports.rcon_lib import arcon +# from geeksbot.imports import message_logging +# logger.info(f'Geeksbot Libs Import Complete - Took {(datetime.utcnow() - start).total_seconds()} seconds') + + +logger.info('Imports Complete') + + +class Geeksbot(commands.Bot): + def __init__(self, *args, **kwargs): + self.default_prefix = 'g#' + kwargs['command_prefix'] = self.default_prefix + self.description = "Geeksbot v2" + kwargs['description'] = self.description + super().__init__(*args, **kwargs) + self.config_dir = 'geeksbot/config/' + self.config_file = 'bot_config.json' + self.extension_dir = 'exts' + self.aio_session = aiohttp.ClientSession(loop=self.loop) + with open(f'{self.config_dir}{self.config_file}') as f: + self.bot_config = json.load(f) + self.embed_color = discord.Colour.from_rgb(49, 107, 111) + self.error_color = discord.Colour.from_rgb(142, 29, 31) + self.owner_id = 351794468870946827 + self.tpe = futures.ThreadPoolExecutor(max_workers=20) + self.process_pool = Pool(processes=4) + self.geo_api = '2d4e419c2be04c8abe91cb5dd1548c72' + self.git_url = 'https://github.com/dustinpianalto/geeksbot_v2' + self.load_default_extensions() + + async def load_ext(self, mod): + self.load_extension(f'geeksbot.{self.extension_dir}.{mod}') + logger.info(f'Extension Loaded: {mod}') + + async def unload_ext(self, mod): + self.unload_extension(f'geeksbot.{self.extension_dir}.{mod}') + logger.info(f'Extension Loaded: {mod}') + + def load_default_extensions(self): + for load_item in self.bot_config['load_list']: + self.loop.create_task(self.load_ext(load_item)) + + async def close(self): + await super().close() + await self.aio_session.close() + + +bot = Geeksbot(case_insensitive=True) + + +@bot.command(hidden=True) +@commands.is_owner() +async def load(ctx, mod=None): + """Allows the owner to load extensions dynamically""" + await bot.load_ext(mod) + await ctx.send(f'{mod} loaded') + + +@bot.command(hidden=True) +@commands.is_owner() +async def reload(ctx, mod=None): + """Allows the owner to reload extensions dynamically""" + if mod == 'all': + load_list = bot.bot_config['load_list'] + for load_item in load_list: + await bot.unload_ext(f'{load_item}') + await bot.load_ext(f'{load_item}') + await ctx.send(f'{load_item} reloaded') + else: + await bot.unload_ext(mod) + await bot.load_ext(mod) + await ctx.send(f'{mod} reloaded') + + +@bot.command(hidden=True) +@commands.is_owner() +async def unload(ctx, mod): + """Allows the owner to unload extensions dynamically""" + await bot.unload_ext(mod) + await ctx.send(f'{mod} unloaded') + + +@bot.event +async def on_message(message): + if message.guild: + message.content = message.content.replace('@everyone', '@\uFFF0everyone').replace('@here', '@\uFFF0here') + await bot.process_commands(message) + + +@bot.event +async def on_ready(): + logger.info('Logged in as {0.name}|{0.id}'.format(bot.user)) + guild = bot.get_guild(396156980974059531) + channel = guild.get_channel(404569276012560386) + await channel.send('Geeksbot v2 Running') + logger.info('Done loading, Geeksbot is active.') + with open(f'{bot.config_dir}restart') as f: + reboot = f.readlines() + if int(reboot[0]) == 1: + await bot.get_channel(int(reboot[1])).send('Restart Finished.') + with open(f'{bot.config_dir}restart', 'w') as f: + f.write('0') + +bot.run(os.environ['TOKEN']) diff --git a/geeksbot/config/bot_config.json b/geeksbot/config/bot_config.json new file mode 100644 index 0000000..0d28390 --- /dev/null +++ b/geeksbot/config/bot_config.json @@ -0,0 +1,5 @@ +{ + "load_list": [ + "admin" + ] +} \ No newline at end of file diff --git a/geeksbot/config/restart b/geeksbot/config/restart new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/geeksbot/config/restart @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/geeksbot/exts/admin.py b/geeksbot/exts/admin.py new file mode 100644 index 0000000..5702e22 --- /dev/null +++ b/geeksbot/exts/admin.py @@ -0,0 +1,76 @@ +import discord +from discord.ext import commands +import logging +import inspect +import os +import psutil +import math +from geeksbot.imports import utils + +admin_logger = logging.getLogger('admin') + + +class Admin(commands.Cog): + def __init__(self, bot): + self.bot = bot + + @commands.command(hidden=True) + @commands.is_owner() + async def reboot(self, ctx): + await ctx.send('Geeksbot is restarting.') + with open(f'/mnt/{self.bot.SECRETS_BUCKET_NAME}/restart', 'w') as f: + f.write(f'1\n{ctx.channel.id}') + admin_logger.info("Rebooting") + # noinspection PyProtectedMember + os._exit(1) + + # TODO Fix view_code + @commands.command(hidden=True) + @commands.is_owner() + async def view_code(self, ctx, code_name): + pag = utils.Paginator(self.bot, prefix='```py', suffix='```') + pag.add(inspect.getsource(self.bot.all_commands[code_name].callback)) + book = utils.Book(pag, (None, ctx.channel, ctx.bot, ctx.message)) + await book.create_book() + + @commands.command(hidden=True) + @commands.is_owner() + async def sysinfo(self, ctx): + """Gets system status for my server.""" + await ctx.send(f'```ml\n' + f'CPU Percentages: {psutil.cpu_percent(percpu=True)}\n' + f'Memory Usage: {psutil.virtual_memory().percent}%\n' + f'Disc Usage: {psutil.disk_usage("/").percent}%\n' + f'```') + + @commands.command(aliases=['oauth', 'link']) + @commands.cooldown(1, 5, type=commands.BucketType.user) + async def invite(self, ctx, guy: discord.User = None): + """Shows you the bot's invite link. + If you pass in an ID of another bot, it gives you the invite link to that bot. + """ + guy = guy or self.bot.user + url = discord.utils.oauth_url(guy.id) + await ctx.send(f'**<{url}>**') + + @commands.command() + @commands.cooldown(1, 5, type=commands.BucketType.user) + async def ping(self, ctx): + """Check the Bot\'s connection to Discord + + For more detailed information set the as comp and it will test the ping + number of times.""" + em = discord.Embed(style='rich', + title=f'Pong 🏓', + color=discord.Colour.green() + ) + msg = await ctx.send(embed=em) + time1 = ctx.message.created_at + time = (msg.created_at - time1).total_seconds() * 1000 + em.description = f'Response Time: **{math.ceil(time)}ms**\n' \ + f'Discord Latency: **{math.ceil(self.bot.latency*1000)}ms**' + await msg.edit(embed=em) + + +def setup(bot): + bot.add_cog(Admin(bot)) diff --git a/geeksbot/imports/checks.py b/geeksbot/imports/checks.py new file mode 100644 index 0000000..e69de29 diff --git a/geeksbot/imports/utils.py b/geeksbot/imports/utils.py new file mode 100644 index 0000000..a06c92d --- /dev/null +++ b/geeksbot/imports/utils.py @@ -0,0 +1,362 @@ +import discord +import asyncio +import typing + + +# noinspection PyShadowingNames +class Paginator: + def __init__(self, + bot: discord.ext.commands.Bot, + *, + max_chars: int = 1970, + max_lines: int = 20, + prefix: str = '```md', + suffix: str = '```', + page_break: str = '\uFFF8', + field_break: str = '\uFFF7', + field_name_char: str = '\uFFF6', + inline_char: str = '\uFFF5', + max_line_length: int = 100, + embed=False): + _max_len = 6000 if embed else 1980 + assert 0 < max_lines <= max_chars + + self._parts = list() + self._prefix = prefix + self._suffix = suffix + self._max_chars = max_chars if max_chars + len(prefix) + len(suffix) + 2 <= _max_len \ + else _max_len - len(prefix) - len(suffix) - 2 + self._max_lines = max_lines - (prefix + suffix).count('\n') + 1 + self._page_break = page_break + self._max_line_length = max_line_length + self._pages = list() + self._max_field_chars = 1014 + self._max_field_name = 256 + self._max_description = 2048 + self._embed = embed + self._field_break = field_break + self._field_name_char = field_name_char + self._inline_char = inline_char + self._embed_title = '' + self._embed_description = '' + self._embed_color = None + self._embed_thumbnail = None + self._embed_url = None + self._bot = bot + + def set_embed_meta(self, title: str=None, + description: str=None, + color: discord.Colour=None, + thumbnail: str=None, + footer: str='', + url: str=None): + if title and len(title) > self._max_field_name: + raise RuntimeError('Provided Title is too long') + else: + self._embed_title = title + if description and len(description) > self._max_description: + raise RuntimeError('Provided Description is too long') + else: + self._embed_description = description + self._embed_color = color + self._embed_thumbnail = thumbnail + self._embed_url = url + + def pages(self) -> typing.List[str]: + _pages = list() + _fields = list() + _page = '' + _lines = 0 + _field_name = '' + _field_value = '' + _inline = False + + def open_page(): + nonlocal _page, _lines, _fields + if not self._embed: + _page = self._prefix + _lines = 0 + else: + _fields = list() + + def close_page(): + nonlocal _page, _lines, _fields + if not self._embed: + _page += self._suffix + _pages.append(_page) + else: + if _fields: + _pages.append(_fields) + open_page() + + open_page() + + if not self._embed: + for part in [str(p) for p in self._parts]: + if part == self._page_break: + close_page() + + new_chars = len(_page) + len(part) + + if new_chars > self._max_chars: + close_page() + elif (_lines + (part.count('\n') + 1 or 1)) > self._max_lines: + close_page() + + _lines += (part.count('\n') + 1 or 1) + _page += '\n' + part + else: + def open_field(name: str): + nonlocal _field_value, _field_name + _field_name = name + _field_value = self._prefix + + def close_field(next_name: str=None): + nonlocal _field_name, _field_value, _fields + _field_value += self._suffix + if _field_value != self._prefix + self._suffix: + _fields.append({'name': _field_name, 'value': _field_value, 'inline': _inline}) + if next_name: + open_field(next_name) + + open_field('\uFFF0') + + for part in [str(p) for p in self._parts]: + if part.strip() == self._page_break: + close_page() + continue + elif part == self._field_break: + if len(_fields) + 1 < 25: + close_field(next_name='\uFFF0') + else: + close_field() + close_page() + continue + + if part.startswith(self._field_name_char): + part = part.replace(self._field_name_char, '') + if part.startswith(self._inline_char): + _inline = True + part = part.replace(self._inline_char, '') + else: + _inline = False + if _field_value and _field_value != self._prefix: + close_field(part) + else: + _field_name = part + continue + + _field_value += '\n' + part + + close_field() + + close_page() + self._pages = _pages + return _pages + + # noinspection PyUnresolvedReferences + def process_pages(self) -> typing.List[str]: + _pages = self._pages or self.pages() + _len_pages = len(_pages) + _len_page_str = len(f'{_len_pages}/{_len_pages}') + if not self._embed: + for i, page in enumerate(_pages): + if len(page) + _len_page_str <= 2000: + _pages[i] = f'{i + 1}/{_len_pages}\n{page}' + else: + for i, page in enumerate(_pages): + em = discord.Embed(title=self._embed_title, + description=self._embed_description, + color=self._bot.embed_color, + ) + if self._embed_thumbnail: + em.set_thumbnail(url=self._embed_thumbnail) + if self._embed_url: + em.url = self._embed_url + if self._embed_color: + em.color = self._embed_color + em.set_footer(text=f'{i + 1}/{_len_pages}') + for field in page: + em.add_field(name=field['name'], value=field['value'], inline=field['inline']) + _pages[i] = em + return _pages + + def __len__(self): + return sum(len(p) for p in self._parts) + + def __eq__(self, other): + # noinspection PyProtectedMember + return self.__class__ == other.__class__ and self._parts == other._parts + + def add_page_break(self, *, to_beginning: bool=False) -> None: + self.add(self._page_break, to_beginning=to_beginning) + + def add(self, item: typing.Any, *, to_beginning: bool=False, keep_intact: bool=False, truncate=False) -> None: + item = str(item) + i = 0 + if not keep_intact and not item == self._page_break: + item_parts = item.strip('\n').split('\n') + for part in item_parts: + if len(part) > self._max_line_length: + if not truncate: + length = 0 + out_str = '' + + def close_line(line): + nonlocal i, out_str, length + self._parts.insert(i, out_str) if to_beginning else self._parts.append(out_str) + i += 1 + out_str = line + ' ' + length = len(out_str) + + bits = part.split(' ') + for bit in bits: + next_len = length + len(bit) + 1 + if next_len <= self._max_line_length: + out_str += bit + ' ' + length = next_len + elif len(bit) > self._max_line_length: + if out_str: + close_line(line='') + for out_str in [bit[i:i + self._max_line_length] + for i in range(0, len(bit), self._max_line_length)]: + close_line('') + else: + close_line(bit) + close_line('') + else: + line = f'{part:.{self._max_line_length-3}}...' + self._parts.insert(i, line) if to_beginning else self._parts.append(line) + else: + self._parts.insert(i, part) if to_beginning else self._parts.append(part) + i += 1 + elif keep_intact and not item == self._page_break: + if len(item) >= self._max_chars or item.count('\n') > self._max_lines: + raise RuntimeError('{item} is too long to keep on a single page and is marked to keep intact.') + if to_beginning: + self._parts.insert(0, item) + else: + self._parts.append(item) + else: + if to_beginning: + self._parts.insert(0, item) + else: + self._parts.append(item) + + +class Book: + def __init__(self, pag: Paginator, ctx: typing.Tuple[typing.Optional[discord.Message], + discord.TextChannel, + discord.ext.commands.Bot, + discord.Message]) -> None: + self._pages = pag.process_pages() + self._len_pages = len(self._pages) + self._current_page = 0 + self._message, self._channel, self._bot, self._calling_message = ctx + self._locked = True + if pag == Paginator(self._bot): + raise RuntimeError('Cannot create a book out of an empty Paginator.') + + def advance_page(self) -> None: + self._current_page += 1 + if self._current_page >= self._len_pages: + self._current_page = 0 + + def reverse_page(self) -> None: + self._current_page += -1 + if self._current_page < 0: + self._current_page = self._len_pages - 1 + + async def display_page(self) -> None: + if isinstance(self._pages[self._current_page], discord.Embed): + if self._message: + await self._message.edit(content=None, embed=self._pages[self._current_page]) + else: + self._message = await self._channel.send(embed=self._pages[self._current_page]) + else: + if self._message: + await self._message.edit(content=self._pages[self._current_page], embed=None) + else: + self._message = await self._channel.send(self._pages[self._current_page]) + + async def create_book(self) -> None: + # noinspection PyUnresolvedReferences + async def reaction_checker(): + # noinspection PyShadowingNames + def check(reaction, user): + if self._locked: + return str(reaction.emoji) in self._bot.book_emojis.values() \ + and user == self._calling_message.author \ + and reaction.message.id == self._message.id + else: + return str(reaction.emoji) in self._bot.book_emojis.values() \ + and reaction.message.id == self._message.id + + await self.display_page() + + if len(self._pages) > 1: + for emoji in self._bot.book_emojis.values(): + try: + await self._message.add_reaction(emoji) + except (discord.Forbidden, KeyError): + pass + else: + try: + await self._message.add_reaction(self._bot.book_emojis['unlock']) + await self._message.add_reaction(self._bot.book_emojis['close']) + except (discord.Forbidden, KeyError): + pass + + while True: + try: + reaction, user = await self._bot.wait_for('reaction_add', timeout=60, check=check) + except asyncio.TimeoutError: + try: + await self._message.clear_reactions() + except (discord.Forbidden, discord.NotFound): + pass + raise asyncio.CancelledError + else: + await self._message.remove_reaction(reaction, user) + if str(reaction.emoji) == self._bot.book_emojis['close']: + await self._calling_message.delete() + await self._message.delete() + raise asyncio.CancelledError + elif str(reaction.emoji) == self._bot.book_emojis['forward']: + self.advance_page() + elif str(reaction.emoji) == self._bot.book_emojis['back']: + self.reverse_page() + elif str(reaction.emoji) == self._bot.book_emojis['end']: + self._current_page = self._len_pages - 1 + elif str(reaction.emoji) == self._bot.book_emojis['start']: + self._current_page = 0 + elif str(reaction.emoji) == self._bot.book_emojis['hash']: + m = await self._channel.send(f'Please enter a number in range 1 to {self._len_pages}') + + def num_check(message): + if self._locked: + return message.content.isdigit() \ + and 0 < int(message.content) <= self._len_pages \ + and message.author == self._calling_message.author + else: + return message.content.isdigit() \ + and 0 < int(message.content) <= self._len_pages + + try: + msg = await self._bot.wait_for('message', timeout=30, check=num_check) + except asyncio.TimeoutError: + await m.edit(content='Message Timed out.') + else: + self._current_page = int(msg.content) - 1 + try: + await m.delete() + await msg.delete() + except (discord.Forbidden, discord.NotFound): + pass + elif str(reaction.emoji) == self._bot.book_emojis['unlock']: + self._locked = False + await self._message.remove_reaction(reaction, self._channel.guild.me) + continue + await self.display_page() + + self._bot.loop.create_task(reaction_checker()) diff --git a/requirements/base.txt b/requirements/base.txt index 08a629d..d28608c 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -3,16 +3,3 @@ python-slugify==3.0.3 # https://github.com/un33k/python-slugify Pillow==6.1.0 # https://github.com/python-pillow/Pillow argon2-cffi==19.1.0 # https://github.com/hynek/argon2_cffi redis==3.3.8 # https://github.com/antirez/redis - -# Django -# ------------------------------------------------------------------------------ -django==2.2.4 # pyup: < 3.0 # https://www.djangoproject.com/ -django-environ==0.4.5 # https://github.com/joke2k/django-environ -django-model-utils==3.2.0 # https://github.com/jazzband/django-model-utils -django-allauth==0.39.1 # https://github.com/pennersr/django-allauth -django-crispy-forms==1.7.2 # https://github.com/django-crispy-forms/django-crispy-forms -django-redis==4.10.0 # https://github.com/niwinz/django-redis - -# Django REST Framework -djangorestframework==3.10.2 # https://github.com/encode/django-rest-framework -coreapi==2.3.3 # https://github.com/core-api/python-client diff --git a/requirements/geeksbot.txt b/requirements/geeksbot.txt new file mode 100644 index 0000000..15a787f --- /dev/null +++ b/requirements/geeksbot.txt @@ -0,0 +1,8 @@ +discord.py +aiohttp<3.6.0 +aiofiles +python-dateutil +psutil +pytz +async_timeout +cached_property \ No newline at end of file diff --git a/requirements/production.txt b/requirements/production.txt index bc16f0a..94a5caa 100644 --- a/requirements/production.txt +++ b/requirements/production.txt @@ -2,10 +2,3 @@ -r ./base.txt -gunicorn==19.9.0 # https://github.com/benoitc/gunicorn -psycopg2==2.8.3 --no-binary psycopg2 # https://github.com/psycopg/psycopg2 -Collectfast==1.0.0 # https://github.com/antonagestam/collectfast - -# Django -# ------------------------------------------------------------------------------ -django-anymail[mailgun]==6.1.0 # https://github.com/anymail/django-anymail diff --git a/requirements/web.txt b/requirements/web.txt new file mode 100644 index 0000000..1d63278 --- /dev/null +++ b/requirements/web.txt @@ -0,0 +1,20 @@ +# Django +# ------------------------------------------------------------------------------ +django==2.2.4 # pyup: < 3.0 # https://www.djangoproject.com/ +django-environ==0.4.5 # https://github.com/joke2k/django-environ +django-model-utils==3.2.0 # https://github.com/jazzband/django-model-utils +django-allauth==0.39.1 # https://github.com/pennersr/django-allauth +django-crispy-forms==1.7.2 # https://github.com/django-crispy-forms/django-crispy-forms +django-redis==4.10.0 # https://github.com/niwinz/django-redis +django-anymail[mailgun]==6.1.0 # https://github.com/anymail/django-anymail +django-debug-toolbar +django-extensions + +# Django REST Framework +djangorestframework==3.10.2 # https://github.com/encode/django-rest-framework +coreapi==2.3.3 # https://github.com/core-api/python-client + +gevent +gunicorn==19.9.0 # https://github.com/benoitc/gunicorn +psycopg2==2.8.3 --no-binary psycopg2 # https://github.com/psycopg/psycopg2 +Collectfast==1.0.0 # https://github.com/antonagestam/collectfast diff --git a/services/Dockerfile-base b/services/Dockerfile-base new file mode 100644 index 0000000..220ee85 --- /dev/null +++ b/services/Dockerfile-base @@ -0,0 +1,30 @@ +FROM python:3.7-alpine AS base + +ENV DEBIAN_FRONTEND noninteractive +ENV PYTHONUNBUFFERED 1 + +RUN adduser --disabled-password --home=/home/geeksbot --gecos "" geeksbot +RUN echo "geeksbot ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers +RUN echo "geeksbot:docker" | chpasswd + +RUN apk update && \ + apk add --virtual build-deps gcc python3-dev musl-dev postgresql-dev \ + # Pillow dependencies + && apk add jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev \ + # CFFI dependencies + && apk add libffi-dev py-cffi \ + # Translations dependencies + && apk add gettext \ + # https://docs.djangoproject.com/en/dev/ref/django-admin/#dbshell + && apk add postgresql-client + +RUN mkdir /code +WORKDIR /code + +ENV LC_ALL C.UTF-8 +ENV LANG C.UTF-8 + +RUN pip install --upgrade pip +RUN pip install virtualenv + +RUN apk update && apk add postgresql-client diff --git a/services/Dockerfile-geeksbot b/services/Dockerfile-geeksbot new file mode 100644 index 0000000..27d2248 --- /dev/null +++ b/services/Dockerfile-geeksbot @@ -0,0 +1,12 @@ +FROM base AS geeksbot + +WORKDIR /code + +COPY requirements/base.txt . +COPY requirements/production.txt . +COPY requirements/geeksbot.txt . + +RUN pip install -r production.txt +RUN pip install -r geeksbot.txt + +CMD ["python", "-m", "geeksbot"] diff --git a/services/Dockerfile-web b/services/Dockerfile-web new file mode 100644 index 0000000..b4081a4 --- /dev/null +++ b/services/Dockerfile-web @@ -0,0 +1,36 @@ +FROM base AS web + +WORKDIR /code + +RUN apk update && apk add nginx && apk add supervisor + +COPY requirements/base.txt . +COPY requirements/production.txt . +COPY requirements/web.txt . + +RUN pip install -r production.txt +RUN pip install -r web.txt + +RUN rm -f /etc/nginx/sites-enabled/default +RUN rm -f /etc/nginx/conf.d/default.conf +COPY ./services/web/nginx.conf /etc/nginx/nginx.conf +COPY ./services/web/geeksbot.conf /etc/nginx/sites-enabled/geeksbot +COPY ./services/web/gunicorn.conf /etc/gunicorn.conf +COPY ./services/web/supervisord.conf /etc/supervisor/supervisord.conf +COPY ./services/web/supervisor_geeksbot.conf /etc/supervisor/conf.d/geeksbot.conf +COPY ./ssl_certs/geeksbot_app/geeksbot_app_cert_chain.crt /etc/ssl/geeksbot_app_cert_chain.crt +COPY ./ssl_certs/geeksbot_app/geeksbot.app.key /etc/ssl/geeksbot.app.key + +RUN rm -rf /tmp/* + +RUN mkdir -p /tmp/logs/nginx +RUN mkdir -p /tmp/logs/geeksbot + +COPY manage.py . +copy entrypoint . +RUN sed -i 's/\r$//g' /code/entrypoint +RUN chmod +x /code/entrypoint + +EXPOSE 80 8000 443 + +ENTRYPOINT [ "/code/entrypoint" ] diff --git a/services/postgresql/postgres.conf b/services/postgresql/postgres.conf new file mode 100644 index 0000000..0415c2c --- /dev/null +++ b/services/postgresql/postgres.conf @@ -0,0 +1,690 @@ +# ----------------------------- +# PostgreSQL configuration file +# ----------------------------- +# +# This file consists of lines of the form: +# +# name = value +# +# (The "=" is optional.) Whitespace may be used. Comments are introduced with +# "#" anywhere on a line. The complete list of parameter names and allowed +# values can be found in the PostgreSQL documentation. +# +# The commented-out settings shown in this file represent the default values. +# Re-commenting a setting is NOT sufficient to revert it to the default value; +# you need to reload the server. +# +# This file is read on server startup and when the server receives a SIGHUP +# signal. If you edit the file on a running system, you have to SIGHUP the +# server for the changes to take effect, run "pg_ctl reload", or execute +# "SELECT pg_reload_conf()". Some parameters, which are marked below, +# require a server shutdown and restart to take effect. +# +# Any parameter can also be given as a command-line option to the server, e.g., +# "postgres -c log_connections=on". Some parameters can be changed at run time +# with the "SET" SQL command. +# +# Memory units: kB = kilobytes Time units: ms = milliseconds +# MB = megabytes s = seconds +# GB = gigabytes min = minutes +# TB = terabytes h = hours +# d = days + + +#------------------------------------------------------------------------------ +# FILE LOCATIONS +#------------------------------------------------------------------------------ + +# The default values of these variables are driven from the -D command-line +# option or PGDATA environment variable, represented here as ConfigDir. + +#data_directory = 'ConfigDir' # use data in another directory + # (change requires restart) +#hba_file = 'ConfigDir/pg_hba.conf' # host-based authentication file + # (change requires restart) +#ident_file = 'ConfigDir/pg_ident.conf' # ident configuration file + # (change requires restart) + +# If external_pid_file is not explicitly set, no extra PID file is written. +#external_pid_file = '' # write an extra PID file + # (change requires restart) + + +#------------------------------------------------------------------------------ +# CONNECTIONS AND AUTHENTICATION +#------------------------------------------------------------------------------ + +# - Connection Settings - + +listen_addresses = '*' + # comma-separated list of addresses; + # defaults to 'localhost'; use '*' for all + # (change requires restart) +#port = 5432 # (change requires restart) +#max_connections = 100 # (change requires restart) +#superuser_reserved_connections = 3 # (change requires restart) +#unix_socket_directories = '/tmp' # comma-separated list of directories + # (change requires restart) +#unix_socket_group = '' # (change requires restart) +#unix_socket_permissions = 0777 # begin with 0 to use octal notation + # (change requires restart) +#bonjour = off # advertise server via Bonjour + # (change requires restart) +#bonjour_name = '' # defaults to the computer name + # (change requires restart) + +# - TCP Keepalives - +# see "man 7 tcp" for details + +#tcp_keepalives_idle = 0 # TCP_KEEPIDLE, in seconds; + # 0 selects the system default +#tcp_keepalives_interval = 0 # TCP_KEEPINTVL, in seconds; + # 0 selects the system default +#tcp_keepalives_count = 0 # TCP_KEEPCNT; + # 0 selects the system default + +# - Authentication - + +#authentication_timeout = 1min # 1s-600s +#password_encryption = md5 # md5 or scram-sha-256 +#db_user_namespace = off + +# GSSAPI using Kerberos +#krb_server_keyfile = '' +#krb_caseins_users = off + +# - SSL - + +#ssl = off +#ssl_ca_file = '' +#ssl_cert_file = 'server.crt' +#ssl_crl_file = '' +#ssl_key_file = 'server.key' +#ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers +#ssl_prefer_server_ciphers = on +#ssl_ecdh_curve = 'prime256v1' +#ssl_dh_params_file = '' +#ssl_passphrase_command = '' +#ssl_passphrase_command_supports_reload = off + + +#------------------------------------------------------------------------------ +# RESOURCE USAGE (except WAL) +#------------------------------------------------------------------------------ + +# - Memory - + +#shared_buffers = 32MB # min 128kB + # (change requires restart) +#huge_pages = try # on, off, or try + # (change requires restart) +#temp_buffers = 8MB # min 800kB +#max_prepared_transactions = 0 # zero disables the feature + # (change requires restart) +# Caution: it is not advisable to set max_prepared_transactions nonzero unless +# you actively intend to use prepared transactions. +#work_mem = 4MB # min 64kB +#maintenance_work_mem = 64MB # min 1MB +#autovacuum_work_mem = -1 # min 1MB, or -1 to use maintenance_work_mem +#max_stack_depth = 2MB # min 100kB +#dynamic_shared_memory_type = posix # the default is the first option + # supported by the operating system: + # posix + # sysv + # windows + # mmap + # use none to disable dynamic shared memory + # (change requires restart) + +# - Disk - + +#temp_file_limit = -1 # limits per-process temp file space + # in kB, or -1 for no limit + +# - Kernel Resources - + +#max_files_per_process = 1000 # min 25 + # (change requires restart) + +# - Cost-Based Vacuum Delay - + +#vacuum_cost_delay = 0 # 0-100 milliseconds +#vacuum_cost_page_hit = 1 # 0-10000 credits +#vacuum_cost_page_miss = 10 # 0-10000 credits +#vacuum_cost_page_dirty = 20 # 0-10000 credits +#vacuum_cost_limit = 200 # 1-10000 credits + +# - Background Writer - + +#bgwriter_delay = 200ms # 10-10000ms between rounds +#bgwriter_lru_maxpages = 100 # max buffers written/round, 0 disables +#bgwriter_lru_multiplier = 2.0 # 0-10.0 multiplier on buffers scanned/round +#bgwriter_flush_after = 0 # measured in pages, 0 disables + +# - Asynchronous Behavior - + +#effective_io_concurrency = 1 # 1-1000; 0 disables prefetching +#max_worker_processes = 8 # (change requires restart) +#max_parallel_maintenance_workers = 2 # taken from max_parallel_workers +#max_parallel_workers_per_gather = 2 # taken from max_parallel_workers +#parallel_leader_participation = on +#max_parallel_workers = 8 # maximum number of max_worker_processes that + # can be used in parallel operations +#old_snapshot_threshold = -1 # 1min-60d; -1 disables; 0 is immediate + # (change requires restart) +#backend_flush_after = 0 # measured in pages, 0 disables + + +#------------------------------------------------------------------------------ +# WRITE-AHEAD LOG +#------------------------------------------------------------------------------ + +# - Settings - + +#wal_level = replica # minimal, replica, or logical + # (change requires restart) +#fsync = on # flush data to disk for crash safety + # (turning this off can cause + # unrecoverable data corruption) +#synchronous_commit = on # synchronization level; + # off, local, remote_write, remote_apply, or on +#wal_sync_method = fsync # the default is the first option + # supported by the operating system: + # open_datasync + # fdatasync (default on Linux) + # fsync + # fsync_writethrough + # open_sync +#full_page_writes = on # recover from partial page writes +#wal_compression = off # enable compression of full-page writes +#wal_log_hints = off # also do full page writes of non-critical updates + # (change requires restart) +#wal_buffers = -1 # min 32kB, -1 sets based on shared_buffers + # (change requires restart) +#wal_writer_delay = 200ms # 1-10000 milliseconds +#wal_writer_flush_after = 1MB # measured in pages, 0 disables + +#commit_delay = 0 # range 0-100000, in microseconds +#commit_siblings = 5 # range 1-1000 + +# - Checkpoints - + +#checkpoint_timeout = 5min # range 30s-1d +#max_wal_size = 1GB +#min_wal_size = 80MB +#checkpoint_completion_target = 0.5 # checkpoint target duration, 0.0 - 1.0 +#checkpoint_flush_after = 0 # measured in pages, 0 disables +#checkpoint_warning = 30s # 0 disables + +# - Archiving - + +#archive_mode = off # enables archiving; off, on, or always + # (change requires restart) +#archive_command = '' # command to use to archive a logfile segment + # placeholders: %p = path of file to archive + # %f = file name only + # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f' +#archive_timeout = 0 # force a logfile segment switch after this + # number of seconds; 0 disables + + +#------------------------------------------------------------------------------ +# REPLICATION +#------------------------------------------------------------------------------ + +# - Sending Servers - + +# Set these on the master and on any standby that will send replication data. + +#max_wal_senders = 10 # max number of walsender processes + # (change requires restart) +#wal_keep_segments = 0 # in logfile segments; 0 disables +#wal_sender_timeout = 60s # in milliseconds; 0 disables + +#max_replication_slots = 10 # max number of replication slots + # (change requires restart) +#track_commit_timestamp = off # collect timestamp of transaction commit + # (change requires restart) + +# - Master Server - + +# These settings are ignored on a standby server. + +#synchronous_standby_names = '' # standby servers that provide sync rep + # method to choose sync standbys, number of sync standbys, + # and comma-separated list of application_name + # from standby(s); '*' = all +#vacuum_defer_cleanup_age = 0 # number of xacts by which cleanup is delayed + +# - Standby Servers - + +# These settings are ignored on a master server. + +#hot_standby = on # "off" disallows queries during recovery + # (change requires restart) +#max_standby_archive_delay = 30s # max delay before canceling queries + # when reading WAL from archive; + # -1 allows indefinite delay +#max_standby_streaming_delay = 30s # max delay before canceling queries + # when reading streaming WAL; + # -1 allows indefinite delay +#wal_receiver_status_interval = 10s # send replies at least this often + # 0 disables +#hot_standby_feedback = off # send info from standby to prevent + # query conflicts +#wal_receiver_timeout = 60s # time that receiver waits for + # communication from master + # in milliseconds; 0 disables +#wal_retrieve_retry_interval = 5s # time to wait before retrying to + # retrieve WAL after a failed attempt + +# - Subscribers - + +# These settings are ignored on a publisher. + +#max_logical_replication_workers = 4 # taken from max_worker_processes + # (change requires restart) +#max_sync_workers_per_subscription = 2 # taken from max_logical_replication_workers + + +#------------------------------------------------------------------------------ +# QUERY TUNING +#------------------------------------------------------------------------------ + +# - Planner Method Configuration - + +#enable_bitmapscan = on +#enable_hashagg = on +#enable_hashjoin = on +#enable_indexscan = on +#enable_indexonlyscan = on +#enable_material = on +#enable_mergejoin = on +#enable_nestloop = on +#enable_parallel_append = on +#enable_seqscan = on +#enable_sort = on +#enable_tidscan = on +#enable_partitionwise_join = off +#enable_partitionwise_aggregate = off +#enable_parallel_hash = on +#enable_partition_pruning = on + +# - Planner Cost Constants - + +#seq_page_cost = 1.0 # measured on an arbitrary scale +#random_page_cost = 4.0 # same scale as above +#cpu_tuple_cost = 0.01 # same scale as above +#cpu_index_tuple_cost = 0.005 # same scale as above +#cpu_operator_cost = 0.0025 # same scale as above +#parallel_tuple_cost = 0.1 # same scale as above +#parallel_setup_cost = 1000.0 # same scale as above + +#jit_above_cost = 100000 # perform JIT compilation if available + # and query more expensive than this; + # -1 disables +#jit_inline_above_cost = 500000 # inline small functions if query is + # more expensive than this; -1 disables +#jit_optimize_above_cost = 500000 # use expensive JIT optimizations if + # query is more expensive than this; + # -1 disables + +#min_parallel_table_scan_size = 8MB +#min_parallel_index_scan_size = 512kB +#effective_cache_size = 4GB + +# - Genetic Query Optimizer - + +#geqo = on +#geqo_threshold = 12 +#geqo_effort = 5 # range 1-10 +#geqo_pool_size = 0 # selects default based on effort +#geqo_generations = 0 # selects default based on effort +#geqo_selection_bias = 2.0 # range 1.5-2.0 +#geqo_seed = 0.0 # range 0.0-1.0 + +# - Other Planner Options - + +#default_statistics_target = 100 # range 1-10000 +#constraint_exclusion = partition # on, off, or partition +#cursor_tuple_fraction = 0.1 # range 0.0-1.0 +#from_collapse_limit = 8 +#join_collapse_limit = 8 # 1 disables collapsing of explicit + # JOIN clauses +#force_parallel_mode = off +#jit = off # allow JIT compilation + + +#------------------------------------------------------------------------------ +# REPORTING AND LOGGING +#------------------------------------------------------------------------------ + +# - Where to Log - + +#log_destination = 'stderr' # Valid values are combinations of + # stderr, csvlog, syslog, and eventlog, + # depending on platform. csvlog + # requires logging_collector to be on. + +# This is used when logging to stderr: +#logging_collector = off # Enable capturing of stderr and csvlog + # into log files. Required to be on for + # csvlogs. + # (change requires restart) + +# These are only used if logging_collector is on: +#log_directory = 'log' # directory where log files are written, + # can be absolute or relative to PGDATA +#log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' # log file name pattern, + # can include strftime() escapes +#log_file_mode = 0600 # creation mode for log files, + # begin with 0 to use octal notation +#log_truncate_on_rotation = off # If on, an existing log file with the + # same name as the new log file will be + # truncated rather than appended to. + # But such truncation only occurs on + # time-driven rotation, not on restarts + # or size-driven rotation. Default is + # off, meaning append to existing files + # in all cases. +#log_rotation_age = 1d # Automatic rotation of logfiles will + # happen after that time. 0 disables. +#log_rotation_size = 10MB # Automatic rotation of logfiles will + # happen after that much log output. + # 0 disables. + +# These are relevant when logging to syslog: +#syslog_facility = 'LOCAL0' +#syslog_ident = 'postgres' +#syslog_sequence_numbers = on +#syslog_split_messages = on + +# This is only relevant when logging to eventlog (win32): +# (change requires restart) +#event_source = 'PostgreSQL' + +# - When to Log - + +#log_min_messages = warning # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic + +#log_min_error_statement = error # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # info + # notice + # warning + # error + # log + # fatal + # panic (effectively off) + +#log_min_duration_statement = -1 # -1 is disabled, 0 logs all statements + # and their durations, > 0 logs only + # statements running at least this number + # of milliseconds + + +# - What to Log - + +#debug_print_parse = off +#debug_print_rewritten = off +#debug_print_plan = off +#debug_pretty_print = on +#log_checkpoints = off +#log_connections = off +#log_disconnections = off +#log_duration = off +#log_error_verbosity = default # terse, default, or verbose messages +#log_hostname = off +#log_line_prefix = '%m [%p] ' # special values: + # %a = application name + # %u = user name + # %d = database name + # %r = remote host and port + # %h = remote host + # %p = process ID + # %t = timestamp without milliseconds + # %m = timestamp with milliseconds + # %n = timestamp with milliseconds (as a Unix epoch) + # %i = command tag + # %e = SQL state + # %c = session ID + # %l = session line number + # %s = session start timestamp + # %v = virtual transaction ID + # %x = transaction ID (0 if none) + # %q = stop here in non-session + # processes + # %% = '%' + # e.g. '<%u%%%d> ' +#log_lock_waits = off # log lock waits >= deadlock_timeout +#log_statement = 'none' # none, ddl, mod, all +#log_replication_commands = off +#log_temp_files = -1 # log temporary files equal or larger + # than the specified size in kilobytes; + # -1 disables, 0 logs all temp files +#log_timezone = 'GMT' + +#------------------------------------------------------------------------------ +# PROCESS TITLE +#------------------------------------------------------------------------------ + +#cluster_name = '' # added to process titles if nonempty + # (change requires restart) +#update_process_title = on + + +#------------------------------------------------------------------------------ +# STATISTICS +#------------------------------------------------------------------------------ + +# - Query and Index Statistics Collector - + +#track_activities = on +#track_counts = on +#track_io_timing = off +#track_functions = none # none, pl, all +#track_activity_query_size = 1024 # (change requires restart) +#stats_temp_directory = 'pg_stat_tmp' + + +# - Monitoring - + +#log_parser_stats = off +#log_planner_stats = off +#log_executor_stats = off +#log_statement_stats = off + + +#------------------------------------------------------------------------------ +# AUTOVACUUM +#------------------------------------------------------------------------------ + +#autovacuum = on # Enable autovacuum subprocess? 'on' + # requires track_counts to also be on. +#log_autovacuum_min_duration = -1 # -1 disables, 0 logs all actions and + # their durations, > 0 logs only + # actions running at least this number + # of milliseconds. +#autovacuum_max_workers = 3 # max number of autovacuum subprocesses + # (change requires restart) +#autovacuum_naptime = 1min # time between autovacuum runs +#autovacuum_vacuum_threshold = 50 # min number of row updates before + # vacuum +#autovacuum_analyze_threshold = 50 # min number of row updates before + # analyze +#autovacuum_vacuum_scale_factor = 0.2 # fraction of table size before vacuum +#autovacuum_analyze_scale_factor = 0.1 # fraction of table size before analyze +#autovacuum_freeze_max_age = 200000000 # maximum XID age before forced vacuum + # (change requires restart) +#autovacuum_multixact_freeze_max_age = 400000000 # maximum multixact age + # before forced vacuum + # (change requires restart) +#autovacuum_vacuum_cost_delay = 20ms # default vacuum cost delay for + # autovacuum, in milliseconds; + # -1 means use vacuum_cost_delay +#autovacuum_vacuum_cost_limit = -1 # default vacuum cost limit for + # autovacuum, -1 means use + # vacuum_cost_limit + + +#------------------------------------------------------------------------------ +# CLIENT CONNECTION DEFAULTS +#------------------------------------------------------------------------------ + +# - Statement Behavior - + +#client_min_messages = notice # values in order of decreasing detail: + # debug5 + # debug4 + # debug3 + # debug2 + # debug1 + # log + # notice + # warning + # error +#search_path = '"$user", public' # schema names +#row_security = on +#default_tablespace = '' # a tablespace name, '' uses the default +#temp_tablespaces = '' # a list of tablespace names, '' uses + # only default tablespace +#check_function_bodies = on +#default_transaction_isolation = 'read committed' +#default_transaction_read_only = off +#default_transaction_deferrable = off +#session_replication_role = 'origin' +#statement_timeout = 0 # in milliseconds, 0 is disabled +#lock_timeout = 0 # in milliseconds, 0 is disabled +#idle_in_transaction_session_timeout = 0 # in milliseconds, 0 is disabled +#vacuum_freeze_min_age = 50000000 +#vacuum_freeze_table_age = 150000000 +#vacuum_multixact_freeze_min_age = 5000000 +#vacuum_multixact_freeze_table_age = 150000000 +#vacuum_cleanup_index_scale_factor = 0.1 # fraction of total number of tuples + # before index cleanup, 0 always performs + # index cleanup +#bytea_output = 'hex' # hex, escape +#xmlbinary = 'base64' +#xmloption = 'content' +#gin_fuzzy_search_limit = 0 +#gin_pending_list_limit = 4MB + +# - Locale and Formatting - + +#datestyle = 'iso, mdy' +#intervalstyle = 'postgres' +#timezone = 'GMT' +#timezone_abbreviations = 'Default' # Select the set of available time zone + # abbreviations. Currently, there are + # Default + # Australia (historical usage) + # India + # You can create your own file in + # share/timezonesets/. +#extra_float_digits = 0 # min -15, max 3 +#client_encoding = sql_ascii # actually, defaults to database + # encoding + +# These settings are initialized by initdb, but they can be changed. +#lc_messages = 'C' # locale for system error message + # strings +#lc_monetary = 'C' # locale for monetary formatting +#lc_numeric = 'C' # locale for number formatting +#lc_time = 'C' # locale for time formatting + +# default configuration for text search +#default_text_search_config = 'pg_catalog.simple' + +# - Shared Library Preloading - + +#shared_preload_libraries = '' # (change requires restart) +#local_preload_libraries = '' +#session_preload_libraries = '' +#jit_provider = 'llvmjit' # JIT library to use + +# - Other Defaults - + +#dynamic_library_path = '$libdir' + + +#------------------------------------------------------------------------------ +# LOCK MANAGEMENT +#------------------------------------------------------------------------------ + +#deadlock_timeout = 1s +#max_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_transaction = 64 # min 10 + # (change requires restart) +#max_pred_locks_per_relation = -2 # negative values mean + # (max_pred_locks_per_transaction + # / -max_pred_locks_per_relation) - 1 +#max_pred_locks_per_page = 2 # min 0 + + +#------------------------------------------------------------------------------ +# VERSION AND PLATFORM COMPATIBILITY +#------------------------------------------------------------------------------ + +# - Previous PostgreSQL Versions - + +#array_nulls = on +#backslash_quote = safe_encoding # on, off, or safe_encoding +#default_with_oids = off +#escape_string_warning = on +#lo_compat_privileges = off +#operator_precedence_warning = off +#quote_all_identifiers = off +#standard_conforming_strings = on +#synchronize_seqscans = on + +# - Other Platforms and Clients - + +#transform_null_equals = off + + +#------------------------------------------------------------------------------ +# ERROR HANDLING +#------------------------------------------------------------------------------ + +#exit_on_error = off # terminate session on any error? +#restart_after_crash = on # reinitialize after backend crash? +#data_sync_retry = off # retry or panic on failure to fsync + # data? + # (change requires restart) + + +#------------------------------------------------------------------------------ +# CONFIG FILE INCLUDES +#------------------------------------------------------------------------------ + +# These options allow settings to be loaded from files other than the +# default postgresql.conf. + +#include_dir = '' # include files ending in '.conf' from + # a directory, e.g., 'conf.d' +#include_if_exists = '' # include file only if it exists +#include = '' # include file + + +#------------------------------------------------------------------------------ +# CUSTOMIZED OPTIONS +#------------------------------------------------------------------------------ + +# Add settings for extensions here diff --git a/services/web/geeksbot.conf b/services/web/geeksbot.conf new file mode 100644 index 0000000..fce9f90 --- /dev/null +++ b/services/web/geeksbot.conf @@ -0,0 +1,48 @@ +upstream app_server { + server 127.0.0.1:8000 fail_timeout=0; +} + +server { + listen 443 ssl; + keepalive_timeout 5; + + ssl_certificate /etc/ssl/geeksbot_app_cert_chain.crt; + ssl_certificate_key /etc/ssl/geeksbot.app.key; + + access_log /tmp/logs/geeksbot/access.log; + error_log /tmp/logs/geeksbot/error.log; + + location /static/ { + alias /code/staticfiles/; + } + + location /error/ { + alias /code/staticfiles/errors/; + } + + location / { + try_files $uri @proxy_to_app; + } + + location @proxy_to_app { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Host $http_host; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_redirect off; + proxy_buffering off; + proxy_read_timeout 180; + proxy_connect_timeout 180; + + if (!-f $request_filename) { + proxy_pass http://app_server; + break; + } + } + + error_page 500 502 503 504 /error/maintenance.html; +} + +server { + listen 80 default_server; + return 301 https://$host$request_uri; +} \ No newline at end of file diff --git a/services/web/gunicorn.conf b/services/web/gunicorn.conf new file mode 100644 index 0000000..6636983 --- /dev/null +++ b/services/web/gunicorn.conf @@ -0,0 +1,10 @@ +import multiprocessing + +bind = "0.0.0.0:8000" +workers = multiprocessing.cpu_count() * 2 + 1 +worker_class = "gevent" +worker_connections = 4096 +timeout = 180 +backlog = 2048 +pidfile = "/tmp/geeksbot.pid" +reload = True \ No newline at end of file diff --git a/services/web/nginx.conf b/services/web/nginx.conf new file mode 100644 index 0000000..14446ff --- /dev/null +++ b/services/web/nginx.conf @@ -0,0 +1,44 @@ +user geeksbot; +master_process off; +# set open fd limit to 30000 +worker_rlimit_nofile 30000; +pid /var/run/nginx.pid; +daemon off; + +events { + worker_connections 4096; + accept_mutex off; +} + +http { + # Basic Settings + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + client_max_body_size 200M; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Logging + access_log /tmp/logs/nginx/access.log; + error_log /tmp/logs/nginx/error.log; + + # Gzip + + gzip on; + gzip_proxied any; + gzip_comp_level 2; + gzip_http_version 1.1; + gzip_buffers 16 8k; + gzip_types text/plain text/css application/json application/x-javascript text/xml applicaion/xml application/xml-rss text/javascript; + gzip_disable "msie6"; + gzip_vary on; + + # Virtual Host Configs + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; +} \ No newline at end of file diff --git a/services/web/supervisor_geeksbot.conf b/services/web/supervisor_geeksbot.conf new file mode 100644 index 0000000..d1a1d10 --- /dev/null +++ b/services/web/supervisor_geeksbot.conf @@ -0,0 +1,15 @@ +[program:geeksbot] +command=/usr/local/bin/gunicorn config.wsgi:application -c /etc/gunicorn.conf +directory=/code/ +stdout_logfile=/tmp/logs/geeksbot/gunicorn.log +autostart=true +autorestart=true +redirect_stderr=true +user=geeksbot + +[program:nginx] +command=/usr/sbin/nginx +stdout_logfile=/tmp/logs/nginx/access.log +stderr_logfile=/tmp/logs/nginx/error.log +autostart=true +autorestart=true diff --git a/services/web/supervisord.conf b/services/web/supervisord.conf new file mode 100644 index 0000000..14ce905 --- /dev/null +++ b/services/web/supervisord.conf @@ -0,0 +1,21 @@ +[unix_http_server] +file=/tmp/supervisor.sock + +[supervisord] +nodaemon=true +logfile=/tmp/logs/supervisord.log +logfile_maxbytes=50MB +logfile_backups=0 +loglevel=info +pidfile=/tmp/supervisord.pid +minfds=1024 ; min available startup file descriptors +minprocs=200 ; min available process descriptors + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///tmp/supervisor.sock + +[include] +files = /etc/supervisor/conf.d/*.conf \ No newline at end of file