download_file behaves differently if a local bot API server is used. Wrong default API url was being used

Added the method `unpinAllGeneralForumTopicMessages` (6.8).

Added the new administrator privileges `can_post_stories`, `can_edit_stories` and `can_delete_stories` to the class `ChatAdministratorRights` and to the method `promoteChatMember` (6.9).

davtelepot/__init__.py

@@ -11,12 +11,13 @@ __author__ = "Davide Testa"
 __email__ = "davide@davte.it"
 __credits__ = ["Marco Origlia", "Nick Lee @Nickoala"]
 __license__ = "GNU General Public License v3.0"
-__version__ = "2.6.2"
+__version__ = "2.9.11"
 __maintainer__ = "Davide Testa"
 __contact__ = "t.me/davte"
 
-from . import (administration_tools, authorization, bot, helper, languages,
-               suggestions, useful_tools, utilities)
+from davtelepot import (administration_tools, api, authorization,
+                        bot, helper, languages, messages, suggestions,
+                        useful_tools, utilities)
 
-__all__ = [administration_tools, authorization, bot, helper, languages,
-           suggestions, useful_tools, utilities]
+__all__ = ['administration_tools', 'api', 'authorization', 'bot', 'helper',
+           'languages', 'messages', 'suggestions', 'useful_tools', 'utilities']

davtelepot/__main__.py

@@ -0,0 +1,5 @@
+from davtelepot.cli import run_from_command_line
+
+
+if __name__ == '__main__':
+    run_from_command_line()

davtelepot/administration_tools.py

@@ -23,19 +23,24 @@ from typing import Union, List, Tuple
 from sqlalchemy.exc import ResourceClosedError
 
 # Project modules
-from . import messages
-from .bot import Bot
-from .utilities import (
+from davtelepot.messages import default_admin_messages, default_talk_messages
+from davtelepot.bot import Bot
+from davtelepot.utilities import (
     async_wrapper, CachedPage, Confirmator, extract, get_cleaned_text,
-    get_user, escape_html_chars, line_drawing_unordered_list, make_button,
+    get_user, clean_html_string, line_drawing_unordered_list, make_button,
     make_inline_keyboard, remove_html_tags, send_part_of_text_file,
-    send_csv_file, make_lines_of_buttons
+    send_csv_file, make_lines_of_buttons, join_path
 )
 
 # Use this parameter in SQL `LIMIT x OFFSET y` clauses
 rows_number_limit = 10
 
 command_description_parser = re.compile(r'(?P<command>\w+)(\s?-\s?(?P<description>.*))?')
+variable_regex = re.compile(r"(?P<name>[a-zA-Z]\w*)\s*=\s*"
+                            r"(?P<value>\d*[.,]?\d+|"
+                            r"True|False|"
+                            r"'[^']*'|"
+                            r"\"[^\"]*\")")
 
 
 async def _forward_to(update,
@@ -130,7 +135,7 @@ def get_talk_panel(bot: Bot,
                 'help_text',
                 update=update,
                 user_record=user_record,
-                q=escape_html_chars(
+                q=clean_html_string(
                     remove_html_tags(text)
                 )
             )
@@ -155,7 +160,7 @@ def get_talk_panel(bot: Bot,
                 'user_not_found',
                 update=update,
                 user_record=user_record,
-                q=escape_html_chars(
+                q=clean_html_string(
                     remove_html_tags(text)
                 )
             )
@@ -362,7 +367,10 @@ async def _talk_button(bot: Bot,
                 len(arguments) < 1
                 or type(arguments[0]) is not int
         ):
-            result = "Errore!"
+            result = bot.get_message(
+                'talk', 'error', 'text',
+                update=update, user_record=user_record
+            )
         else:
             with bot.db as db:
                 other_user_record = db['users'].find_one(
@@ -381,7 +389,10 @@ async def _talk_button(bot: Bot,
                 len(arguments) < 1
                 or type(arguments[0]) is not int
         ):
-            result = "Errore!"
+            result = bot.get_message(
+                'talk', 'error', 'text',
+                update=update, user_record=user_record
+            )
         elif not Confirmator.get('stop_bots').confirm(telegram_id):
             result = bot.get_message(
                 'talk', 'end_session',
@@ -524,7 +535,7 @@ async def _stop_button(bot: Bot,
     return result
 
 
-async def _send_bot_database(bot, update, user_record):
+async def _send_bot_database(bot: Bot, user_record: OrderedDict, language: str):
     if not all(
             [
                 bot.db_url.endswith('.db'),
@@ -533,20 +544,21 @@ async def _send_bot_database(bot, update, user_record):
     ):
         return bot.get_message(
             'admin', 'db_command', 'not_sqlite',
-            update=update, user_record=user_record,
+            language=language,
             db_type=bot.db_url.partition(':///')[0]
         )
-    await bot.send_document(
+    sent_update = await bot.send_document(
         chat_id=user_record['telegram_id'],
         document_path=extract(bot.db.url, starter='sqlite:///'),
         caption=bot.get_message(
             'admin', 'db_command', 'file_caption',
-            update=update, user_record=user_record
+            language=language
         )
     )
     return bot.get_message(
-        'admin', 'db_command', 'db_sent',
-        update=update, user_record=user_record
+        'admin', 'db_command',
+        ('error' if isinstance(sent_update, Exception) else 'db_sent'),
+        language=language
     )
 
 
@@ -1704,7 +1716,7 @@ async def _father_button(bot: Bot, user_record: OrderedDict,
                                 prefix='father:///',
                                 delimiter='|',
                                 data=['settings', 'edit', 'select',
-                                      selected_record['id'], 'edit_descr']
+                                      selected_record['id'], 'edit_description']
                             ),
                             make_button(
                                 text=bot.get_message(
@@ -1730,7 +1742,7 @@ async def _father_button(bot: Bot, user_record: OrderedDict,
                         ],
                         2
                     )
-                elif len(data) > 3 and data[3] == 'edit_descr':
+                elif len(data) > 3 and data[3] == 'edit_description':
                     result, text, reply_markup = await edit_bot_father_settings_via_message(
                         bot=bot,
                         user_record=user_record,
@@ -1798,6 +1810,40 @@ async def _father_button(bot: Bot, user_record: OrderedDict,
     return result
 
 
+async def _config_command(bot: Bot, update: dict,
+                          user_record: dict, language: str):
+    text = get_cleaned_text(
+        update,
+        bot,
+        ['config']
+    )
+    if not text:
+        return bot.get_message('admin', 'config_command',
+                               'instructions',
+                               user_record=user_record,
+                               language=language)
+    match = variable_regex.match(text)
+    if not match:
+        return bot.get_message('admin', 'config_command',
+                               'invalid_input',
+                               user_record=user_record,
+                               language=language)
+    match = match.groupdict()
+    if (',' in match['value']
+            and not match['value'].startswith('\'')
+            and not match['value'].startswith('"')):
+        match['value'] = match['value'].replace(',', '.')
+    new_variable = f"{match['name']} = {match['value']}"
+    with open(join_path(bot.path, 'data', 'config.py'),
+              'a') as configuration_file:
+        configuration_file.write(f"{new_variable}\n")
+    return bot.get_message('admin', 'config_command',
+                           'success',
+                           new_variable=new_variable,
+                           user_record=user_record,
+                           language=language)
+
+
 def init(telegram_bot: Bot,
          talk_messages: dict = None,
          admin_messages: dict = None,
@@ -1811,10 +1857,10 @@ def init(telegram_bot: Bot,
     )
     asyncio.ensure_future(get_package_updates(telegram_bot))
     if talk_messages is None:
-        talk_messages = messages.default_talk_messages
+        talk_messages = default_talk_messages
     telegram_bot.messages['talk'] = talk_messages
     if admin_messages is None:
-        admin_messages = messages.default_admin_messages
+        admin_messages = default_admin_messages
     telegram_bot.messages['admin'] = admin_messages
     db = telegram_bot.db
     if 'bot_father_commands' not in db.tables:
@@ -1823,11 +1869,11 @@ def init(telegram_bot: Bot,
         )
         table.create_column(
             'command',
-            db.types.string
+            db.types.string(100)
         )
         table.create_column(
             'description',
-            db.types.string
+            db.types.string(300)
         )
         table.create_column(
             'hidden',
@@ -1839,7 +1885,7 @@ def init(telegram_bot: Bot,
         )
     if 'talking_sessions' not in db.tables:
         table = db.create_table(
-            table_name='users'
+            table_name='talking_sessions'
         )
         table.create_column(
             'user',
@@ -1879,8 +1925,10 @@ def init(telegram_bot: Bot,
                           description=admin_messages[
                               'db_command']['description'],
                           authorization_level='admin')
-    async def send_bot_database(bot, update, user_record):
-        return await _send_bot_database(bot, update, user_record)
+    async def send_bot_database(bot, user_record, language):
+        return await _send_bot_database(bot=bot,
+                                        user_record=user_record,
+                                        language=language)
 
     @telegram_bot.command(command='/errors',
                           aliases=[],
@@ -2010,3 +2058,18 @@ def init(telegram_bot: Bot,
                                       update=update,
                                       user_record=user_record,
                                       language=language)
+
+    @telegram_bot.command(command='/config',
+                          aliases=[],
+                          **{key: admin_messages['config_command'][key]
+                             for key in ('reply_keyboard_button',
+                                         'description',
+                                         'help_section',)
+                             },
+                          show_in_keyboard=False,
+                          authorization_level='admin')
+    async def config_command(bot, update, user_record, language):
+        return await _config_command(bot=bot,
+                                     update=update,
+                                     user_record=user_record,
+                                     language=language)

davtelepot/api_helper.py

@@ -3,14 +3,18 @@
 # Standard library modules
 import argparse
 import asyncio
+import inspect
 import logging
 
 # Third party modules
+import os
+from typing import List
+
 import aiohttp
 from bs4 import BeautifulSoup
 
 # Project modules
-from . import api
+from davtelepot.api import TelegramBot
 
 api_url = "https://core.telegram.org/bots/api"
 
@@ -42,6 +46,29 @@ class TelegramApiMethod(object):
         """Return method description."""
         return self._description
 
+    @property
+    def description_80chars(self):
+        """Return method description, breaking lines at 80 characters."""
+        result, current_line = '', ''
+        indentation = 8
+        redundant_string = "Use this method to "
+        for n, paragraph in enumerate(self.description.replace('.', '.\n').split('\n')):
+            additional_indentation = 0
+            if n == 0 and paragraph.startswith(redundant_string):
+                paragraph = paragraph[len(redundant_string)].upper() + paragraph[len(redundant_string)+1:]
+            for word in paragraph.split(' '):
+                if len(current_line) + len(word) > 80 - indentation - additional_indentation:
+                    additional_indentation = max(additional_indentation, 4)
+                    result += f"{current_line.strip()}\n{' ' * additional_indentation}"
+                    current_line = ""
+                current_line += f"{word} "
+            if len(current_line):
+                result += f"{current_line.strip()}\n"
+                current_line = ""
+            if n == 0:
+                result += '\n'
+        return result.strip()
+
     @property
     def table(self):
         """Return method parameters table."""
@@ -52,7 +79,7 @@ class TelegramApiMethod(object):
         return self._parameters
 
     @property
-    def parameters_with_types(self):
+    def parameters_with_types(self) -> List[str]:
         return [
             f"{parameter['name']}: {parameter['type']}"
             for parameter in self._parameters
@@ -98,16 +125,14 @@ class TelegramApiMethod(object):
         return parameters
 
 
-async def print_api_methods(loop=None,
-                            filename=None,
+async def print_api_methods(filename=None,
                             print_all=False,
-                            output_file=None):
+                            output_file=None,
+                            input_file=None):
     """Get information from Telegram bot API web page."""
-    if loop is None:
-        loop = asyncio.get_event_loop()
-    implemented_methods = dir(api.TelegramBot)
+    implemented_methods = dir(TelegramBot)
+    if input_file is None or not os.path.isfile(input_file):
         async with aiohttp.ClientSession(
-        loop=loop,
             timeout=aiohttp.ClientTimeout(
                 total=100
             )
@@ -119,6 +144,12 @@ async def print_api_methods(loop=None,
                     await response.text(),
                     "html.parser"
                 )
+    else:
+        with open(input_file, 'r') as local_web_page:
+            web_page = BeautifulSoup(
+                ''.join(local_web_page.readlines()),
+                "html.parser"
+            )
     if filename is not None:
         with open(filename, 'w') as _file:
             _file.write(web_page.decode())
@@ -146,40 +177,84 @@ async def print_api_methods(loop=None,
                 )
             )
     new_line = '\n'
+    new_methods = []
+    edited_methods = []
+    for method in methods:
+        if print_all or method.name not in implemented_methods:
+            new_methods.append(method)
+        else:
+            parameters = set(parameter['name'] for parameter in method.parameters)
+            implemented_parameters = set(
+                parameter.strip('_')  # Parameter `type` becomes `type_` in python
+                for parameter in inspect.signature(
+                    getattr(TelegramBot,
+                            method.name)
+                ).parameters.keys()
+                if parameter != 'self'
+            )
+            new_parameters = parameters - implemented_parameters
+            deprecated_parameters = implemented_parameters - parameters - {'kwargs'}
+            if new_parameters or deprecated_parameters:
+                edited_methods.append(
+                    dict(
+                        name=method.name,
+                        new_parameters=new_parameters,
+                        deprecated_parameters=deprecated_parameters
+                    )
+                )
     if output_file:
         with open(output_file, 'w') as file:
+            if new_methods:
                 file.write(
                     "from typing import List, Union\n"
-                "from davtelepot.api import TelegramBot\n"
-                "self = TelegramBot('fake_token')\n\n\n"
+                    "from davtelepot.api import TelegramBot\n\n\n"
+                    "# noinspection PyPep8Naming\n"
+                    "class Bot(TelegramBot):\n\n"
                 )
             file.writelines(
-                f"async def {method.name}("
-                f"{', '.join(method.parameters_with_types)}"
-                "):\n"
-                "    \"\"\""
-                f"{method.description.replace(new_line, new_line + ' ' * 4)}\n"
-                "    See https://core.telegram.org/bots/api#"
+                f"    async def {method.name}("
+                f"{', '.join(['self'] + method.parameters_with_types)}"
+                f"):\n"
+                f"        \"\"\""
+                f"{method.description_80chars.replace(new_line, new_line + ' ' * 8)}\n"
+                f"        See https://core.telegram.org/bots/api#"
                 f"{method.name.lower()} for details.\n"
-                "    \"\"\"\n"
-                "    return await self.api_request(\n"
+                f"        \"\"\"\n"
+                f"        return await self.api_request(\n"
                 f"            '{method.name}',\n"
-                "        parameters=locals()\n"
-                "    )\n\n\n"
-                for method in methods
-                if print_all or method.name not in implemented_methods
+                f"            parameters=locals()\n"
+                f"        )\n\n"
+                for method in new_methods
             )
+            if edited_methods:
+                file.write('\n# === EDITED METHODS ===\n')
+            for method in edited_methods:
+                file.write(f'\n"""{method["name"]}\n')
+                if method['new_parameters']:
+                    file.write("    New parameters: "
+                               + ", ".join(method['new_parameters'])
+                               + "\n")
+                if method['deprecated_parameters']:
+                    file.write("    Deprecated parameters: "
+                               + ", ".join(method['deprecated_parameters'])
+                               + "\n")
+                file.write('"""\n')
     else:
         print(
             '\n'.join(
                 f"NAME\n\t{method.name}\n"
-                f"PARAMETERS\n\t{', '.join(method.parameters_with_types)}\n"
+                f"PARAMETERS\n\t{', '.join(['self'] + method.parameters_with_types)}\n"
                 f"DESCRIPTION\n\t{method.description}\n"
                 f"TABLE\n\t{method.print_parameters_table()}\n\n"
-                for method in methods
-                if print_all or method.name not in implemented_methods
+                for method in new_methods
             )
         )
+        for method in edited_methods:
+            print(method['name'])
+            if method['new_parameters']:
+                print("\tNew parameters: " + ", ".join(method['new_parameters']))
+            if method['deprecated_parameters']:
+                print("\tDeprecated parameters: " + ", ".join(method['deprecated_parameters']))
 
 
 def main():
@@ -202,16 +277,20 @@ def main():
                             default=None,
                             required=False,
                             help='File path to store methods implementation')
+    cli_parser.add_argument('--in', '--input', '-i', type=str,
+                            default=None,
+                            required=False,
+                            help='File path to read Telegram API web page')
     cli_arguments = vars(cli_parser.parse_args())
     filename = cli_arguments['file']
     print_all = cli_arguments['all']
     output_file = cli_arguments['out']
-    loop = asyncio.get_event_loop()
-    loop.run_until_complete(
-        print_api_methods(loop=loop,
-                          filename=filename,
+    input_file = cli_arguments['in']
+    asyncio.run(
+        print_api_methods(filename=filename,
                           print_all=print_all,
-                          output_file=output_file)
+                          output_file=output_file,
+                          input_file=input_file)
     )
     logging.info("Done!")
 

davtelepot/authorization.py

@@ -5,9 +5,9 @@ from collections import OrderedDict
 from typing import Callable, List, Union
 
 # Project modules
-from .bot import Bot
-from .messages import default_authorization_messages
-from .utilities import (
+from davtelepot.bot import Bot
+from davtelepot.messages import default_authorization_messages
+from davtelepot.utilities import (
     Confirmator, get_cleaned_text, get_user, make_button, make_inline_keyboard
 )
 
@@ -61,6 +61,8 @@ DEFAULT_ROLES[100] = {
     'can_be_appointed_by': [1, 2, 3]
 }
 
+max_records_per_query = 15
+
 
 class Role:
     """Authorization level for users of a bot."""
@@ -74,7 +76,7 @@ class Role:
         """Instantiate Role object.
 
         code : int
-            The higher the code, the less privileges are connected to that
+            The higher the code, the fewer privileges are connected to that
                 role. Use 0 for banned users.
         name : str
             Short name for role.
@@ -198,8 +200,16 @@ class Role:
         """
         if user_role is None:
             user_role = cls.get_user_role(user_record=user_record)
+        long_name = ' '.join(
+            [user_record[key]
+             for key in ('first_name', 'last_name')
+             if key in user_record
+             and user_record[key]]
+        )
+        if long_name:
+            long_name = f" ({long_name})"
         return (
-            f"""👤 {get_user(record=user_record)}\n"""
+            f"👤 {get_user(record=user_record)}{long_name}\n"
             f"🔑 <i>{user_role.singular.capitalize()}</i> {user_role.symbol}"
         )
 
@@ -228,6 +238,7 @@ class Role:
             make_button(
                 f"{role.symbol} {role.singular.capitalize()}",
                 prefix='auth:///',
+                delimiter='|',
                 data=['set', user_record['id'], code]
             )
             for code, role in cls.roles.items()
@@ -252,7 +263,7 @@ class Role:
         return text, buttons
 
     def __eq__(self, other: 'Role'):
-        """Return True if self is equal to other."""
+        """Return True if `self` is equal to `other`."""
         return self.code == other.code
 
     def __gt__(self, other: 'Role'):
@@ -274,11 +285,11 @@ class Role:
         return not self.__ge__(other)
 
     def __le__(self, other: 'Role'):
-        """Return True if self is superior or equal to other."""
+        """Return True if `self` is superior or equal to `other`."""
         return not self.__gt__(other)
 
     def __ne__(self, other: 'Role'):
-        """Return True if self is not equal to other."""
+        """Return True if `self` is not equal to `other`."""
         return not self.__eq__(other)
 
     def __str__(self):
@@ -312,14 +323,37 @@ def get_authorization_function(bot: Bot):
     return is_authorized
 
 
+def get_browse_buttons(bot: Bot, language: str):
+    return [
+        make_button(
+            text=bot.get_message('authorization', 'auth_button',
+                                 'browse', 'browse_button_az',
+                                 language=language),
+            prefix='auth:///',
+            delimiter='|',
+            data=['browse', 'az'],
+        ),
+        make_button(
+            text=bot.get_message('authorization', 'auth_button',
+                                 'browse', 'browse_button_by_role',
+                                 language=language),
+            prefix='auth:///',
+            delimiter='|',
+            data=['browse', 'role'],
+        ),
+    ]
+
+
 async def _authorization_command(bot: Bot,
                                  update: dict,
                                  user_record: OrderedDict,
+                                 language: str,
                                  mode: str = 'auth'):
     db = bot.db
     text = get_cleaned_text(bot=bot, update=update, replace=[mode])
     reply_markup = None
     admin_record = user_record.copy()
+    query_id = None
     user_record = None
     admin_role = bot.Role.get_user_role(user_record=admin_record)
     result = bot.get_message(
@@ -333,6 +367,9 @@ async def _authorization_command(bot: Bot,
                 update=update, user_record=admin_record,
                 command=mode
             )
+            buttons = get_browse_buttons(bot=bot, language=language)
+            reply_markup = make_inline_keyboard(buttons, 2)
+            user_record = -1
         else:  # No text, command used in reply to another message
             update = update['reply_to_message']
             # Forwarded message: get both the user who forwarded and the original author
@@ -349,6 +386,36 @@ async def _authorization_command(bot: Bot,
                     telegram_id=update['from']['id']
                 )
     else:  # Get users matching the input text
+        query_text = (
+            "SELECT * "
+            "FROM users "
+            "WHERE COALESCE("
+            "   first_name || last_name || username,"
+            "   last_name || username,"
+            "   first_name || username,"
+            "   username,"
+            "   first_name || last_name,"
+            "   last_name,"
+            "   first_name"
+            f") LIKE '%{text}%' "
+            "ORDER BY COALESCE("
+            "   username || first_name || last_name,"
+            "   username || last_name,"
+            "   username || first_name,"
+            "   username,"
+            "   first_name || last_name,"
+            "   first_name,"
+            "   last_name"
+            ")"
+        )
+        query_id = bot.db['queries'].upsert(
+            dict(query=query_text),
+            ['query']
+        )
+        if query_id is True:
+            query_id = bot.db['queries'].find_one(
+                query=query_text
+            )['id']
         user_record = list(
             db.query(
                 "SELECT * "
@@ -372,28 +439,21 @@ async def _authorization_command(bot: Bot,
             update=update, user_record=admin_record
         )
     elif type(user_record) is list and len(user_record) > 1:  # If many users match
-        result = bot.get_message(
-            'authorization', 'auth_command', 'choose_user',
-            update=update, user_record=admin_record,
-            n=len(user_record)
-        )
-        reply_markup = make_inline_keyboard(
-            [
-                make_button(
-                    f"👤 {get_user(user, link_profile=False)}",
-                    prefix='auth:///',
-                    data=['show', user['id']]
-                )
-                for user in user_record[:30]
-            ],
-            3
-        )
+        browse_panel = await _authorization_button(bot=bot,
+                                                   update=update,
+                                                   user_record=admin_record,
+                                                   language=language,
+                                                   data=['browse', query_id])
+        if 'edit' in browse_panel:
+            return browse_panel['edit']
     elif type(user_record) is list and len(user_record) == 0:  # If query was provided but no user matches
         result = bot.get_message(
             'authorization', 'auth_command', 'no_match',
             update=update, user_record=admin_record,
         )
-    elif isinstance(user_record, dict):  # If 1 user matches
+    elif (type(user_record) is list and len(user_record) == 1) or isinstance(user_record, dict):  # If 1 user matches
+        if type(user_record) is list:
+            user_record = user_record[0]
         # Ban user if admin can do it
         user_role = bot.Role.get_user_role(user_record=user_record)
         if mode == 'ban' and admin_role > user_role:
@@ -407,7 +467,19 @@ async def _authorization_command(bot: Bot,
             user_record=user_record,
             admin_record=admin_record
         )
+        if bot.db['user_profile_photos'].find_one(user_id=user_record['id']):
+            buttons.append(
+                make_button(
+                    text=bot.get_message('authorization', 'auth_button',
+                                         'profile_picture_button',
+                                         language=language),
+                    prefix='auth:///',
+                    delimiter='|',
+                    data=['picture', user_record['id']]
+                )
+            )
         reply_markup = make_inline_keyboard(buttons, 1)
+        reply_markup['inline_keyboard'].append(get_browse_buttons(bot=bot, language=language))
     return dict(
         text=result,
         reply_markup=reply_markup,
@@ -418,23 +490,145 @@ async def _authorization_command(bot: Bot,
 async def _authorization_button(bot: Bot,
                                 update: dict,
                                 user_record: OrderedDict,
+                                language: str,
                                 data: Union[str, List[Union[int, str]]]):
     if len(data) == 0:
         data = ['']
     command, *arguments = data
     user_id = user_record['telegram_id']
-    if len(arguments) > 0:
+    if len(arguments) > 0 and command in ['show', 'set']:
         other_user_id = arguments[0]
     else:
         other_user_id = None
     result, text, reply_markup = '', '', None
-    db = bot.db
-    if command in ['show']:
-        other_user_record = db['users'].find_one(id=other_user_id)
+    query_text = None
+    if command in ['browse'] and len(arguments) >= 1:
+        mode = arguments[0]
+        offset = arguments[1] if len(arguments) > 1 else 0
+        user_records = []
+        if mode == 'choose':
+            update['text'] = ''
+            answer_update = await _authorization_command(bot=bot,
+                                                         update=update,
+                                                         user_record=user_record,
+                                                         language=language,
+                                                         mode='auth')
+            text = answer_update['text']
+            reply_markup = answer_update['reply_markup']
+        elif isinstance(mode, int) or (isinstance(mode, str) and mode.isnumeric()):
+            query_record = bot.db['queries'].find_one(id=int(mode))
+            if query_record:
+                query_text = query_record['query']
+        elif mode == 'az':
+            query_text = (
+                "SELECT * "
+                "FROM users "
+                "ORDER BY COALESCE("
+                "   username || first_name || last_name,"
+                "   username || last_name,"
+                "   username || first_name,"
+                "   username,"
+                "   first_name || last_name,"
+                "   first_name,"
+                "   last_name"
+                ")"
+            )
+        elif mode == 'role':
+            query_text = (
+                "SELECT * "
+                "FROM users "
+                "ORDER BY ("
+                "   CASE WHEN privileges = 0 THEN 5000 "
+                "   ELSE privileges END) "
+                "ASC, COALESCE( "
+                "   username || first_name || last_name,"
+                "   username || last_name,"
+                "   username || first_name,"
+                "   username,"
+                "   first_name || last_name,"
+                "   first_name,"
+                "   last_name"
+                ") "
+            )
+        n = 0
+        if query_text:
+            user_records = list(
+                bot.db.query(
+                    f"{query_text} "
+                    f"LIMIT {max_records_per_query + 1} "
+                    f"OFFSET {offset * max_records_per_query} "
+                )
+            )
+            for record in bot.db.query("SELECT COUNT(*) n "
+                                       f"FROM ({query_text})"):
+                n = record['n']
+        if user_records:
+            text = bot.get_message(
+                'authorization', 'auth_command', 'choose_user',
+                update=update, user_record=user_record,
+                n=n
+            )
+            reply_markup = make_inline_keyboard(
+                [
+                    make_button(
+                        f"{bot.Role.get_user_role(user_record=user).symbol} {get_user(user, link_profile=False)}",
+                        prefix='auth:///',
+                        delimiter='|',
+                        data=['show', user['id'], command, mode, offset]
+                    )
+                    for user in user_records[:max_records_per_query]
+                ],
+                3
+            )
+            if n > max_records_per_query:
+                reply_markup['inline_keyboard'].append(
+                    [
+                        make_button(
+                            text='◀️',
+                            prefix='auth:///',
+                            delimiter='|',
+                            data=['browse', mode, offset - 1 if offset else n // max_records_per_query]
+                        ),
+                        make_button(
+                            text='↩️',
+                            prefix='auth:///',
+                            delimiter='|',
+                            data=['browse', 'choose']
+                        ),
+                        make_button(
+                            text='▶️',
+                            prefix='auth:///',
+                            delimiter='|',
+                            data=['browse', mode, offset + 1 if n > (offset + 1) * max_records_per_query else 0]
+                        )
+                    ]
+                )
+    elif command in ['show']:
+        other_user_record = bot.db['users'].find_one(id=other_user_id)
         text, buttons = bot.Role.get_user_role_text_and_buttons(
             user_record=other_user_record,
             admin_record=user_record
         )
+        if bot.db['user_profile_photos'].find_one(user_id=other_user_record['id']):
+            buttons.append(
+                make_button(
+                    text=bot.get_message('authorization', 'auth_button',
+                                         'profile_picture_button',
+                                         language=language),
+                    prefix='auth:///',
+                    delimiter='|',
+                    data=['picture', other_user_record['id']]
+                )
+            )
+        if len(arguments) > 2:
+            buttons.append(
+                make_button(
+                    text='↩️',
+                    prefix='auth:///',
+                    delimiter='|',
+                    data=data[2:]
+                )
+            )
         reply_markup = make_inline_keyboard(buttons, 1)
     elif command in ['set'] and len(arguments) > 1:
         other_user_id, new_privileges, *_ = arguments
@@ -446,7 +640,7 @@ async def _authorization_button(bot: Bot,
                 'authorization', 'auth_button', 'confirm',
                 update=update, user_record=user_record,
             )
-        other_user_record = db['users'].find_one(id=other_user_id)
+        other_user_record = bot.db['users'].find_one(id=other_user_id)
         user_role = bot.Role.get_user_role(user_record=user_record)
         other_user_role = bot.Role.get_user_role(user_record=other_user_record)
         if other_user_role.code == new_privileges:
@@ -467,6 +661,7 @@ async def _authorization_button(bot: Bot,
                             update=update, user_record=user_record
                         ),
                         prefix='auth:///',
+                        delimiter='|',
                         data=['show', other_user_id]
                     )
                 ],
@@ -485,20 +680,21 @@ async def _authorization_button(bot: Bot,
                             update=update, user_record=user_record
                         ),
                         prefix='auth:///',
+                        delimiter='|',
                         data=['show', other_user_id]
                     )
                 ],
                 1
             )
         else:
-            db['users'].update(
+            bot.db['users'].update(
                 dict(
                     id=other_user_id,
                     privileges=new_privileges
                 ),
                 ['id']
             )
-            other_user_record = db['users'].find_one(id=other_user_id)
+            other_user_record = bot.db['users'].find_one(id=other_user_id)
             result = bot.get_message(
                 'authorization', 'auth_button', 'appointed',
                 update=update, user_record=user_record
@@ -507,7 +703,41 @@ async def _authorization_button(bot: Bot,
                 user_record=other_user_record,
                 admin_record=user_record
             )
+            if bot.db['user_profile_photos'].find_one(user_id=other_user_record['id']):
+                buttons.append(
+                    make_button(
+                        text=bot.get_message('authorization', 'auth_button',
+                                             'profile_picture_button',
+                                             language=language),
+                        prefix='auth:///',
+                        delimiter='|',
+                        data=['picture', other_user_record['id']]
+                    )
+                )
             reply_markup = make_inline_keyboard(buttons, 1)
+    elif command in ['picture'] and len(arguments) > 0:
+        photo_record = bot.db['user_profile_photos'].find_one(
+            user_id=arguments[0],
+            order_by=['-update_datetime'],
+        )
+        other_user_record = bot.db['users'].find_one(id=arguments[0])
+        if photo_record is None:
+            result = bot.get_message('admin', 'error', 'text',
+                                     language=language)
+        else:
+            caption, buttons = bot.Role.get_user_role_text_and_buttons(
+                user_record=other_user_record,
+                admin_record=user_record
+            )
+            await bot.sendPhoto(
+                chat_id=user_record['telegram_id'],
+                photo=photo_record['telegram_file_id'],
+                caption=caption,
+                reply_markup=make_inline_keyboard(
+                    buttons=buttons
+                ),
+                parse_mode='HTML'
+            )
     if text:
         return dict(
             text=result,
@@ -570,18 +800,24 @@ def init(telegram_bot: Bot,
                                   authorization_messages['auth_command']['description']
                           ),
                           authorization_level='moderator')
-    async def authorization_command(bot, update, user_record):
-        return await _authorization_command(bot, update, user_record)
+    async def authorization_command(bot, update, user_record, language):
+        return await _authorization_command(bot=bot, update=update,
+                                            user_record=user_record,
+                                            language=language)
 
     @telegram_bot.button('auth:///',
                          description=authorization_messages['auth_button']['description'],
                          separator='|',
                          authorization_level='moderator')
-    async def authorization_button(bot, update, user_record, data):
-        return await _authorization_button(bot, update, user_record, data)
+    async def authorization_button(bot, update, user_record, language, data):
+        return await _authorization_button(bot=bot, update=update,
+                                           user_record=user_record,
+                                           language=language, data=data)
 
     @telegram_bot.command('/ban', aliases=[], show_in_keyboard=False,
                           description=authorization_messages['ban_command']['description'],
                           authorization_level='moderator')
-    async def ban_command(bot, update, user_record):
-        return await _authorization_command(bot, update, user_record, mode='ban')
+    async def ban_command(bot, update, user_record, language):
+        return await _authorization_command(bot=bot, update=update,
+                                            user_record=user_record,
+                                            language=language, mode='ban')

davtelepot/cli.py

@@ -0,0 +1,199 @@
+import argparse
+import asyncio
+import logging
+import os.path
+import sys
+from typing import Any, Dict, Union
+
+import davtelepot.authorization as authorization
+import davtelepot.administration_tools as administration_tools
+import davtelepot.helper as helper
+from davtelepot.bot import Bot
+from davtelepot.utilities import (get_cleaned_text, get_secure_key,
+                                  get_user, join_path, json_read, json_write,
+                                  line_drawing_unordered_list)
+
+
+def dir_path(path):
+    if os.path.isdir(path) and os.access(path, os.W_OK):
+        return path
+    else:
+        raise argparse.ArgumentTypeError(f"`{path}` is not a valid path")
+
+
+def get_cli_arguments() -> Dict[str, Any]:
+    default_path = join_path(os.path.dirname(__file__), 'data')
+    cli_parser = argparse.ArgumentParser(
+        description='Run a davtelepot-powered Telegram bot from command line.',
+        allow_abbrev=False,
+    )
+    cli_parser.add_argument('-a', '--action', type=str,
+                            default='run',
+                            required=False,
+                            help='Action to perform (currently supported: run).')
+    cli_parser.add_argument('-p', '--path', type=dir_path,
+                            default=default_path,
+                            required=False,
+                            help='Folder to store secrets, data and log files.')
+    cli_parser.add_argument('-l', '--log_file', type=argparse.FileType('a'),
+                            default=None,
+                            required=False,
+                            help='File path to store full log')
+    cli_parser.add_argument('-e', '--error_log_file', type=argparse.FileType('a'),
+                            default=None,
+                            required=False,
+                            help='File path to store only error log')
+    cli_parser.add_argument('-t', '--token', type=str,
+                            required=False,
+                            help='Telegram bot token (you may get one from t.me/botfather)')
+    cli_parsed_arguments = vars(cli_parser.parse_args())
+    for key in cli_parsed_arguments:
+        if key.endswith('_file') and cli_parsed_arguments[key]:
+            cli_parsed_arguments[key] = cli_parsed_arguments[key].name
+    for key, default in {'error_log_file': "davtelepot.errors",
+                         'log_file': "davtelepot.log"}.items():
+        if cli_parsed_arguments[key] is None:
+            cli_parsed_arguments[key] = join_path(cli_parsed_arguments['path'], default)
+    return cli_parsed_arguments
+
+
+def set_loggers(log_file: str = 'davtelepot.log',
+                error_log_file: str = 'davtelepot.errors'):
+    root_logger = logging.getLogger()
+    root_logger.setLevel(logging.DEBUG)
+    log_formatter = logging.Formatter(
+        "%(asctime)s [%(module)-10s %(levelname)-8s]     %(message)s",
+        style='%'
+    )
+
+    file_handler = logging.FileHandler(log_file, mode="a", encoding="utf-8")
+    file_handler.setFormatter(log_formatter)
+    file_handler.setLevel(logging.DEBUG)
+    root_logger.addHandler(file_handler)
+
+    file_handler = logging.FileHandler(error_log_file, mode="a", encoding="utf-8")
+    file_handler.setFormatter(log_formatter)
+    file_handler.setLevel(logging.ERROR)
+    root_logger.addHandler(file_handler)
+
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(log_formatter)
+    console_handler.setLevel(logging.DEBUG)
+    root_logger.addHandler(console_handler)
+
+
+async def elevate_to_admin(bot: Bot, update: dict, user_record: dict,
+                           secret: str) -> Union[str, None]:
+    text = get_cleaned_text(update=update, bot=bot,
+                            replace=['00elevate_', 'elevate '])
+    if text == secret:
+        bot.db['users'].upsert(dict(id=user_record['id'], privileges=1), ['id'])
+        return "👑 You have been granted full powers! 👑"
+    else:
+        print(f"The secret entered (`{text}`) is wrong. Enter `{secret}` instead.")
+
+
+def allow_elevation_to_admin(telegram_bot: Bot) -> None:
+    secret = get_secure_key(length=15)
+
+    @telegram_bot.additional_task('BEFORE')
+    async def print_secret():
+        await telegram_bot.get_me()
+        logging.info(f"To get administration privileges, enter code {secret} "
+                     f"or click here: https://t.me/{telegram_bot.name}?start=00elevate_{secret}")
+
+    @telegram_bot.command(command='/elevate', aliases=['00elevate_'], show_in_keyboard=False,
+                          authorization_level='anybody')
+    async def _elevate_to_admin(bot, update, user_record):
+        return await elevate_to_admin(bot=bot, update=update,
+                                      user_record=user_record,
+                                      secret=secret)
+    return
+
+
+def send_single_message(telegram_bot: Bot):
+    records = []
+    text, last_text = '', ''
+    offset = 0
+    max_shown = 3
+    while True:
+        if text == '+' and len(records) > max_shown:
+            offset += 1
+        elif offset > 0 and text == '-':
+            offset -= 1
+        else:
+            offset = 0
+        if text in ('+', '-'):
+            text = last_text
+        condition = (f"WHERE username LIKE '%{text}%' "
+                     f"OR first_name LIKE '%{text}%' "
+                     f"OR last_name LIKE '%{text}%' ")
+        records = list(telegram_bot.db.query("SELECT username, first_name, "
+                                             "last_name, telegram_id "
+                                             "FROM users "
+                                             f"{condition} "
+                                             f"LIMIT {max_shown+1} "
+                                             f"OFFSET {offset*max_shown} "))
+        if len(records) == 1 and offset == 0:
+            break
+        last_text = text
+        print("=== Users ===",
+              line_drawing_unordered_list(
+                  list(map(lambda x: get_user(x, False),
+                           records[:max_shown]))
+                  + (['...'] if len(records) >= max_shown else [])
+              ),
+              sep='\n')
+        text = input("Select a recipient: write part of their name.\t\t")
+    while True:
+        text = input(f"Write a message for {get_user(records[0], False)}\t\t")
+        if input("Should I send it? Y to send, anything else cancel\t\t").lower() == "y":
+            break
+
+    async def send_and_print_message():
+        sent_message = await telegram_bot.send_one_message(chat_id=records[0]['telegram_id'], text=text)
+        print(sent_message)
+
+    asyncio.run(send_and_print_message())
+    return
+
+
+def run_from_command_line():
+    arguments = get_cli_arguments()
+    stored_arguments_file = os.path.join(arguments['path'],
+                                         'cli_args.json')
+    for key, value in json_read(file_=stored_arguments_file,
+                                default={}).items():
+        if key not in arguments or not arguments[key]:
+            arguments[key] = value
+    set_loggers(**{k: v
+                   for k, v in arguments.items()
+                   if k in ('log_file', 'error_log_file')})
+    if 'error_log_file' in arguments:
+        Bot.set_class_errors_file_path(file_path=arguments['error_log_file'])
+    if 'log_file' in arguments:
+        Bot.set_class_log_file_path(file_path=arguments['log_file'])
+    if 'path' in arguments:
+        Bot.set_class_path(arguments['path'])
+    if 'token' in arguments and arguments['token']:
+        token = arguments['token']
+    else:
+        token = input("Enter bot Token:\t\t")
+        arguments['token'] = token
+    json_write(arguments, stored_arguments_file)
+    bot = Bot(token=token, database_url=join_path(arguments['path'], 'bot.db'))
+    action = arguments['action'] if 'action' in arguments else 'run'
+    if action == 'run':
+        administration_tools.init(telegram_bot=bot)
+        authorization.init(telegram_bot=bot)
+        allow_elevation_to_admin(telegram_bot=bot)
+        helper.init(telegram_bot=bot)
+        exit_state = Bot.run(**{k: v
+                                for k, v in arguments.items()
+                                if k in ('local_host', 'port')})
+        sys.exit(exit_state)
+    if action == 'send':
+        try:
+            send_single_message(telegram_bot=bot)
+        except KeyboardInterrupt:
+            print("\nExiting...")

davtelepot/database.py

@@ -2,6 +2,7 @@
 
 # Standard library modules
 import logging
+from typing import Tuple
 
 # Third party modules
 import dataset
@@ -26,7 +27,7 @@ class ObjectWithDatabase(object):
         """Instantiate object and open connection with database."""
         if database_url is None:
             database_url = 'database.db'
-        if ':///' not in database_url:
+        if '://' not in database_url:
             # Default database engine is sqlite, which operates on a
             # single-file database having `.db` extension
             if not database_url.endswith('.db'):
@@ -71,3 +72,32 @@ class ObjectWithDatabase(object):
                     )
                 except Exception as e:
                     logging.error(f"{e}")
+
+    def add_table_and_columns_if_not_existent(self,
+                                              table_name: str,
+                                              columns: Tuple[
+                                                  Tuple[str,
+                                                        dataset.database.Types],
+                                                  ...] = None):
+        """Create table (if it does not exist) and add given columns (if missing).
+
+        @param table_name: Table name (string)
+        @param columns: Table columns as tuples of column name and type
+        @return: None
+        """
+        if table_name not in self.db.tables:
+            table = self.db.create_table(table_name=table_name)
+            logging.info(f"Created table `{table_name}`")
+        else:
+            table = self.db[table_name]
+        if columns is None:
+            columns = []
+        for column_name, column_type in columns:
+            if not table.has_column(column_name):
+                table.create_column(
+                    column_name,
+                    column_type
+                )
+                logging.info(f"Added column `{column_name}` "
+                             f"(type `{column_type}`) "
+                             f"to table `{table_name}`")

davtelepot/helper.py

@@ -1,11 +1,14 @@
 """Make a self-consistent bot help section."""
 
 # Project modules
-from .bot import Bot
-from .messages import default_help_messages
-from .utilities import (
+from collections import OrderedDict
+
+from davtelepot.bot import Bot
+from davtelepot.messages import default_help_messages
+from davtelepot.utilities import (
     get_cleaned_text, make_inline_keyboard,
-    make_lines_of_buttons, make_button
+    make_lines_of_buttons, make_button,
+    recursive_dictionary_update
 )
 
 
@@ -233,27 +236,35 @@ async def _help_button(bot, update, user_record, data):
     return result
 
 
-async def _start_command(bot, update, user_record):
+async def _start_command(bot: Bot, update: dict,
+                         user_record: OrderedDict, language: str):
     text = get_cleaned_text(update=update, bot=bot, replace=['start'])
     if not text:
         return await _help_command(bot, update, user_record)
     update['text'] = text
     await bot.text_message_handler(
         update=update,
-        user_record=None
+        user_record=user_record,
+        language=language
     )
     return
 
 
-def init(telegram_bot, help_messages=None):
+def init(telegram_bot: Bot, help_messages: dict = None):
     """Assign parsers, commands, buttons and queries to given `bot`."""
     if help_messages is None:
         help_messages = default_help_messages
+    else:
+        help_messages = recursive_dictionary_update(
+            default_help_messages.copy(),
+            help_messages.copy()
+        )
     telegram_bot.messages['help'] = help_messages
 
     @telegram_bot.command("/start", authorization_level='everybody')
-    async def start_command(bot, update, user_record):
-        return await _start_command(bot, update, user_record)
+    async def start_command(bot, update, user_record, language):
+        return await _start_command(bot=bot, update=update,
+                                    user_record=user_record, language=language)
 
     @telegram_bot.command(command='/help', aliases=['00help'],
                           reply_keyboard_button=help_messages['help_command'][

davtelepot/languages.py

@@ -6,8 +6,8 @@ from collections import OrderedDict
 import logging
 
 # Project modules
-from .messages import default_language_messages
-from .utilities import extract, make_button, make_inline_keyboard
+from davtelepot.messages import default_language_messages
+from davtelepot.utilities import extract, make_button, make_inline_keyboard
 
 
 class MultiLanguageObject(object):

davtelepot/messages.py

@@ -14,7 +14,11 @@ davtelepot_messages = {
                   "Questo messaggio è troppo lungo per essere inviato come "
                   "messaggi separati.",
         }
-    }
+    },
+    'part': {
+        'en': "part",
+        'it': "parte",
+    },
 }
 
 default_admin_messages = {
@@ -32,14 +36,73 @@ default_admin_messages = {
             'it': "annulla",
         },
     },
+    'config_command': {
+        'description': {
+            'en': "Add a variable to the configuration file",
+            'it': "Aggiungi una variabile al file di configurazione",
+        },
+        'help_section': None,
+        'instructions': {
+            'en': "<b>Config 🔧</b>\n\n"
+                  "<i>Send me a new configuration variable, for example:</i>\n"
+                  "<code>variable = 2</code>",
+            'it': "<b>Configurazione 🔧</b>\n\n"
+                  "<i>Mandami una variabile da aggiungere al file di "
+                  "configurazione, per esempio:</i>\n"
+                  "<code>variabile = 2</code>",
+        },
+        'invalid_input': {
+            'en': "<b>❌ Invalid input 🔧</b>\n\n"
+                  "<i>Send me a new configuration variable, for example:</i>\n"
+                  "<code>int_var = 2\n"
+                  "str_var = 'test'\n"
+                  "bool_var = False\n"
+                  "float_var = 3.5</code>",
+            'it': "<b>❌ Configurazione scorretta 🔧</b>\n\n"
+                  "<i>Mandami una variabile da aggiungere al file di "
+                  "configurazione, per esempio:</i>\n"
+                  "<code>var_int = 2\n"
+                  "var_str = 'test'\n"
+                  "var_bool = False\n"
+                  "var_float = 3.5</code>",
+        },
+        'reply_keyboard_button': {
+            'en': "Config 🔧",
+            'it': "Configurazione 🔧",
+        },
+        'success': {
+            'en': "<b>✔️ Configuration updated 🔧</b>\n\n"
+                  "The following variable has been added to the configuration "
+                  "file:\n"
+                  "<code>{new_variable}</code>\n\n"
+                  "/restart to apply",
+            'it': "<b>✔️ Configurazione aggiornata 🔧</b>\n\n"
+                  "La seguente variabile è stata aggiunta al file di "
+                  "configurazione:\n"
+                  "<code>{new_variable}</code>\n\n"
+                  "Per rendere effettive le modifiche fai /restart",
+        },
+    },
     'confirm': {
         'en': "🔄 Click again to confirm",
         'it': "🔄 Clicka di nuovo per confermare",
     },
     'db_command': {
+        'db_sent': {
+            'en': "Database sent.",
+            'it': "Database inviato.",
+        },
         'description': {
             'en': "Ask for bot database via Telegram",
-            'it': "Ricevi il database del bot via Telegram"
+            'it': "Ricevi il database del bot via Telegram",
+        },
+        'error': {
+            'en': "Error sending database.",
+            'it': "Errore durante l'invio del database.",
+        },
+        'file_caption': {
+            'en': "Here is bot database.",
+            'it': "Ecco il database!"
         },
         'not_sqlite': {
             'en': "Only SQLite databases may be sent via Telegram, since they "
@@ -49,14 +112,6 @@ default_admin_messages = {
                   "in quanto composti di un solo file.\n"
                   "Questo bot ha invece un database `{db_type}`."
         },
-        'file_caption': {
-            'en': "Here is bot database.",
-            'it': "Ecco il database!"
-        },
-        'db_sent': {
-            'en': "Database sent.",
-            'it': "Database inviato."
-        }
     },
     'error': {
         'text': {
@@ -71,12 +126,12 @@ default_admin_messages = {
         },
         'no_log': {
             'en': "Sorry but no errors log file is set.\n"
-                  "To set it, use `bot.set_errors_file_name` instance method"
+                  "To set it, use `bot.set_errors_file_name` instance method "
                   "or `Bot.set_class_errors_file_name` class method.",
             'it': "Spiacente ma il file di log degli errori non è stato "
                   "impostato.\n"
                   "Per impostarlo, usa il metodo d'istanza "
-                  "`bot.set_errors_file_name` o il metodo di classe"
+                  "`bot.set_errors_file_name` o il metodo di classe "
                   "`Bot.set_class_errors_file_name`."
         },
         'empty_log': {
@@ -701,17 +756,35 @@ default_authorization_messages = {
         }
     },
     'auth_button': {
-        'description': {
-            'en': "Edit user permissions",
-            'it': "Cambia il grado di autorizzazione di un utente"
+        'appointed': {
+            'en': "Permission granted",
+            'it': "Permesso conferito"
+        },
+        'browse': {
+            'browse_button_az': {
+                'en': "🆎 Users A-Z",
+                'it': "🆎 Utenti A-Z",
+            },
+            'browse_button_by_role': {
+                'en': "👑 Users by role",
+                'it': "👑 Utenti per ruolo",
+            },
+        },
+        'back_to_user': {
+            'en': "Back to user",
+            'it': "Torna all'utente"
         },
         'confirm': {
             'en': "Are you sure?",
             'it': "Sicuro sicuro?"
         },
-        'back_to_user': {
-            'en': "Back to user",
-            'it': "Torna all'utente"
+        'description': {
+            'en': "Edit user permissions",
+            'it': "Cambia il grado di autorizzazione di un utente"
+        },
+        'no_change': {
+            'en': "No change suggested!",
+            'it': "È già così!"
         },
         'permission_denied': {
             'user': {
@@ -724,14 +797,10 @@ default_authorization_messages = {
                 'it': "Non hai l'autorità di conferire questo permesso!"
             }
         },
-        'no_change': {
-            'en': "No change suggested!",
-            'it': "È già così!"
+        'profile_picture_button': {
+            'en': "🖼 Profile picture",
+            'it': "🖼 Foto profilo",
         },
-        'appointed': {
-            'en': "Permission granted",
-            'it': "Permesso conferito"
-        }
     },
 }
 
@@ -973,6 +1042,12 @@ default_talk_messages = {
         'en': 'End session?',
         'it': 'Chiudere la sessione?',
     },
+    'error': {
+        'text': {
+            'en': "❌️ Error!",
+            'it': "❌️ Errore!"
+        },
+    },
     'help_text': {
         'en': 'Press the button to search for user.',
         'it': 'Premi il pulsante per scegliere un utente.',

davtelepot/suggestions.py

@@ -4,12 +4,10 @@
 import asyncio
 import datetime
 
-# Third party modules
-import davtelepot
-
 # Project modules
-from .messages import default_suggestion_messages
-from .utilities import (
+import davtelepot
+from davtelepot.messages import default_suggestion_messages
+from davtelepot.utilities import (
     async_wrapper, get_cleaned_text, make_button,
     make_inline_keyboard, send_csv_file
 )
@@ -262,7 +260,7 @@ def init(telegram_bot: davtelepot.bot.Bot, suggestion_messages=None):
         )
         table.create_column(
             'suggestion',
-            types.text
+            types.string(2048)
         )
         table.create_column(
             'created',

davtelepot/tools/merge_files.py

@@ -0,0 +1,66 @@
+"""Use this tool to merge davtelepot-split files.
+
+Documents larger than 50 MB are automatically split by `send_document` method
+    and can be merged with this tool.
+Example:
+python3 -m davtelepot.tools.merge_files my_file.pdf
+"""
+
+import argparse
+import glob
+import logging
+import os
+
+
+def merge_files(input_file_path, output_file_path):
+    input_directory = os.path.dirname(os.path.abspath(input_file_path))
+    input_file_name = os.path.basename(os.path.abspath(input_file_path))
+    if output_file_path:
+        output_directory = os.path.dirname(os.path.abspath(output_file_path))
+        output_file_name = os.path.basename(os.path.abspath(output_file_path))
+    else:
+        output_directory = input_directory
+        output_file_name = input_file_name + 'out'
+    with open(os.path.join(output_directory, output_file_name), 'wb') as output_file:
+        for file_name in sorted(glob.glob(os.path.join(input_directory, input_file_name) + '*')):
+            with open(file_name, 'rb') as input_file:
+                output_file.write(input_file.read())
+
+
+def main():
+    # noinspection SpellCheckingInspection
+    log_formatter = logging.Formatter(
+        "%(asctime)s [%(module)-15s %(levelname)-8s]     %(message)s",
+        style='%'
+    )
+    root_logger = logging.getLogger()
+    root_logger.setLevel(logging.DEBUG)
+
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(log_formatter)
+    console_handler.setLevel(logging.DEBUG)
+    root_logger.addHandler(console_handler)
+
+    # Parse command-line arguments
+    cli_parser = argparse.ArgumentParser(description='Merge split files',
+                                         allow_abbrev=False)
+    cli_parser.add_argument('--input', '--file-path', '--path',
+                            '-i', '-f', '-p',
+                            type=str,
+                            required=True,
+                            help='Input file name (except ` - part n`)')
+    cli_parser.add_argument('--output', '--output-file-path', '--out',
+                            '-o',
+                            type=str,
+                            default=None,
+                            required=False,
+                            help='Output file name (defaults to input file name)')
+    cli_arguments = vars(cli_parser.parse_args())
+    input_file_path = cli_arguments['input']
+    output_file_path = cli_arguments['output']
+    merge_files(input_file_path=input_file_path,
+                output_file_path=output_file_path)
+
+
+if __name__ == '__main__':
+    main()

davtelepot/useful_tools.py

@@ -13,11 +13,12 @@ from collections import OrderedDict
 from typing import List, Union
 
 # Project modules
-from .api import TelegramError
-from .bot import Bot
-from .messages import default_useful_tools_messages
-from .utilities import (get_cleaned_text, get_user, make_button,
-                        make_inline_keyboard, recursive_dictionary_update, )
+from davtelepot.api import TelegramError
+from davtelepot.bot import Bot
+from davtelepot.messages import default_useful_tools_messages
+from davtelepot.utilities import (get_cleaned_text, get_user, make_button,
+                                  make_inline_keyboard,
+                                  recursive_dictionary_update, )
 
 
 def get_calc_buttons() -> OrderedDict:
@@ -664,7 +665,7 @@ def init(telegram_bot: Bot, useful_tools_messages=None):
         )
         table.create_column(
             'expression',
-            types.string
+            types.string(8192)
         )
 
     @telegram_bot.command(command='/calc',

davtelepot/utilities.py

@@ -16,12 +16,12 @@ import string
 import time
 
 from difflib import SequenceMatcher
+from typing import Tuple, Union
 
 # Third party modules
-from typing import Union
 
 import aiohttp
-from aiohttp import web
+import dataset
 from bs4 import BeautifulSoup
 
 
@@ -504,7 +504,9 @@ async def async_wrapper(coroutine, *args1, **kwargs1):
             )
             await my_coroutine(a=1, b=5)
 
-        asyncio.get_event_loop().run_until_complete(main())
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        asyncio.run(main())
     ```
     """
     async def wrapped_coroutine(*args2, bot=None, update=None, user_record=None, **kwargs2):
@@ -1250,7 +1252,7 @@ def parse_datetime_interval_string(text):
             result_text.pop()
             if len(result_text) > 0 and result_text[-1].lower() in TIME_WORDS:
                 result_text.pop()
-    result_text = escape_html_chars(
+    result_text = clean_html_string(
         ' '.join(result_text)
     )
     parsers = list(
@@ -1329,6 +1331,22 @@ MONTH_NAMES_ITA[10] = "ottobre"
 MONTH_NAMES_ITA[11] = "novembre"
 MONTH_NAMES_ITA[12] = "dicembre"
 
+allowed_html_tags = ['b', 'strong',
+                     'i', 'em',
+                     'u', 'ins',
+                     's', 'strike', 'del',
+                     'span', 'tg-spoiler',
+                     'a',
+                     'code', 'pre']
+
+HTML_SYMBOLS = collections.OrderedDict()
+HTML_SYMBOLS["&"] = "&"
+HTML_SYMBOLS["<"] = "&lt;"
+HTML_SYMBOLS[">"] = "&gt;"
+HTML_SYMBOLS["\""] = "&quot;"
+
+html_numeric_code_regex = re.compile(r'&amp;(?P<code>#\d{2,3};)')
+
 
 def beautytd(td):
     """Format properly timedeltas."""
@@ -1409,67 +1427,57 @@ def beautydt(dt):
     return result
 
 
-HTML_SYMBOLS = MyOD()
-HTML_SYMBOLS["&"] = "&amp;"
-HTML_SYMBOLS["<"] = "&lt;"
-HTML_SYMBOLS[">"] = "&gt;"
-HTML_SYMBOLS["\""] = "&quot;"
-HTML_SYMBOLS["&lt;b&gt;"] = "<b>"
-HTML_SYMBOLS["&lt;/b&gt;"] = "</b>"
-HTML_SYMBOLS["&lt;i&gt;"] = "<i>"
-HTML_SYMBOLS["&lt;/i&gt;"] = "</i>"
-HTML_SYMBOLS["&lt;code&gt;"] = "<code>"
-HTML_SYMBOLS["&lt;/code&gt;"] = "</code>"
-HTML_SYMBOLS["&lt;pre&gt;"] = "<pre>"
-HTML_SYMBOLS["&lt;/pre&gt;"] = "</pre>"
-HTML_SYMBOLS["&lt;a href=&quot;"] = "<a href=\""
-HTML_SYMBOLS["&quot;&gt;"] = "\">"
-HTML_SYMBOLS["&lt;/a&gt;"] = "</a>"
+def clean_html_string(text: str) -> str:
+    """Escape HTML symbols, unless part of a valid tag or numeric code character.
 
-HTML_TAGS = [
-    None, "<b>", "</b>",
-    None, "<i>", "</i>",
-    None, "<code>", "</code>",
-    None, "<pre>", "</pre>",
-    None, "<a href=\"", "\">", "</a>",
-    None
-]
-
-
-def remove_html_tags(text):
-    """Remove HTML tags from `text`."""
-    for tag in HTML_TAGS:
-        if tag is None:
-            continue
-        text = text.replace(tag, '')
+    Find valid HTML tags;
+    if there are any, choose the first occurring and call the function
+        recursively on what comes before the tag, inside the tag and after the
+        tag, preserving the tag opening and close as they are;
+    if there aren't any, escape HTML symbols except for `&` in HTML numeric code
+        characters (`&#` followed by 2 or 3 digits followed by `;`).
+    """
+    first_match = None
+    for tag in allowed_html_tags:
+        if tag in ('a', ):  # <a> must have href attribute
+            attribute = r" href=\".*\""
+        elif tag in ('span', ):  # <span> must have class attribute with "tg-spoiler" value
+            attribute = r" class=\"tg-spoiler\""
+        elif tag in ('code',):  # <code> may have a class with a programming language as value
+            attribute = r"( class=\".*\")?"
+        else:
+            attribute = ""
+        match = re.search(
+            rf'(?P<opening><{tag}{attribute}>)'
+            rf'(?P<body>.*?)'
+            rf'(?P<close></{tag}>)',
+            text,
+            flags=re.DOTALL
+        )
+        if match and (first_match is None or match.start() < first_match.start()):
+            first_match = match
+    if first_match is not None:
+        groups = first_match.groupdict()
+        text = (f"{clean_html_string(text[:first_match.start()])}"
+                f"{groups['opening']}{clean_html_string(groups['body'])}{groups['close']}"
+                f"{clean_html_string(text[first_match.end():])}")
+    else:
+        for key, value in HTML_SYMBOLS.items():
+            text = text.replace(key, value)
+        if re.search(html_numeric_code_regex, text):
+            text = re.sub(html_numeric_code_regex, r'&\g<code>', text)
     return text
 
 
 def escape_html_chars(text):
-    """Escape HTML chars if not part of a tag."""
-    for s, r in HTML_SYMBOLS.items():
-        text = text.replace(s, r)
-    copy = text
-    expected_tag = None
-    while copy:
-        min_ = min(
-            (
-                dict(
-                    position=copy.find(tag) if tag in copy else len(copy),
-                    tag=tag
-                )
-                for tag in HTML_TAGS
-                if tag
-            ),
-            key=lambda x: x['position'],
-            default=0
-        )
-        if min_['position'] == len(copy):
-            break
-        if expected_tag and min_['tag'] != expected_tag:
-            return text.replace('<', '_').replace('>', '_')
-        expected_tag = HTML_TAGS[HTML_TAGS.index(min_['tag'])+1]
-        copy = extract(copy, min_['tag'])
+    logging.error("`escape_html_chars` function deprecated, use `clean_html_string` instead.")
+    return clean_html_string(text)
+
+
+def remove_html_tags(text):
+    """Remove HTML tags from `text`."""
+    for tag in allowed_html_tags:
+        text = re.sub(rf'</?{tag}( (href|class)=\".*\")?>', '', text)
     return text
 
 
@@ -1566,7 +1574,7 @@ def run_aiohttp_server(app, *args, **kwargs):
     """
     loop = asyncio.new_event_loop()
     asyncio.set_event_loop(loop)
-    web.run_app(app, *args, **kwargs)
+    aiohttp.web.run_app(app, *args, **kwargs)
 
 
 def custom_join(_list, joiner, final=None):
@@ -1609,8 +1617,9 @@ async def dummy_coroutine(*args, **kwargs):
     return
 
 
-async def send_csv_file(bot, chat_id, query, caption=None,
-                        file_name='File.csv', user_record=None, update=None):
+async def send_csv_file(bot, chat_id: int, query: str, caption: str = None,
+                        file_name: str = 'File.csv', language: str = None,
+                        user_record=None, update=None):
     """Run a query on `bot` database and send result as CSV file to `chat_id`.
 
     Optional parameters `caption` and `file_name` may be passed to this
@@ -1618,6 +1627,9 @@ async def send_csv_file(bot, chat_id, query, caption=None,
     """
     if update is None:
         update = dict()
+    if language is None:
+        language = bot.get_language(update=update,
+                                    user_record=user_record)
     try:
         with bot.db as db:
             record = db.query(
@@ -1633,14 +1645,14 @@ async def send_csv_file(bot, chat_id, query, caption=None,
     except Exception as e:
         text = "{message}\n{e}".format(
             message=bot.get_message('admin', 'query_button', 'error',
-                                    user_record=user_record, update=update),
+                                    language=language),
             e=e
         )
     for x, y in {'&lt;': '<', '\n': '\r\n'}.items():
         text = text.replace(x, y)
     if len(text) == 0:
         text = bot.get_message('admin', 'query_button', 'empty_file',
-                               user_record=user_record, update=update)
+                               language=language)
     with io.BytesIO(text.encode('utf-8')) as f:
         f.name = file_name
         return await bot.send_document(
@@ -1654,7 +1666,8 @@ async def send_part_of_text_file(bot, chat_id, file_path, caption=None,
                                  file_name='File.txt', user_record=None,
                                  update=None,
                                  reversed_=True,
-                                 limit=None):
+                                 limit=None,
+                                 encoding='utf-8'):
     """Send `lines` lines of text file via `bot` in `chat_id`.
 
     If `reversed`, read the file from last line.
@@ -1665,7 +1678,7 @@ async def send_part_of_text_file(bot, chat_id, file_path, caption=None,
     if update is None:
         update = dict()
     try:
-        with open(file_path, 'r') as log_file:
+        with open(file_path, 'r', encoding=encoding) as log_file:
             lines = log_file.readlines()
             if reversed_:
                 lines = lines[::-1]
@@ -1700,3 +1713,54 @@ def recursive_dictionary_update(one: dict, other: dict) -> dict:
             else:
                 one[key] = val
     return one
+
+
+async def aio_subprocess_shell(command: str,
+                               stdout=None,
+                               stderr=None) -> Tuple[str, str]:
+    """Run `command` in a subprocess shell.
+
+    Await for the subprocess to end and return standard error and output.
+    On error, log errors.
+    If `stdout` and/or `stderr` are given, use them as output and/or error pipes.
+    Example of non-null standard output/error pipe: asyncio.subprocess.PIPE
+    """
+    try:
+        _subprocess = await asyncio.create_subprocess_shell(
+            command,
+            stdout=stdout,
+            stderr=stderr
+        )
+        stdout, stderr = await _subprocess.communicate()
+        if stdout:
+            stdout = stdout.decode().strip()
+        if stderr:
+            stderr = stderr.decode().strip()
+    except Exception as e:
+        logging.error(
+            "Exception {e}:\n{o}\n{er}".format(
+                e=e,
+                o=(stdout.decode().strip() if stdout else ''),
+                er=(stderr.decode().strip() if stderr else '')
+            )
+        )
+    return stdout, stderr
+
+
+def join_path(*args):
+    return os.path.abspath(os.path.join(*args))
+
+
+def add_table_and_columns_if_not_existent(database: dataset.Database,
+                                          table_name: str,
+                                          columns: Tuple[Tuple, ...]):
+    if table_name not in database.tables:
+        table = database.create_table(table_name=table_name)
+    else:
+        table = database[table_name]
+    for column_name, column_type in columns:
+        if not table.has_column(column_name):
+            table.create_column(
+                column_name,
+                column_type
+            )

examples/a_simple_bot.py

@@ -46,7 +46,7 @@ import sys
 try:
     from davtelepot.bot import Bot
     from davtelepot.utilities import (
-        get_cleaned_text, make_inline_keyboard, make_button
+        get_cleaned_text, get_user, make_inline_keyboard, make_button
     )
 except ImportError:
     logging.error(
@@ -64,23 +64,27 @@ except ImportError:
 path = os.path.dirname(os.path.abspath(__file__))
 
 
-def initialize_bot(bot):
+def initialize_bot(telegram_bot):
     """Take a bot and set commands."""
-    bot.set_callback_data_separator('|')
+    telegram_bot.set_callback_data_separator('|')
 
-    @bot.command(command='foo', aliases=['Foo'], show_in_keyboard=True,
+    @telegram_bot.command(command='foo', aliases=['Foo'],
+                          show_in_keyboard=True,
                           description="Reply 'bar' to 'foo'",
                           authorization_level='everybody')
     async def foo_command(bot, update, user_record):
-        return 'Bar!'
+        return (
+            "Bar!\n\n"
+            f"You wrote: {update['text']}\n"
+            f"I am @{bot.name} and you are {get_user(user_record)}"
+        )
 
-    def is_bar_text_message(text):
-        return text.startswith('bar')
+    def is_bar_text_message(lowered_text):
+        return lowered_text.startswith('bar')
 
-    @bot.parser(condition=is_bar_text_message,
+    @telegram_bot.parser(condition=is_bar_text_message,
                          description='Reply Foo to users who write Bar',
-                authorization_level='everybody',
-                argument='text')
+                         authorization_level='everybody')
     async def bar_parser(bot, update):
         text_except_foo = get_cleaned_text(update, bot, ['bar'])
         return f"Foo!\n{text_except_foo}"
@@ -99,16 +103,16 @@ def initialize_bot(bot):
             3
         )
 
-    @bot.command(command='buttons')
+    @telegram_bot.command(command='buttons')
     async def buttons_command():
         return dict(
             text="Press a button!",
             reply_markup=get_keyboard()
         )
 
-    @bot.button(prefix='button:///', separator='|',
+    @telegram_bot.button(prefix='button:///', separator='|',
                          authorization_level='everybody')
-    async def buttons_button(bot, update, user_record, data):
+    async def buttons_button(data):
         button_number = data[0]
         return dict(
             edit=dict(
@@ -120,12 +124,12 @@ def initialize_bot(bot):
     def starts_with_a(text):
         return text.startswith('a')
 
-    @bot.query(
+    @telegram_bot.query(
         condition=starts_with_a,
         description='Mirror query text if it starts with letter `a`',
         authorization_level='everybody'
     )
-    async def inline_query(bot, update, user_record):
+    async def inline_query(update):
         return dict(
             type='article',
             id=10,
@@ -135,7 +139,7 @@ def initialize_bot(bot):
             )
         )
 
-    bot.set_default_inline_query_answer(
+    telegram_bot.set_default_inline_query_answer(
         dict(
             type='article',
             id=0,
@@ -146,7 +150,7 @@ def initialize_bot(bot):
         )
     )
 
-    bot.set_unknown_command_message(
+    telegram_bot.set_unknown_command_message(
         "<b>Currently supported features</b>\n\n"
         "- /foo (or text starting with `foo`): replies `Bar!`.\n"
         "- Text starting with `bar`: replies `Foo!` followed by the rest of "
@@ -177,10 +181,10 @@ def _main():
     )
     root_logger = logging.getLogger()
     root_logger.setLevel(logging.DEBUG)
-    consoleHandler = logging.StreamHandler()
-    consoleHandler.setFormatter(log_formatter)
-    consoleHandler.setLevel(logging.DEBUG)
-    root_logger.addHandler(consoleHandler)
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(log_formatter)
+    console_handler.setLevel(logging.DEBUG)
+    root_logger.addHandler(console_handler)
 
     # Instantiate, initialize and make `simple_bot` run.
     simple_bot = Bot(token=simple_bot_token, database_url=f"{path}/bot.db")

examples/more_bots_together.py

@@ -28,7 +28,7 @@ except ImportError:
 from a_simple_bot import initialize_bot
 
 # Get path of current script
-os.path.dirname(os.path.abspath(__file__))
+path = os.path.dirname(os.path.abspath(__file__))
 
 
 def _main():
@@ -96,10 +96,10 @@ def _main():
     )
     root_logger = logging.getLogger()
     root_logger.setLevel(logging.DEBUG)
-    consoleHandler = logging.StreamHandler()
-    consoleHandler.setFormatter(log_formatter)
-    consoleHandler.setLevel(logging.DEBUG)
-    root_logger.addHandler(consoleHandler)
+    console_handler = logging.StreamHandler()
+    console_handler.setFormatter(log_formatter)
+    console_handler.setLevel(logging.DEBUG)
+    root_logger.addHandler(console_handler)
 
     # Instantiate, initialize and make bots run.
     webhook_bot = Bot(

merge_and_push.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-
-# Merge develop into master and push both branches; checkout to develop at the end.
-
-git checkout master;
-git merge develop;
-git checkout develop;
-git push origin master develop;
-git push bitbucket master develop;
-git push github master;

push_to_remotes.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Merge develop into master and push both branches; checkout to develop at the end.
+
+git push origin main develop;
+git push bitbucket main develop;
+git push github main;

requirements.txt

@@ -1,3 +1,5 @@
 aiohttp
 bs4
 dataset
+beautifulsoup4
+SQLAlchemy

update_package.sh

@@ -1,23 +1,24 @@
 #!/bin/bash
 
 # Get current directory
-this_script_directory=$(cd `dirname $0` && pwd)
+this_script_directory=$(cd "$(dirname "$0")" && pwd)
+packenv="";
 
 # Python virtual environment directory: packenv variable in my_config.sh
-source $this_script_directory/my_config.sh;
+source "$this_script_directory"/my_config.sh;
 
 # Ensure the success of importing procedure
-if [ -z ${packenv} ];
+if [ -z "${packenv}" ];
 then
   printf "Please set in ""my_config.sh"" the path to bot python virtual environment\n\nExample:\npackenv=""path/to/virtual/env""\n";
   exit;
 fi
 
-# Merge, push, build and publish package to pypi.org
-bash merge_and_push.sh;
+# Push, build and publish package to pypi.org
+bash "$this_script_directory"/push_to_remotes.sh;
 rm -rf "$this_script_directory/build";
 rm -rf "$this_script_directory/dist";
 rm -rf "$this_script_directory/davtelepot.egg-info";
-$packenv/python setup.py sdist;
-$packenv/pip wheel --no-index --no-deps --wheel-dir dist dist/*.tar.gz;
-$packenv/twine upload --skip-existing dist/*;
+"$packenv"/python setup.py sdist;
+"$packenv"/pip wheel --no-index --no-deps --wheel-dir dist dist/*.tar.gz;
+"$packenv"/twine upload --skip-existing dist/*;