adds runtime.txt

Allow gifs in channel

README.md

@@ -5,10 +5,11 @@ Head to https://www.originprotocol.com/developers to learn more about what we're
 # Telegram Bot
 
 - Deletes messages matching specified patterns
-- Bans users for posting messagses matching specified patterns
+- Bans users for posting messages matching specified patterns
 - Bans users with usernames matching specified patterns
-- Records logs of converstations
+- Records logs of conversations
 - Logs an English translation of any foreign languages using Google Translate
+- Uses textblob for basic sentiment analysis of both polarity and subjectivity
 
 ## Installation
 
@@ -52,6 +53,23 @@ export TELEGRAM_BOT_TOKEN="4813829027:ADJFKAf0plousH2EZ2jBfxxRWFld3oK34ya"
 - `DEBUG` : If set to anything except `false`, will put bot into debug mode. This means that all actions will be logged into the chat itself, and more things will be logged.
 - `ADMIN_EXEMPT` : If set to anything except `false`, admin users will be exempt from monitoring. Reccomended to be set, but useful to turn off for debugging.
 - `NOTIFY_CHAT` : ID of chat to report actions. Can be useful if you have an admin-only chat where you want to monitor the bot's activity. E.g. `-140532994`
+- `CMC_API_KEY`: If you want the `/price` bot command to work, make sure to set a CoinMarketcap API key
+
+## Download the corpus for Textblob
+
+For sentiment analysis to work, you'll need to download the latest corpus file for textblob. You can do this by running:
+
+```
+python -m textblob.download_corpora
+```
+
+If you're running the bot on Heroku, set an environment variable named `NLTK_DATA` to `/app/nltk_data` by running:
+
+```
+heroku config:set NLTK_DATA='/app/nltk_data'
+```
+
+## Message ban patterns
 
 Sample bash file to set `MESSAGE_BAN_PATTERNS`:
 
@@ -66,7 +84,7 @@ read -r -d '' MESSAGE_BAN_PATTERNS << 'EOF'
 EOF
 ```
 
-## Attachements
+## Attachments
 
 By default, any attachments other than images or animations will cause the message to be hidden.
 

bot.py

@@ -9,16 +9,203 @@ This bot logs all messages sent in a Telegram Group to a database.
 
 """
 
-from __future__ import print_function
+#from __future__ import print_function
-import sys
-from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
 import os
-from model import User, Message, MessageHide, UserBan, session
+import sys
-from time import strftime
 import re
 import unidecode
+import locale
+import traceback
+from time import strftime
+from datetime import datetime, timedelta
+
+import requests
+import telegram
+from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
+from model import User, Message, MessageHide, UserBan, session
 from mwt import MWT
 from googletrans import Translator
+from textblob import TextBlob
+
+# Used with monetary formatting
+locale.setlocale(locale.LC_ALL, '')
+
+# Price data cache duration
+CACHE_DURATION = timedelta(minutes=15)
+
+# CMC IDs can be retrived at:
+# https://pro-api.coinmarketcap.com/v1/cryptocurrency/map?symbol=[SYMBOL]
+CMC_SYMBOL_TO_ID = {
+    'OGN': 5117,
+    'USDT': 825,
+    'USDC': 3408,
+    'DAI': 4943,
+}
+CMC_API_KEY = os.environ.get('CMC_API_KEY')
+CMC_USD_QUOTE_URL = 'https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest?id={}'
+CMC_BTC_QUOTE_URL = 'https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest?id={}&convert=BTC'
+
+
+def first_of(attr, match, it):
+    """ Return the first item in a set with an attribute that matches match """
+    if it is not None:
+        for i in it:
+            try:
+                if getattr(i, attr) == match:
+                    return i
+            except: pass
+
+    return None
+
+
+def command_from_message(message, default=None):
+    """ Extracts the first command from a Telegram Message """
+    if not message or not message.text:
+        return default
+
+    command = None
+    text = message.text
+    entities = message.entities
+    command_def = first_of('type', 'bot_command', entities)
+
+    if command_def:
+        command = text[command_def.offset:command_def.length]
+
+    return command or default
+
+
+def cmc_get_data(jso, cmc_id, pair_symbol='USD'):
+    """ Pull relevant data from a response object """
+    if not jso:
+        return None
+
+    data = jso.get('data', {})
+    specific_data = data.get(str(cmc_id), {})
+    quote = specific_data.get('quote', {})
+    symbol_data = quote.get(pair_symbol, {})
+    return {
+        'price': symbol_data.get('price'),
+        'volume': symbol_data.get('volume_24h'),
+        'percent_change': symbol_data.get('percent_change_24h'),
+        'market_cap': symbol_data.get('market_cap'),
+    }
+
+
+def decimal_format(v, decimals=2):
+    if not v:
+        v = 0
+    f = locale.format_string('%.{}f'.format(decimals), v, grouping=True)
+    return '{}'.format(f)
+
+
+def monetary_format(v, decimals=2):
+    return '${}'.format(decimal_format(v, decimals))
+
+def btc_format(v):
+    return decimal_format(v, decimals=8)
+
+
+
+class TokenData:
+    def __init__(self, symbol, price=None, stamp=datetime.now()):
+        self.symbol = symbol
+        self._price = price
+        self._btc_price = 0
+        self._percent_change = 0
+        self._btc_percent_change = 0
+        self._volume = 0
+        self._market_cap = 0
+        if price is not None:
+            self.stamp = stamp
+        else:
+            self.stamp = None
+
+    def _fetch_from_cmc(self, url_template):
+        """ Get quote data for a specific known symbol """
+        jso = None
+
+        cmc_id = CMC_SYMBOL_TO_ID.get(self.symbol)
+        url = url_template.format(cmc_id)
+        r = requests.get(url, headers={
+            'X-CMC_PRO_API_KEY': CMC_API_KEY,
+            'Accept': 'application/json',
+        })
+        if r.status_code != 200:
+            print('Failed to fetch price data for id: {}'.format(cmc_id))
+            return None
+        try:
+            jso = r.json()
+        except Exception:
+            print('Error parsing JSON')
+            return None
+        return jso
+
+    def update(self):
+        """ Fetch price from binance """
+        jso = None
+        data = None
+        if self.stamp is None or (
+            self.stamp is not None
+            and self.stamp < datetime.now() - CACHE_DURATION
+        ):
+            # CMC USD
+            try:
+                jso = self._fetch_from_cmc(CMC_USD_QUOTE_URL)
+                data = cmc_get_data(jso, CMC_SYMBOL_TO_ID[self.symbol])
+            except Exception as err:
+                print('Error fetching data: ', str(err))
+                print(traceback.format_exc())
+            if data is not None:
+                self._price = data.get('price')
+                self._percent_change = data.get('percent_change')
+                self._volume = data.get('volume')
+                self._market_cap = data.get('market_cap')
+                self.stamp = datetime.now()
+
+            # CMC BTC
+            try:
+                jso = self._fetch_from_cmc(CMC_BTC_QUOTE_URL)
+                data = cmc_get_data(jso, CMC_SYMBOL_TO_ID[self.symbol], 'BTC')
+            except Exception as err:
+                print('Error fetching data: ', str(err))
+                print(traceback.format_exc())
+            if data is not None:
+                self._btc_price = data.get('price')
+                self._btc_percent_change = data.get('percent_change')
+
+    @property
+    def price(self):
+        self.update()
+        return self._price
+
+    @property
+    def btc_price(self):
+        self.update()
+        return self._btc_price
+
+    @property
+    def btc_percent_change(self):
+        self.update()
+        return self._btc_percent_change
+
+    @property
+    def volume(self):
+        self.update()
+        return self._volume
+
+    @property
+    def percent_change(self):
+        self.update()
+        pc = str(self._percent_change)
+        if pc and not pc.startswith('-'):
+            pc = '+{}'.format(pc)
+        return pc
+
+    @property
+    def market_cap(self):
+        self.update()
+        return self._market_cap
+
 
 class TelegramMonitorBot:
 
@@ -42,10 +229,17 @@ class TelegramMonitorBot:
             print("🔵 MESSAGE_BAN_PATTERNS:\n", os.environ['MESSAGE_BAN_PATTERNS'])
             print("🔵 MESSAGE_HIDE_PATTERNS:\n", os.environ['MESSAGE_HIDE_PATTERNS'])
             print("🔵 NAME_BAN_PATTERNS:\n", os.environ['NAME_BAN_PATTERNS'])
+            print("🔵 IGNORE_USER_IDS:\n", os.environ.get('IGNORE_USER_IDS'))
 
         # Channel to notify of violoations, e.g. '@channelname'
         self.notify_chat = os.environ['NOTIFY_CHAT'] if 'NOTIFY_CHAT' in os.environ else None
 
+        # Ignore these user IDs
+        if not os.environ.get('IGNORE_USER_IDS'):
+            self.ignore_user_ids = []
+        else:
+            self.ignore_user_ids = list(map(int, os.environ['IGNORE_USER_IDS'].split(',')))
+
         # List of chat ids that bot should monitor
         self.chat_ids = (
             list(map(int, os.environ['CHAT_IDS'].split(',')))
@@ -72,6 +266,22 @@ class TelegramMonitorBot:
             re.IGNORECASE | re.VERBOSE)
             if self.name_ban_patterns else None)
 
+        # Mime type document check
+        # NOTE: All gifs appear to be converted to video/mp4
+        mime_types = os.environ.get('ALLOWED_MIME_TYPES', 'video/mp4')
+        self.allowed_mime_types = set(map(lambda s: s.strip(), mime_types.split(',')))
+
+        # Comamnds
+        self.available_commands = ['flip', 'unflip']
+        if CMC_API_KEY is not None:
+            self.available_commands.append('price')
+
+        print('Available commands: {}'.format(', '.join(self.available_commands)))
+
+        # Cached token prices
+        self.cached_prices = {}
+        self.last_message_out = None
+
 
     @MWT(timeout=60*60)
     def get_admin_ids(self, bot, chat_id):
@@ -107,7 +317,8 @@ class TelegramMonitorBot:
             s.commit()
             s.close()
             # Notify channel
-            bot.sendMessage(chat_id=self.notify_chat, text=log_message)
+            if self.notify_chat:
+                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
         if self.name_ban_re and self.name_ban_re.search(update.message.from_user.username or ''):
             # Logging
@@ -126,7 +337,8 @@ class TelegramMonitorBot:
             s.commit()
             s.close()
             # Notify channel
-            bot.sendMessage(chat_id=self.notify_chat, text=log_message)
+            if self.notify_chat:
+                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
 
     def security_check_message(self, bot, update):
@@ -158,7 +370,8 @@ class TelegramMonitorBot:
             s.commit()
             s.close()
             # Notify channel
-            bot.sendMessage(chat_id=self.notify_chat, text=log_message)
+            if self.notify_chat:
+                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
         if self.message_ban_re and self.message_ban_re.search(message):
             # Logging
@@ -179,7 +392,8 @@ class TelegramMonitorBot:
             s.commit()
             s.close()
             # Notify channel
-            bot.sendMessage(chat_id=self.notify_chat, text=log_message)
+            if self.notify_chat:
+                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
         elif self.message_hide_re and self.message_hide_re.search(message):
             # Logging
@@ -198,7 +412,8 @@ class TelegramMonitorBot:
             s.commit()
             s.close()
             # Notify channel
-            bot.sendMessage(chat_id=self.notify_chat, text=log_message)
+            if self.notify_chat:
+                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
 
     def attachment_check(self, bot, update):
@@ -209,6 +424,10 @@ class TelegramMonitorBot:
             update.message.voice):
             # Logging
             if update.message.document:
+                # GIFs are documents and allowed
+                mime_type = update.message.document.mime_type
+                if mime_type and mime_type in self.allowed_mime_types:
+                    return
                 log_message = "❌ HIDE DOCUMENT: {}".format(update.message.document.__dict__)
             else:
                 log_message = "❌ HIDE NON-DOCUMENT ATTACHMENT"
@@ -226,25 +445,55 @@ class TelegramMonitorBot:
             s.commit()
             s.close()
             # Notify channel
-            bot.sendMessage(chat_id=self.notify_chat, text=log_message)
+            if self.notify_chat:
+                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
 
     def logger(self, bot, update):
-        """ Primary Logger. Handles incoming bot messages and saves them to DB """
+        """ Primary Logger. Handles incoming bot messages and saves them to DB
+
+        :param bot: telegram.Bot https://python-telegram-bot.readthedocs.io/en/stable/telegram.bot.html
+        :param update: telegram.Update https://python-telegram-bot.readthedocs.io/en/stable/telegram.update.html
+        """
+
+        if (
+            update.effective_user is None
+            or update.effective_user.id in self.ignore_user_ids
+        ):
+            print("{}: Ignoring update.".format(update.update_id))
+            return
+
         try:
-            user = update.message.from_user
+
+            message = update.message
+
+            # message is optional
+            if message is None:
+
+                if update.effective_message is None:
+                    print("No message included in update")
+                    return
+
+                message = update.effective_message
+
+            if message:
+
+                user = message.from_user
 
                 # Limit bot to monitoring certain chats
-            if update.message.chat_id not in self.chat_ids:
+                if message.chat_id not in self.chat_ids:
+                    from_user = "UNKNOWN"
+                    if user:
+                        from_user = user.id
                     print("Message from user {} is from chat_id not being monitored: {}".format(
-                    user.id,
+                        from_user,
-                    update.message.chat_id)
+                        message.chat_id)
                     )
                     return
 
                 if self.id_exists(user.id):
-                self.log_message(user.id, update.message.text,
+                    self.log_message(user.id, message.text,
-                                 update.message.chat_id)
+                                     message.chat_id)
                 else:
                     add_user_success = self.add_user(
                         user.id,
@@ -254,7 +503,7 @@ class TelegramMonitorBot:
 
                     if add_user_success:
                         self.log_message(
-                        user.id, update.message.text, update.message.chat_id)
+                            user.id, message.text, message.chat_id)
                         print("User added: {}".format(user.id))
                     else:
                         print("Something went wrong adding the user {}".format(user.id), file=sys.stderr)
@@ -263,7 +512,7 @@ class TelegramMonitorBot:
                     user.username or
                     "{} {}".format(user.first_name, user.last_name) or
                     "<none>").encode('utf-8')
-            if update.message.text:
+                if message.text:
                     print("{} {} ({}) : {}".format(
                         strftime("%Y-%m-%dT%H:%M:%S"),
                         user.id,
@@ -277,8 +526,14 @@ class TelegramMonitorBot:
                         user_name)
                     )
 
+            else:
+                print("Update and user not logged because no message was found")
+
             # Don't check admin activity
-            is_admin = update.message.from_user.id in self.get_admin_ids(bot, update.message.chat_id)
+            is_admin = False
+            if message:
+                is_admin = message.from_user.id in self.get_admin_ids(bot, message.chat_id)
+
             if is_admin and self.admin_exempt:
                 print("👮‍♂️ Skipping checks. User is admin: {}".format(user.id))
             else:
@@ -288,7 +543,8 @@ class TelegramMonitorBot:
                 self.security_check_message(bot, update)
 
         except Exception as e:
-            print("Error: {}".format(e))
+            print("Error[521]: {}".format(e))
+            print(traceback.format_exc())
             print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
 
     # DB queries
@@ -303,25 +559,36 @@ class TelegramMonitorBot:
 
         return bool_set
 
-
     def log_message(self, user_id, user_message, chat_id):
+
+        if user_message is None:
+            user_message = "[NO MESSAGE]"
+
         try:
             s = session()
             language_code = english_message = ""
+            polarity = subjectivity = 0.0
             try:
+                # translate to English & log the original language
                 translator = Translator()
                 translated = translator.translate(user_message)
                 language_code = translated.src
                 english_message = translated.text
+                # run basic sentiment analysis on the translated English string
+                analysis = TextBlob(english_message)
+                polarity = analysis.sentiment.polarity
+                subjectivity = analysis.sentiment.subjectivity
             except Exception as e:
-                print(e.message)
+                print("Error translating message: {}".format(e))
-            msg1 = Message(user_id=user_id, message=user_message,
+            msg1 = Message(user_id=user_id, message=user_message, chat_id=chat_id, 
-                           chat_id=chat_id, language_code=language_code, english_message=english_message)
+                language_code=language_code, english_message=english_message, polarity=polarity,
+                subjectivity=subjectivity)
             s.add(msg1)
             s.commit()
             s.close()
         except Exception as e:
-            print("Error: {}".format(e))
+            print("Error logging message: {}".format(e))
+            print(traceback.format_exc())
 
 
     def add_user(self, user_id, first_name, last_name, username):
@@ -337,8 +604,94 @@ class TelegramMonitorBot:
             s.close()
             return self.id_exists(user_id)
         except Exception as e:
-            print("Error: {}".format(e))
+            print("Error[347]: {}".format(e))
+            print(traceback.format_exc())
 
+    def handle_command(self, bot, update):
+        """ Handles commands
+
+        Note: Args reversed from docs?  Maybe version differences?  Docs say
+        cb(update, context) but we're getting cb(bot, update).
+
+        update: Update: https://python-telegram-bot.readthedocs.io/en/stable/telegram.update.html#telegram.Update
+        context: CallbackContext: https://python-telegram-bot.readthedocs.io/en/stable/telegram.ext.callbackcontext.html
+
+        hi: says hi
+        price: prints the OGN price
+        """
+        chat_id = None
+        command = None
+        message_id = update.effective_message.message_id
+
+        command = command_from_message(update.effective_message)
+
+        if update.effective_message.chat:
+            chat_id = update.effective_message.chat.id
+
+        print('command: {} seen in chat_id {}'.format(command, chat_id))
+
+        if command == '/hi':
+            bot.send_message(chat_id, 'Yo whattup, @{}!'.format(update.effective_user.username))
+
+        elif command == '/flip':
+            bot.send_message(chat_id, '╯°□°）╯︵ ┻━┻')
+
+        elif command == '/unflip':
+            bot.send_message(chat_id, '┬──┬ ¯\\_(՞▃՞ ¯\\_)')
+
+        elif command == '/price':
+            """ Price, 24 hour %, 24 hour volume, and market cap """
+            symbol = 'OGN'
+            if symbol not in self.cached_prices:
+                self.cached_prices[symbol] = TokenData(symbol)
+            pdata = self.cached_prices[symbol]
+            message = """
+*Origin Token* (OGN)
+*USD Price*: {} ({}%)
+*BTC Price*: {} ({}%)
+*Market Cap*: {}
+*Volume(24h)*: {}
+
+@{}""".format(
+                monetary_format(pdata.price, decimals=5),
+                pdata.percent_change,
+                btc_format(pdata.btc_price),
+                pdata.btc_percent_change,
+                monetary_format(pdata.market_cap),
+                monetary_format(pdata.volume),
+                update.effective_user.username,
+            )
+
+            # If the last message we sent was price, delete it to reduce spam
+            if (
+                self.last_message_out
+                and self.last_message_out.get('type') == 'price'
+                and self.last_message_out['message'].message_id
+            ):
+                try:
+                    bot.delete_message(
+                        chat_id,
+                        self.last_message_out['message'].message_id
+                    )
+                except Exception as err:
+                    print('Unable to delete previous price message: ', err)
+                    print(traceback.format_exc())
+
+            self.last_message_out = {
+                'type': 'price',
+                'message': bot.send_message(
+                    chat_id,
+                    message,
+                    parse_mode=telegram.ParseMode.MARKDOWN
+                ),
+            }
+
+            # Delete the command message as well
+            try:
+                bot.delete_message(chat_id, message_id)
+            except Exception:
+                # nbd if we cannot delete it
+                pass
 
     def error(self, bot, update, error):
         """ Log Errors caused by Updates. """
@@ -357,6 +710,15 @@ class TelegramMonitorBot:
 
         # on different commands - answer in Telegram
 
+        # on commands
+        dp.add_handler(
+            CommandHandler(
+                command=self.available_commands,
+                callback=self.handle_command,
+                filters=Filters.all,
+            )
+        )
+
         # on noncommand i.e message - echo the message on Telegram
         dp.add_handler(MessageHandler(
             Filters.all,

model.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, DateTime, BigInteger, String, Integer, ForeignKey, func
+from sqlalchemy import Column, DateTime, BigInteger, String, Integer, Numeric, ForeignKey, func
 from sqlalchemy.orm import relationship, backref
 from sqlalchemy.ext.declarative import declarative_base
 import os

requirements.txt

@@ -1,7 +1,9 @@
-psycopg2==2.7.3.2
+psycopg2==2.8.4
 python-telegram-bot==9.0.0
-SQLAlchemy==1.2.2
+SQLAlchemy==1.3.13
 configparser==3.5.0
 Unidecode==1.0.22
 googletrans==2.4.0
-textblob
+textblob==0.15.3
+ipython==5.5.0
+requests>=2.23.0

runtime.txt

@@ -0,0 +1 @@
+python-3.9.2