README.md

@@ -5,11 +5,10 @@ Head to https://www.originprotocol.com/developers to learn more about what we're
 # Telegram Bot
 
 - Deletes messages matching specified patterns
-- Bans users for posting messages matching specified patterns
+- Bans users for posting messagses matching specified patterns
 - Bans users with usernames matching specified patterns
-- Records logs of conversations
+- Records logs of converstations
 - Logs an English translation of any foreign languages using Google Translate
-- Uses textblob for basic sentiment analysis of both polarity and subjectivity
 
 ## Installation
 
@@ -53,23 +52,6 @@ export TELEGRAM_BOT_TOKEN="4813829027:ADJFKAf0plousH2EZ2jBfxxRWFld3oK34ya"
 - `DEBUG` : If set to anything except `false`, will put bot into debug mode. This means that all actions will be logged into the chat itself, and more things will be logged.
 - `ADMIN_EXEMPT` : If set to anything except `false`, admin users will be exempt from monitoring. Reccomended to be set, but useful to turn off for debugging.
 - `NOTIFY_CHAT` : ID of chat to report actions. Can be useful if you have an admin-only chat where you want to monitor the bot's activity. E.g. `-140532994`
-- `CMC_API_KEY`: If you want the `/price` bot command to work, make sure to set a CoinMarketcap API key
-
-## Download the corpus for Textblob
-
-For sentiment analysis to work, you'll need to download the latest corpus file for textblob. You can do this by running:
-
-```
-python -m textblob.download_corpora
-```
-
-If you're running the bot on Heroku, set an environment variable named `NLTK_DATA` to `/app/nltk_data` by running:
-
-```
-heroku config:set NLTK_DATA='/app/nltk_data'
-```
-
-## Message ban patterns
 
 Sample bash file to set `MESSAGE_BAN_PATTERNS`:
 
@@ -84,7 +66,7 @@ read -r -d '' MESSAGE_BAN_PATTERNS << 'EOF'
 EOF
 ```
 
-## Attachments
+## Attachements
 
 By default, any attachments other than images or animations will cause the message to be hidden.
 

bot.py

@@ -9,203 +9,16 @@ This bot logs all messages sent in a Telegram Group to a database.
 
 """
 
-#from __future__ import print_function
+from __future__ import print_function
-import os
 import sys
+from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
+import os
+from model import User, Message, MessageHide, UserBan, session
+from time import strftime
 import re
 import unidecode
-import locale
-import traceback
-from time import strftime
-from datetime import datetime, timedelta
-
-import requests
-import telegram
-from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
-from model import User, Message, MessageHide, UserBan, session
 from mwt import MWT
 from googletrans import Translator
-from textblob import TextBlob
-
-# Used with monetary formatting
-locale.setlocale(locale.LC_ALL, '')
-
-# Price data cache duration
-CACHE_DURATION = timedelta(minutes=15)
-
-# CMC IDs can be retrived at:
-# https://pro-api.coinmarketcap.com/v1/cryptocurrency/map?symbol=[SYMBOL]
-CMC_SYMBOL_TO_ID = {
-    'OGN': 5117,
-    'USDT': 825,
-    'USDC': 3408,
-    'DAI': 4943,
-}
-CMC_API_KEY = os.environ.get('CMC_API_KEY')
-CMC_USD_QUOTE_URL = 'https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest?id={}'
-CMC_BTC_QUOTE_URL = 'https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest?id={}&convert=BTC'
-
-
-def first_of(attr, match, it):
-    """ Return the first item in a set with an attribute that matches match """
-    if it is not None:
-        for i in it:
-            try:
-                if getattr(i, attr) == match:
-                    return i
-            except: pass
-
-    return None
-
-
-def command_from_message(message, default=None):
-    """ Extracts the first command from a Telegram Message """
-    if not message or not message.text:
-        return default
-
-    command = None
-    text = message.text
-    entities = message.entities
-    command_def = first_of('type', 'bot_command', entities)
-
-    if command_def:
-        command = text[command_def.offset:command_def.length]
-
-    return command or default
-
-
-def cmc_get_data(jso, cmc_id, pair_symbol='USD'):
-    """ Pull relevant data from a response object """
-    if not jso:
-        return None
-
-    data = jso.get('data', {})
-    specific_data = data.get(str(cmc_id), {})
-    quote = specific_data.get('quote', {})
-    symbol_data = quote.get(pair_symbol, {})
-    return {
-        'price': symbol_data.get('price'),
-        'volume': symbol_data.get('volume_24h'),
-        'percent_change': symbol_data.get('percent_change_24h'),
-        'market_cap': symbol_data.get('market_cap'),
-    }
-
-
-def decimal_format(v, decimals=2):
-    if not v:
-        v = 0
-    f = locale.format_string('%.{}f'.format(decimals), v, grouping=True)
-    return '{}'.format(f)
-
-
-def monetary_format(v, decimals=2):
-    return '${}'.format(decimal_format(v, decimals))
-
-def btc_format(v):
-    return decimal_format(v, decimals=8)
-
-
-
-class TokenData:
-    def __init__(self, symbol, price=None, stamp=datetime.now()):
-        self.symbol = symbol
-        self._price = price
-        self._btc_price = 0
-        self._percent_change = 0
-        self._btc_percent_change = 0
-        self._volume = 0
-        self._market_cap = 0
-        if price is not None:
-            self.stamp = stamp
-        else:
-            self.stamp = None
-
-    def _fetch_from_cmc(self, url_template):
-        """ Get quote data for a specific known symbol """
-        jso = None
-
-        cmc_id = CMC_SYMBOL_TO_ID.get(self.symbol)
-        url = url_template.format(cmc_id)
-        r = requests.get(url, headers={
-            'X-CMC_PRO_API_KEY': CMC_API_KEY,
-            'Accept': 'application/json',
-        })
-        if r.status_code != 200:
-            print('Failed to fetch price data for id: {}'.format(cmc_id))
-            return None
-        try:
-            jso = r.json()
-        except Exception:
-            print('Error parsing JSON')
-            return None
-        return jso
-
-    def update(self):
-        """ Fetch price from binance """
-        jso = None
-        data = None
-        if self.stamp is None or (
-            self.stamp is not None
-            and self.stamp < datetime.now() - CACHE_DURATION
-        ):
-            # CMC USD
-            try:
-                jso = self._fetch_from_cmc(CMC_USD_QUOTE_URL)
-                data = cmc_get_data(jso, CMC_SYMBOL_TO_ID[self.symbol])
-            except Exception as err:
-                print('Error fetching data: ', str(err))
-                print(traceback.format_exc())
-            if data is not None:
-                self._price = data.get('price')
-                self._percent_change = data.get('percent_change')
-                self._volume = data.get('volume')
-                self._market_cap = data.get('market_cap')
-                self.stamp = datetime.now()
-
-            # CMC BTC
-            try:
-                jso = self._fetch_from_cmc(CMC_BTC_QUOTE_URL)
-                data = cmc_get_data(jso, CMC_SYMBOL_TO_ID[self.symbol], 'BTC')
-            except Exception as err:
-                print('Error fetching data: ', str(err))
-                print(traceback.format_exc())
-            if data is not None:
-                self._btc_price = data.get('price')
-                self._btc_percent_change = data.get('percent_change')
-
-    @property
-    def price(self):
-        self.update()
-        return self._price
-
-    @property
-    def btc_price(self):
-        self.update()
-        return self._btc_price
-
-    @property
-    def btc_percent_change(self):
-        self.update()
-        return self._btc_percent_change
-
-    @property
-    def volume(self):
-        self.update()
-        return self._volume
-
-    @property
-    def percent_change(self):
-        self.update()
-        pc = str(self._percent_change)
-        if pc and not pc.startswith('-'):
-            pc = '+{}'.format(pc)
-        return pc
-
-    @property
-    def market_cap(self):
-        self.update()
-        return self._market_cap
-
 
 class TelegramMonitorBot:
 
@@ -229,17 +42,10 @@ class TelegramMonitorBot:
             print("🔵 MESSAGE_BAN_PATTERNS:\n", os.environ['MESSAGE_BAN_PATTERNS'])
             print("🔵 MESSAGE_HIDE_PATTERNS:\n", os.environ['MESSAGE_HIDE_PATTERNS'])
             print("🔵 NAME_BAN_PATTERNS:\n", os.environ['NAME_BAN_PATTERNS'])
-            print("🔵 IGNORE_USER_IDS:\n", os.environ.get('IGNORE_USER_IDS'))
 
         # Channel to notify of violoations, e.g. '@channelname'
         self.notify_chat = os.environ['NOTIFY_CHAT'] if 'NOTIFY_CHAT' in os.environ else None
 
-        # Ignore these user IDs
-        if not os.environ.get('IGNORE_USER_IDS'):
-            self.ignore_user_ids = []
-        else:
-            self.ignore_user_ids = list(map(int, os.environ['IGNORE_USER_IDS'].split(',')))
-
         # List of chat ids that bot should monitor
         self.chat_ids = (
             list(map(int, os.environ['CHAT_IDS'].split(',')))
@@ -266,22 +72,6 @@ class TelegramMonitorBot:
             re.IGNORECASE | re.VERBOSE)
             if self.name_ban_patterns else None)
 
-        # Mime type document check
-        # NOTE: All gifs appear to be converted to video/mp4
-        mime_types = os.environ.get('ALLOWED_MIME_TYPES', 'video/mp4')
-        self.allowed_mime_types = set(map(lambda s: s.strip(), mime_types.split(',')))
-
-        # Comamnds
-        self.available_commands = ['flip', 'unflip']
-        if CMC_API_KEY is not None:
-            self.available_commands.append('price')
-
-        print('Available commands: {}'.format(', '.join(self.available_commands)))
-
-        # Cached token prices
-        self.cached_prices = {}
-        self.last_message_out = None
-
 
     @MWT(timeout=60*60)
     def get_admin_ids(self, bot, chat_id):
@@ -317,8 +107,7 @@ class TelegramMonitorBot:
             s.commit()
             s.close()
             # Notify channel
-            if self.notify_chat:
+            bot.sendMessage(chat_id=self.notify_chat, text=log_message)
-                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
         if self.name_ban_re and self.name_ban_re.search(update.message.from_user.username or ''):
             # Logging
@@ -337,8 +126,7 @@ class TelegramMonitorBot:
             s.commit()
             s.close()
             # Notify channel
-            if self.notify_chat:
+            bot.sendMessage(chat_id=self.notify_chat, text=log_message)
-                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
 
     def security_check_message(self, bot, update):
@@ -370,8 +158,7 @@ class TelegramMonitorBot:
             s.commit()
             s.close()
             # Notify channel
-            if self.notify_chat:
+            bot.sendMessage(chat_id=self.notify_chat, text=log_message)
-                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
         if self.message_ban_re and self.message_ban_re.search(message):
             # Logging
@@ -392,8 +179,7 @@ class TelegramMonitorBot:
             s.commit()
             s.close()
             # Notify channel
-            if self.notify_chat:
+            bot.sendMessage(chat_id=self.notify_chat, text=log_message)
-                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
         elif self.message_hide_re and self.message_hide_re.search(message):
             # Logging
@@ -412,8 +198,7 @@ class TelegramMonitorBot:
             s.commit()
             s.close()
             # Notify channel
-            if self.notify_chat:
+            bot.sendMessage(chat_id=self.notify_chat, text=log_message)
-                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
 
     def attachment_check(self, bot, update):
@@ -424,10 +209,6 @@ class TelegramMonitorBot:
             update.message.voice):
             # Logging
             if update.message.document:
-                # GIFs are documents and allowed
-                mime_type = update.message.document.mime_type
-                if mime_type and mime_type in self.allowed_mime_types:
-                    return
                 log_message = "❌ HIDE DOCUMENT: {}".format(update.message.document.__dict__)
             else:
                 log_message = "❌ HIDE NON-DOCUMENT ATTACHMENT"
@@ -445,55 +226,25 @@ class TelegramMonitorBot:
             s.commit()
             s.close()
             # Notify channel
-            if self.notify_chat:
+            bot.sendMessage(chat_id=self.notify_chat, text=log_message)
-                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
 
     def logger(self, bot, update):
-        """ Primary Logger. Handles incoming bot messages and saves them to DB
+        """ Primary Logger. Handles incoming bot messages and saves them to DB """
-
-        :param bot: telegram.Bot https://python-telegram-bot.readthedocs.io/en/stable/telegram.bot.html
-        :param update: telegram.Update https://python-telegram-bot.readthedocs.io/en/stable/telegram.update.html
-        """
-
-        if (
-            update.effective_user is None
-            or update.effective_user.id in self.ignore_user_ids
-        ):
-            print("{}: Ignoring update.".format(update.update_id))
-            return
-
         try:
-
+            user = update.message.from_user
-            message = update.message
-
-            # message is optional
-            if message is None:
-
-                if update.effective_message is None:
-                    print("No message included in update")
-                    return
-
-                message = update.effective_message
-
-            if message:
-
-                user = message.from_user
 
             # Limit bot to monitoring certain chats
-                if message.chat_id not in self.chat_ids:
+            if update.message.chat_id not in self.chat_ids:
-                    from_user = "UNKNOWN"
-                    if user:
-                        from_user = user.id
                 print("Message from user {} is from chat_id not being monitored: {}".format(
-                        from_user,
+                    user.id,
-                        message.chat_id)
+                    update.message.chat_id)
                 )
                 return
 
             if self.id_exists(user.id):
-                    self.log_message(user.id, message.text,
+                self.log_message(user.id, update.message.text,
-                                     message.chat_id)
+                                 update.message.chat_id)
             else:
                 add_user_success = self.add_user(
                     user.id,
@@ -503,7 +254,7 @@ class TelegramMonitorBot:
 
                 if add_user_success:
                     self.log_message(
-                            user.id, message.text, message.chat_id)
+                        user.id, update.message.text, update.message.chat_id)
                     print("User added: {}".format(user.id))
                 else:
                     print("Something went wrong adding the user {}".format(user.id), file=sys.stderr)
@@ -512,7 +263,7 @@ class TelegramMonitorBot:
                 user.username or
                 "{} {}".format(user.first_name, user.last_name) or
                 "<none>").encode('utf-8')
-                if message.text:
+            if update.message.text:
                 print("{} {} ({}) : {}".format(
                     strftime("%Y-%m-%dT%H:%M:%S"),
                     user.id,
@@ -526,14 +277,8 @@ class TelegramMonitorBot:
                     user_name)
                 )
 
-            else:
-                print("Update and user not logged because no message was found")
-
             # Don't check admin activity
-            is_admin = False
+            is_admin = update.message.from_user.id in self.get_admin_ids(bot, update.message.chat_id)
-            if message:
-                is_admin = message.from_user.id in self.get_admin_ids(bot, message.chat_id)
-
             if is_admin and self.admin_exempt:
                 print("👮‍♂️ Skipping checks. User is admin: {}".format(user.id))
             else:
@@ -543,8 +288,7 @@ class TelegramMonitorBot:
                 self.security_check_message(bot, update)
 
         except Exception as e:
-            print("Error[521]: {}".format(e))
+            print("Error: {}".format(e))
-            print(traceback.format_exc())
             print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
 
     # DB queries
@@ -559,36 +303,25 @@ class TelegramMonitorBot:
 
         return bool_set
 
+
     def log_message(self, user_id, user_message, chat_id):
-
-        if user_message is None:
-            user_message = "[NO MESSAGE]"
-
         try:
             s = session()
             language_code = english_message = ""
-            polarity = subjectivity = 0.0
             try:
-                # translate to English & log the original language
                 translator = Translator()
                 translated = translator.translate(user_message)
                 language_code = translated.src
                 english_message = translated.text
-                # run basic sentiment analysis on the translated English string
-                analysis = TextBlob(english_message)
-                polarity = analysis.sentiment.polarity
-                subjectivity = analysis.sentiment.subjectivity
             except Exception as e:
-                print("Error translating message: {}".format(e))
+                print(e.message)
-            msg1 = Message(user_id=user_id, message=user_message, chat_id=chat_id, 
+            msg1 = Message(user_id=user_id, message=user_message,
-                language_code=language_code, english_message=english_message, polarity=polarity,
+                           chat_id=chat_id, language_code=language_code, english_message=english_message)
-                subjectivity=subjectivity)
             s.add(msg1)
             s.commit()
             s.close()
         except Exception as e:
-            print("Error logging message: {}".format(e))
+            print("Error: {}".format(e))
-            print(traceback.format_exc())
 
 
     def add_user(self, user_id, first_name, last_name, username):
@@ -604,94 +337,8 @@ class TelegramMonitorBot:
             s.close()
             return self.id_exists(user_id)
         except Exception as e:
-            print("Error[347]: {}".format(e))
+            print("Error: {}".format(e))
-            print(traceback.format_exc())
 
-    def handle_command(self, bot, update):
-        """ Handles commands
-
-        Note: Args reversed from docs?  Maybe version differences?  Docs say
-        cb(update, context) but we're getting cb(bot, update).
-
-        update: Update: https://python-telegram-bot.readthedocs.io/en/stable/telegram.update.html#telegram.Update
-        context: CallbackContext: https://python-telegram-bot.readthedocs.io/en/stable/telegram.ext.callbackcontext.html
-
-        hi: says hi
-        price: prints the OGN price
-        """
-        chat_id = None
-        command = None
-        message_id = update.effective_message.message_id
-
-        command = command_from_message(update.effective_message)
-
-        if update.effective_message.chat:
-            chat_id = update.effective_message.chat.id
-
-        print('command: {} seen in chat_id {}'.format(command, chat_id))
-
-        if command == '/hi':
-            bot.send_message(chat_id, 'Yo whattup, @{}!'.format(update.effective_user.username))
-
-        elif command == '/flip':
-            bot.send_message(chat_id, '╯°□°）╯︵ ┻━┻')
-
-        elif command == '/unflip':
-            bot.send_message(chat_id, '┬──┬ ¯\\_(՞▃՞ ¯\\_)')
-
-        elif command == '/price':
-            """ Price, 24 hour %, 24 hour volume, and market cap """
-            symbol = 'OGN'
-            if symbol not in self.cached_prices:
-                self.cached_prices[symbol] = TokenData(symbol)
-            pdata = self.cached_prices[symbol]
-            message = """
-*Origin Token* (OGN)
-*USD Price*: {} ({}%)
-*BTC Price*: {} ({}%)
-*Market Cap*: {}
-*Volume(24h)*: {}
-
-@{}""".format(
-                monetary_format(pdata.price, decimals=5),
-                pdata.percent_change,
-                btc_format(pdata.btc_price),
-                pdata.btc_percent_change,
-                monetary_format(pdata.market_cap),
-                monetary_format(pdata.volume),
-                update.effective_user.username,
-            )
-
-            # If the last message we sent was price, delete it to reduce spam
-            if (
-                self.last_message_out
-                and self.last_message_out.get('type') == 'price'
-                and self.last_message_out['message'].message_id
-            ):
-                try:
-                    bot.delete_message(
-                        chat_id,
-                        self.last_message_out['message'].message_id
-                    )
-                except Exception as err:
-                    print('Unable to delete previous price message: ', err)
-                    print(traceback.format_exc())
-
-            self.last_message_out = {
-                'type': 'price',
-                'message': bot.send_message(
-                    chat_id,
-                    message,
-                    parse_mode=telegram.ParseMode.MARKDOWN
-                ),
-            }
-
-            # Delete the command message as well
-            try:
-                bot.delete_message(chat_id, message_id)
-            except Exception:
-                # nbd if we cannot delete it
-                pass
 
     def error(self, bot, update, error):
         """ Log Errors caused by Updates. """
@@ -710,15 +357,6 @@ class TelegramMonitorBot:
 
         # on different commands - answer in Telegram
 
-        # on commands
-        dp.add_handler(
-            CommandHandler(
-                command=self.available_commands,
-                callback=self.handle_command,
-                filters=Filters.all,
-            )
-        )
-
         # on noncommand i.e message - echo the message on Telegram
         dp.add_handler(MessageHandler(
             Filters.all,

model.py

@@ -1,4 +1,4 @@
-from sqlalchemy import Column, DateTime, BigInteger, String, Integer, Numeric, ForeignKey, func
+from sqlalchemy import Column, DateTime, BigInteger, String, Integer, ForeignKey, func
 from sqlalchemy.orm import relationship, backref
 from sqlalchemy.ext.declarative import declarative_base
 import os

requirements.txt

@@ -1,9 +1,7 @@
-psycopg2==2.8.4
+psycopg2==2.7.3.2
 python-telegram-bot==9.0.0
-SQLAlchemy==1.3.13
+SQLAlchemy==1.2.2
 configparser==3.5.0
 Unidecode==1.0.22
 googletrans==2.4.0
-textblob==0.15.3
+textblob
-ipython==5.5.0
-requests>=2.23.0

runtime.txt

@@ -1 +0,0 @@
-python-3.9.2