adds runtime.txt

Allow gifs in channel

.gitignore

@@ -1,6 +1,9 @@
 # Never commit config data
 config.cnf
 
+# Env vars for "real" installation
+env.sh
+
 # Byte-compiled / optimized / DLL files
 *.py[cod]
 

README.md

@@ -1,46 +1,78 @@
+![origin_github_banner](https://user-images.githubusercontent.com/673455/37314301-f8db9a90-2618-11e8-8fee-b44f38febf38.png)
+
+Head to https://www.originprotocol.com/developers to learn more about what we're building and how to get involved.
+
 # Telegram Bot
 
 - Deletes messages matching specified patterns
-- Bans users for posting messagses matching specified patterns
+- Bans users for posting messages matching specified patterns
 - Bans users with usernames matching specified patterns
-- Records logs of converstations
+- Records logs of conversations
+- Logs an English translation of any foreign languages using Google Translate
+- Uses textblob for basic sentiment analysis of both polarity and subjectivity
 
 ## Installation
 
- - Required: Python 3.x, pip, PostgreSQL
+- Required: Python 3.x, pip, PostgreSQL
- - Create virtualenv
+- Create virtualenv
- - Clone this repo
+- Clone this repo
- - `pip install --upgrade -r requirements.txt`
+- `pip install --upgrade -r requirements.txt`
 
 ## Database setup
- - Store database URL in environment variable.
+
- ```
+- Store database URL in environment variable.
- export TELEGRAM_BOT_POSTGRES_URL="postgresql://<user>:<password>@localhost:5432/<databasename>"
+
- ```
+```
- - Run: `python model.py` to setup the DB tables.
+export TELEGRAM_BOT_POSTGRES_URL="postgresql://<user>:<password>@localhost:5432/<databasename>"
+```
+
+- Run: `python model.py` to setup the DB tables.
 
 ## Setup
 
- - Create a Telegram bot by talking to `@BotFather` : https://core.telegram.org/bots#creating-a-new-bot
+- Create a Telegram bot by talking to `@BotFather` : https://core.telegram.org/bots#creating-a-new-bot
- - Use `/setprivacy` with `@BotFather` in order to allow it to see all messages in a group.
+- Use `/setprivacy` with `@BotFather` in order to allow it to see all messages in a group.
- - Store your Telegram Bot Token in environment variable `TELEGRAM_BOT_TOKEN`. It will look similar to this:
+- Store your Telegram Bot Token in environment variable `TELEGRAM_BOT_TOKEN`. It will look similar to this:
 
- ```
+```
- export TELEGRAM_BOT_TOKEN="4813829027:ADJFKAf0plousH2EZ2jBfxxRWFld3oK34ya"
+export TELEGRAM_BOT_TOKEN="4813829027:ADJFKAf0plousH2EZ2jBfxxRWFld3oK34ya"
- ```
+```
- - Create your Telegram group.
- - Add your bot to the group like so: https://stackoverflow.com/questions/37338101/how-to-add-a-bot-to-a-telegram-group
- - Make your bot an admin in the group
 
-## Configuring patterns
+- Create your Telegram group.
+- Add your bot to the group like so: https://stackoverflow.com/questions/37338101/how-to-add-a-bot-to-a-telegram-group
+- Make your bot an admin in the group
 
-- Regex patterns will be read from the following env variables
+## Configuration with ENV vars
-	- `MESSAGE_BAN_PATTERNS` Messages matching this will ban the user.
+
-	- `MESSAGE_HIDE_PATTERNS` Messages matching this will be hidden/deleted
+- `MESSAGE_BAN_PATTERNS` : **REQUIRED** Regex pattern. Messages matching this will ban the user.
-	- `NAME_BAN_PATTERNS` Users with usernames or first/last names maching this will be banned from the group.
+- `MESSAGE_HIDE_PATTERNS` : **REQUIRED** Regex pattern. Messages matching this will be hidden/deleted
-	- `SAFE_USER_IDS` User ID's that are except from these checkes. Note that the bot cannot ban admin users, but can delete their messages.
+- `NAME_BAN_PATTERNS` **REQUIRED** Regex pattern. Users with usernames or first/last names maching this will be banned from the group.
+- `CHAT_IDS` : **REQUIRED**. Comma-seperated list of IDs of chat(s) that should be monitored. To find out the ID of a chat, add the bot to a chat and type some messages there. The bot log will report an error that it got messages `from chat_id not being monitored: XXX` where XXX is the chat ID. e.g. `-240532994,-150531679`
+- `TELEGRAM_BOT_TOKEN` : **REQUIRED**. Token for bot to control. e.g. `4813829027:ADJFKAf0plousH2EZ2jBfxxRWFld3oK34ya`
+- `TELEGRAM_BOT_POSTGRES_URL` : **REQUIRED**. URI for postgres instance to log activity to. e.g. `postgresql://localhost/postgres`
+- `DEBUG` : If set to anything except `false`, will put bot into debug mode. This means that all actions will be logged into the chat itself, and more things will be logged.
+- `ADMIN_EXEMPT` : If set to anything except `false`, admin users will be exempt from monitoring. Reccomended to be set, but useful to turn off for debugging.
+- `NOTIFY_CHAT` : ID of chat to report actions. Can be useful if you have an admin-only chat where you want to monitor the bot's activity. E.g. `-140532994`
+- `CMC_API_KEY`: If you want the `/price` bot command to work, make sure to set a CoinMarketcap API key
+
+## Download the corpus for Textblob
+
+For sentiment analysis to work, you'll need to download the latest corpus file for textblob. You can do this by running:
+
+```
+python -m textblob.download_corpora
+```
+
+If you're running the bot on Heroku, set an environment variable named `NLTK_DATA` to `/app/nltk_data` by running:
+
+```
+heroku config:set NLTK_DATA='/app/nltk_data'
+```
+
+## Message ban patterns
 
 Sample bash file to set `MESSAGE_BAN_PATTERNS`:
+
 ```
 read -r -d '' MESSAGE_BAN_PATTERNS << 'EOF'
 # ETH Address
@@ -52,11 +84,17 @@ read -r -d '' MESSAGE_BAN_PATTERNS << 'EOF'
 EOF
 ```
 
+## Attachments
+
+By default, any attachments other than images or animations will cause the message to be hidden.
+
 ## Running
 
 ### Locally
- - Run: `python bot.py` to start logger
+
- - Messages will be displayed on `stdout` as they are logged.
+- Run: `python bot.py` to start logger
+- Messages will be displayed on `stdout` as they are logged.
 
 ### On Heroku
- - You must enable the worker on Heroku app dashboard. (By default it is off.)
+
+- You must enable the worker on Heroku app dashboard. (By default it is off.)

bin/install_textblob_corpora

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+source $BIN_DIR/utils
+
+echo "-----> Starting corpora installation"
+
+# Assumes NLTK_DATA environment variable is already set
+# $ heroku config:set NLTK_DATA='/app/nltk_data'
+
+# Install the default corpora to NLTK_DATA directory
+python -m textblob.download_corpora
+
+# Open the NLTK_DATA directory
+cd ${NLTK_DATA}
+
+# Delete all of the zip files in the NLTK DATA directory
+find . -name "*.zip" -type f -delete
+
+echo "-----> Finished corpora installatio"

bin/post_compile

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+if [ -f bin/install_textblob_corpora ]; then
+    echo "-----> Running install_textblob_corpora"
+    chmod +x bin/install_textblob_corpora
+    bin/install_textblob_corpora
+fi
+
+echo "-----> Post-compile done"

bot.py

@@ -9,26 +9,236 @@ This bot logs all messages sent in a Telegram Group to a database.
 
 """
 
-from __future__ import print_function
+#from __future__ import print_function
-import sys
-from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
 import os
-from model import User, Message, MessageHide, UserBan, session
+import sys
-from time import strftime
 import re
 import unidecode
+import locale
+import traceback
+from time import strftime
+from datetime import datetime, timedelta
+
+import requests
+import telegram
+from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
+from model import User, Message, MessageHide, UserBan, session
 from mwt import MWT
+from googletrans import Translator
+from textblob import TextBlob
+
+# Used with monetary formatting
+locale.setlocale(locale.LC_ALL, '')
+
+# Price data cache duration
+CACHE_DURATION = timedelta(minutes=15)
+
+# CMC IDs can be retrived at:
+# https://pro-api.coinmarketcap.com/v1/cryptocurrency/map?symbol=[SYMBOL]
+CMC_SYMBOL_TO_ID = {
+    'OGN': 5117,
+    'USDT': 825,
+    'USDC': 3408,
+    'DAI': 4943,
+}
+CMC_API_KEY = os.environ.get('CMC_API_KEY')
+CMC_USD_QUOTE_URL = 'https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest?id={}'
+CMC_BTC_QUOTE_URL = 'https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest?id={}&convert=BTC'
+
+
+def first_of(attr, match, it):
+    """ Return the first item in a set with an attribute that matches match """
+    if it is not None:
+        for i in it:
+            try:
+                if getattr(i, attr) == match:
+                    return i
+            except: pass
+
+    return None
+
+
+def command_from_message(message, default=None):
+    """ Extracts the first command from a Telegram Message """
+    if not message or not message.text:
+        return default
+
+    command = None
+    text = message.text
+    entities = message.entities
+    command_def = first_of('type', 'bot_command', entities)
+
+    if command_def:
+        command = text[command_def.offset:command_def.length]
+
+    return command or default
+
+
+def cmc_get_data(jso, cmc_id, pair_symbol='USD'):
+    """ Pull relevant data from a response object """
+    if not jso:
+        return None
+
+    data = jso.get('data', {})
+    specific_data = data.get(str(cmc_id), {})
+    quote = specific_data.get('quote', {})
+    symbol_data = quote.get(pair_symbol, {})
+    return {
+        'price': symbol_data.get('price'),
+        'volume': symbol_data.get('volume_24h'),
+        'percent_change': symbol_data.get('percent_change_24h'),
+        'market_cap': symbol_data.get('market_cap'),
+    }
+
+
+def decimal_format(v, decimals=2):
+    if not v:
+        v = 0
+    f = locale.format_string('%.{}f'.format(decimals), v, grouping=True)
+    return '{}'.format(f)
+
+
+def monetary_format(v, decimals=2):
+    return '${}'.format(decimal_format(v, decimals))
+
+def btc_format(v):
+    return decimal_format(v, decimals=8)
+
+
+
+class TokenData:
+    def __init__(self, symbol, price=None, stamp=datetime.now()):
+        self.symbol = symbol
+        self._price = price
+        self._btc_price = 0
+        self._percent_change = 0
+        self._btc_percent_change = 0
+        self._volume = 0
+        self._market_cap = 0
+        if price is not None:
+            self.stamp = stamp
+        else:
+            self.stamp = None
+
+    def _fetch_from_cmc(self, url_template):
+        """ Get quote data for a specific known symbol """
+        jso = None
+
+        cmc_id = CMC_SYMBOL_TO_ID.get(self.symbol)
+        url = url_template.format(cmc_id)
+        r = requests.get(url, headers={
+            'X-CMC_PRO_API_KEY': CMC_API_KEY,
+            'Accept': 'application/json',
+        })
+        if r.status_code != 200:
+            print('Failed to fetch price data for id: {}'.format(cmc_id))
+            return None
+        try:
+            jso = r.json()
+        except Exception:
+            print('Error parsing JSON')
+            return None
+        return jso
+
+    def update(self):
+        """ Fetch price from binance """
+        jso = None
+        data = None
+        if self.stamp is None or (
+            self.stamp is not None
+            and self.stamp < datetime.now() - CACHE_DURATION
+        ):
+            # CMC USD
+            try:
+                jso = self._fetch_from_cmc(CMC_USD_QUOTE_URL)
+                data = cmc_get_data(jso, CMC_SYMBOL_TO_ID[self.symbol])
+            except Exception as err:
+                print('Error fetching data: ', str(err))
+                print(traceback.format_exc())
+            if data is not None:
+                self._price = data.get('price')
+                self._percent_change = data.get('percent_change')
+                self._volume = data.get('volume')
+                self._market_cap = data.get('market_cap')
+                self.stamp = datetime.now()
+
+            # CMC BTC
+            try:
+                jso = self._fetch_from_cmc(CMC_BTC_QUOTE_URL)
+                data = cmc_get_data(jso, CMC_SYMBOL_TO_ID[self.symbol], 'BTC')
+            except Exception as err:
+                print('Error fetching data: ', str(err))
+                print(traceback.format_exc())
+            if data is not None:
+                self._btc_price = data.get('price')
+                self._btc_percent_change = data.get('percent_change')
+
+    @property
+    def price(self):
+        self.update()
+        return self._price
+
+    @property
+    def btc_price(self):
+        self.update()
+        return self._btc_price
+
+    @property
+    def btc_percent_change(self):
+        self.update()
+        return self._btc_percent_change
+
+    @property
+    def volume(self):
+        self.update()
+        return self._volume
+
+    @property
+    def percent_change(self):
+        self.update()
+        pc = str(self._percent_change)
+        if pc and not pc.startswith('-'):
+            pc = '+{}'.format(pc)
+        return pc
+
+    @property
+    def market_cap(self):
+        self.update()
+        return self._market_cap
+
 
 class TelegramMonitorBot:
 
 
     def __init__(self):
-        self.debug = os.environ.get('DEBUG') is not None
+        self.debug = (
+            (os.environ.get('DEBUG') is not None) and
+            (os.environ.get('DEBUG').lower() != "false"))
 
-        # Users to notify of violoations
+        # Are admins exempt from having messages checked?
-        self.notify_user_ids = (
+        self.admin_exempt = (
-            list(map(int, os.environ['NOTIFY_USER_IDS'].split(',')))
+            (os.environ.get('ADMIN_EXEMPT') is not None) and
-            if "NOTIFY_USER_IDS" in os.environ else [])
+            (os.environ.get('ADMIN_EXEMPT').lower() != "false"))
+
+        if (self.debug):
+            print("🔵 debug:", self.debug)
+            print("🔵 admin_exempt:", self.admin_exempt)
+            print("🔵 TELEGRAM_BOT_POSTGRES_URL:", os.environ["TELEGRAM_BOT_POSTGRES_URL"])
+            print("🔵 TELEGRAM_BOT_TOKEN:", os.environ["TELEGRAM_BOT_TOKEN"])
+            print("🔵 NOTIFY_CHAT:", os.environ['NOTIFY_CHAT'] if 'NOTIFY_CHAT' in os.environ else "<undefined>")
+            print("🔵 MESSAGE_BAN_PATTERNS:\n", os.environ['MESSAGE_BAN_PATTERNS'])
+            print("🔵 MESSAGE_HIDE_PATTERNS:\n", os.environ['MESSAGE_HIDE_PATTERNS'])
+            print("🔵 NAME_BAN_PATTERNS:\n", os.environ['NAME_BAN_PATTERNS'])
+            print("🔵 IGNORE_USER_IDS:\n", os.environ.get('IGNORE_USER_IDS'))
+
+        # Channel to notify of violoations, e.g. '@channelname'
+        self.notify_chat = os.environ['NOTIFY_CHAT'] if 'NOTIFY_CHAT' in os.environ else None
+
+        # Ignore these user IDs
+        if not os.environ.get('IGNORE_USER_IDS'):
+            self.ignore_user_ids = []
+        else:
+            self.ignore_user_ids = list(map(int, os.environ['IGNORE_USER_IDS'].split(',')))
 
         # List of chat ids that bot should monitor
         self.chat_ids = (
@@ -56,6 +266,22 @@ class TelegramMonitorBot:
             re.IGNORECASE | re.VERBOSE)
             if self.name_ban_patterns else None)
 
+        # Mime type document check
+        # NOTE: All gifs appear to be converted to video/mp4
+        mime_types = os.environ.get('ALLOWED_MIME_TYPES', 'video/mp4')
+        self.allowed_mime_types = set(map(lambda s: s.strip(), mime_types.split(',')))
+
+        # Comamnds
+        self.available_commands = ['flip', 'unflip']
+        if CMC_API_KEY is not None:
+            self.available_commands.append('price')
+
+        print('Available commands: {}'.format(', '.join(self.available_commands)))
+
+        # Cached token prices
+        self.cached_prices = {}
+        self.last_message_out = None
+
 
     @MWT(timeout=60*60)
     def get_admin_ids(self, bot, chat_id):
@@ -71,19 +297,15 @@ class TelegramMonitorBot:
     def security_check_username(self, bot, update):
         """ Test username for security violations """
 
-        full_name = (update.message.from_user.first_name + " "
+        full_name = "{} {}".format(
-            + update.message.from_user.last_name)
+            update.message.from_user.first_name,
+            update.message.from_user.last_name)
         if self.name_ban_re and self.name_ban_re.search(full_name):
             # Logging
-            log_message = "Ban match full name: {}".format(full_name.encode('utf-8'))
+            log_message = "❌ 🙅‍♂️ BAN MATCH FULL NAME: {}".format(full_name.encode('utf-8'))
             if self.debug:
                 update.message.reply_text(log_message)
             print(log_message)
-            for notify_user_id in self.notify_user_ids:
-                print (notify_user_id,"gets notified")
-                bot.send_message(
-                    chat_id=notify_user_id,
-                    text=log_message)
             # Ban the user
             self.ban_user(update)
             # Log in database
@@ -94,17 +316,16 @@ class TelegramMonitorBot:
             s.add(userBan)
             s.commit()
             s.close()
+            # Notify channel
+            if self.notify_chat:
+                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
         if self.name_ban_re and self.name_ban_re.search(update.message.from_user.username or ''):
             # Logging
-            log_message = "Ban match username: {}".format(update.message.from_user.username.encode('utf-8'))
+            log_message = "❌ 🙅‍♂️ BAN MATCH USERNAME: {}".format(update.message.from_user.username.encode('utf-8'))
             if self.debug:
                 update.message.reply_text(log_message)
             print(log_message)
-            for notify_user_id in self.notify_user_ids:
-                bot.send_message(
-                    chat_id=notify_user_id,
-                    text=log_message)
             # Ban the user
             self.ban_user(update)
             # Log in database
@@ -115,26 +336,49 @@ class TelegramMonitorBot:
             s.add(userBan)
             s.commit()
             s.close()
+            # Notify channel
+            if self.notify_chat:
+                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
 
     def security_check_message(self, bot, update):
         """ Test message for security violations """
 
+        if not update.message.text:
+            return
+
         # Remove accents from letters (é->e, ñ->n, etc...)
         message = unidecode.unidecode(update.message.text)
         # TODO: Replace lookalike unicode characters:
         # https://github.com/wanderingstan/Confusables
 
-        if self.message_ban_re and self.message_ban_re.search(message):
+        # Hide forwarded messages
+        if update.message.forward_date is not None:
             # Logging
-            log_message = "Ban message match: {}".format(update.message.text.encode('utf-8'))
+            log_message = "❌ HIDE FORWARDED: {}".format(update.message.text.encode('utf-8'))
+            if self.debug:
+                update.message.reply_text(log_message)
+            print(log_message)
+            # Delete the message
+            update.message.delete()
+            # Log in database
+            s = session()
+            messageHide = MessageHide(
+                user_id=update.message.from_user.id,
+                message=update.message.text)
+            s.add(messageHide)
+            s.commit()
+            s.close()
+            # Notify channel
+            if self.notify_chat:
+                bot.send_message(chat_id=self.notify_chat, text=log_message)
+
+        if self.message_ban_re and self.message_ban_re.search(message):
+            # Logging
+            log_message = "❌ 🙅‍♂️ BAN MATCH: {}".format(update.message.text.encode('utf-8'))
             if self.debug:
                 update.message.reply_text(log_message)
             print(log_message)
-            for notify_user_id in self.notify_user_ids:
-                bot.send_message(
-                    chat_id=notify_user_id,
-                    text=log_message)
             # Any message that causes a ban gets deleted
             update.message.delete()
             # Ban the user
@@ -147,17 +391,16 @@ class TelegramMonitorBot:
             s.add(userBan)
             s.commit()
             s.close()
+            # Notify channel
+            if self.notify_chat:
+                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
         elif self.message_hide_re and self.message_hide_re.search(message):
             # Logging
-            log_message = "Hide match: {}".format(update.message.text.encode('utf-8'))
+            log_message = "❌ 🙈 HIDE MATCH: {}".format(update.message.text.encode('utf-8'))
             if self.debug:
                 update.message.reply_text(log_message)
             print(log_message)
-            for notify_user_id in self.notify_user_ids:
-                bot.send_message(
-                    chat_id=notify_user_id,
-                    text=log_message)
             # Delete the message
             update.message.delete()
             # Log in database
@@ -168,55 +411,141 @@ class TelegramMonitorBot:
             s.add(messageHide)
             s.commit()
             s.close()
+            # Notify channel
+            if self.notify_chat:
+                bot.send_message(chat_id=self.notify_chat, text=log_message)
+
+
+    def attachment_check(self, bot, update):
+        """ Hide messages with attachments (except photo or video) """
+        if (update.message.audio or
+            update.message.document or
+            update.message.game or
+            update.message.voice):
+            # Logging
+            if update.message.document:
+                # GIFs are documents and allowed
+                mime_type = update.message.document.mime_type
+                if mime_type and mime_type in self.allowed_mime_types:
+                    return
+                log_message = "❌ HIDE DOCUMENT: {}".format(update.message.document.__dict__)
+            else:
+                log_message = "❌ HIDE NON-DOCUMENT ATTACHMENT"
+            if self.debug:
+                update.message.reply_text(log_message)
+            print(log_message)
+            # Delete the message
+            update.message.delete()
+            # Log in database
+            s = session()
+            messageHide = MessageHide(
+                user_id=update.message.from_user.id,
+                message=update.message.text)
+            s.add(messageHide)
+            s.commit()
+            s.close()
+            # Notify channel
+            if self.notify_chat:
+                bot.send_message(chat_id=self.notify_chat, text=log_message)
 
 
     def logger(self, bot, update):
-        """ Primary Logger. Handles incoming bot messages and saves them to DB """
+        """ Primary Logger. Handles incoming bot messages and saves them to DB
+
+        :param bot: telegram.Bot https://python-telegram-bot.readthedocs.io/en/stable/telegram.bot.html
+        :param update: telegram.Update https://python-telegram-bot.readthedocs.io/en/stable/telegram.update.html
+        """
+
+        if (
+            update.effective_user is None
+            or update.effective_user.id in self.ignore_user_ids
+        ):
+            print("{}: Ignoring update.".format(update.update_id))
+            return
+
         try:
-            user = update.message.from_user
 
-            # Limit bot to monitoring certain chats
+            message = update.message
-            if update.message.chat_id not in self.chat_ids:
-                print("Message from user {} is from chat_id not being monitored: {}".format(
-                    user.id,
-                    update.message.chat_id)
-                )
-                return
 
-            if self.id_exists(user.id):
+            # message is optional
-                self.log_message(user.id, update.message.text)
+            if message is None:
-            else:
-                add_user_success = self.add_user(
-                    user.id,
-                    user.first_name,
-                    user.last_name,
-                    user.username)
 
-                if add_user_success:
+                if update.effective_message is None:
-                    self.log_message(user.id, update.message.text)
+                    print("No message included in update")
-                    print("User added: {}".format(user.id))
+                    return
+
+                message = update.effective_message
+
+            if message:
+
+                user = message.from_user
+
+                # Limit bot to monitoring certain chats
+                if message.chat_id not in self.chat_ids:
+                    from_user = "UNKNOWN"
+                    if user:
+                        from_user = user.id
+                    print("Message from user {} is from chat_id not being monitored: {}".format(
+                        from_user,
+                        message.chat_id)
+                    )
+                    return
+
+                if self.id_exists(user.id):
+                    self.log_message(user.id, message.text,
+                                     message.chat_id)
                 else:
-                    print("Something went wrong adding the user {}".format(user.id), file=sys.stderr)
+                    add_user_success = self.add_user(
+                        user.id,
+                        user.first_name,
+                        user.last_name,
+                        user.username)
 
-            if update.message.text:
+                    if add_user_success:
-                print("{} {} ({}) : {}".format(
+                        self.log_message(
-                    strftime("%Y-%m-%dT%H:%M:%S"),
+                            user.id, message.text, message.chat_id)
-                    user.id,
+                        print("User added: {}".format(user.id))
-                    (user.username or (user.first_name + " " + user.last_name) or "").encode('utf-8'),
+                    else:
-                    update.message.text.encode('utf-8'))
+                        print("Something went wrong adding the user {}".format(user.id), file=sys.stderr)
-                )
 
-            if (self.debug or
+                user_name = (
-                update.message.from_user.id not in self.get_admin_ids(bot, update.message.chat_id)):
+                    user.username or
+                    "{} {}".format(user.first_name, user.last_name) or
+                    "<none>").encode('utf-8')
+                if message.text:
+                    print("{} {} ({}) : {}".format(
+                        strftime("%Y-%m-%dT%H:%M:%S"),
+                        user.id,
+                        user_name,
+                        update.message.text.encode('utf-8'))
+                    )
+                else:
+                    print("{} {} ({}) : non-message".format(
+                        strftime("%Y-%m-%dT%H:%M:%S"),
+                        user.id,
+                        user_name)
+                    )
+
+            else:
+                print("Update and user not logged because no message was found")
+
+            # Don't check admin activity
+            is_admin = False
+            if message:
+                is_admin = message.from_user.id in self.get_admin_ids(bot, message.chat_id)
+
+            if is_admin and self.admin_exempt:
+                print("👮‍♂️ Skipping checks. User is admin: {}".format(user.id))
+            else:
                 # Security checks
+                self.attachment_check(bot, update)
                 self.security_check_username(bot, update)
                 self.security_check_message(bot, update)
-            else:
-                print("Skipping checks. User is admin: {}".format(user.id))
 
         except Exception as e:
-            print("Error: {}".format(e))
+            print("Error[521]: {}".format(e))
-
+            print(traceback.format_exc())
+            print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
 
     # DB queries
     def id_exists(self, id_value):
@@ -230,16 +559,36 @@ class TelegramMonitorBot:
 
         return bool_set
 
+    def log_message(self, user_id, user_message, chat_id):
+
+        if user_message is None:
+            user_message = "[NO MESSAGE]"
 
-    def log_message(self, user_id, user_message):
         try:
             s = session()
-            msg1 = Message(user_id=user_id, message=user_message)
+            language_code = english_message = ""
+            polarity = subjectivity = 0.0
+            try:
+                # translate to English & log the original language
+                translator = Translator()
+                translated = translator.translate(user_message)
+                language_code = translated.src
+                english_message = translated.text
+                # run basic sentiment analysis on the translated English string
+                analysis = TextBlob(english_message)
+                polarity = analysis.sentiment.polarity
+                subjectivity = analysis.sentiment.subjectivity
+            except Exception as e:
+                print("Error translating message: {}".format(e))
+            msg1 = Message(user_id=user_id, message=user_message, chat_id=chat_id, 
+                language_code=language_code, english_message=english_message, polarity=polarity,
+                subjectivity=subjectivity)
             s.add(msg1)
             s.commit()
             s.close()
         except Exception as e:
-            print("Error: {}".format(e))
+            print("Error logging message: {}".format(e))
+            print(traceback.format_exc())
 
 
     def add_user(self, user_id, first_name, last_name, username):
@@ -255,8 +604,94 @@ class TelegramMonitorBot:
             s.close()
             return self.id_exists(user_id)
         except Exception as e:
-            print("Error: {}".format(e))
+            print("Error[347]: {}".format(e))
+            print(traceback.format_exc())
 
+    def handle_command(self, bot, update):
+        """ Handles commands
+
+        Note: Args reversed from docs?  Maybe version differences?  Docs say
+        cb(update, context) but we're getting cb(bot, update).
+
+        update: Update: https://python-telegram-bot.readthedocs.io/en/stable/telegram.update.html#telegram.Update
+        context: CallbackContext: https://python-telegram-bot.readthedocs.io/en/stable/telegram.ext.callbackcontext.html
+
+        hi: says hi
+        price: prints the OGN price
+        """
+        chat_id = None
+        command = None
+        message_id = update.effective_message.message_id
+
+        command = command_from_message(update.effective_message)
+
+        if update.effective_message.chat:
+            chat_id = update.effective_message.chat.id
+
+        print('command: {} seen in chat_id {}'.format(command, chat_id))
+
+        if command == '/hi':
+            bot.send_message(chat_id, 'Yo whattup, @{}!'.format(update.effective_user.username))
+
+        elif command == '/flip':
+            bot.send_message(chat_id, '╯°□°）╯︵ ┻━┻')
+
+        elif command == '/unflip':
+            bot.send_message(chat_id, '┬──┬ ¯\\_(՞▃՞ ¯\\_)')
+
+        elif command == '/price':
+            """ Price, 24 hour %, 24 hour volume, and market cap """
+            symbol = 'OGN'
+            if symbol not in self.cached_prices:
+                self.cached_prices[symbol] = TokenData(symbol)
+            pdata = self.cached_prices[symbol]
+            message = """
+*Origin Token* (OGN)
+*USD Price*: {} ({}%)
+*BTC Price*: {} ({}%)
+*Market Cap*: {}
+*Volume(24h)*: {}
+
+@{}""".format(
+                monetary_format(pdata.price, decimals=5),
+                pdata.percent_change,
+                btc_format(pdata.btc_price),
+                pdata.btc_percent_change,
+                monetary_format(pdata.market_cap),
+                monetary_format(pdata.volume),
+                update.effective_user.username,
+            )
+
+            # If the last message we sent was price, delete it to reduce spam
+            if (
+                self.last_message_out
+                and self.last_message_out.get('type') == 'price'
+                and self.last_message_out['message'].message_id
+            ):
+                try:
+                    bot.delete_message(
+                        chat_id,
+                        self.last_message_out['message'].message_id
+                    )
+                except Exception as err:
+                    print('Unable to delete previous price message: ', err)
+                    print(traceback.format_exc())
+
+            self.last_message_out = {
+                'type': 'price',
+                'message': bot.send_message(
+                    chat_id,
+                    message,
+                    parse_mode=telegram.ParseMode.MARKDOWN
+                ),
+            }
+
+            # Delete the command message as well
+            try:
+                bot.delete_message(chat_id, message_id)
+            except Exception:
+                # nbd if we cannot delete it
+                pass
 
     def error(self, bot, update, error):
         """ Log Errors caused by Updates. """
@@ -275,9 +710,18 @@ class TelegramMonitorBot:
 
         # on different commands - answer in Telegram
 
+        # on commands
+        dp.add_handler(
+            CommandHandler(
+                command=self.available_commands,
+                callback=self.handle_command,
+                filters=Filters.all,
+            )
+        )
+
         # on noncommand i.e message - echo the message on Telegram
         dp.add_handler(MessageHandler(
-            Filters.text,
+            Filters.all,
             lambda bot, update : self.logger(bot, update)
         ))
 

env_sample.sh

@@ -1,6 +1,7 @@
 # Example env vars for bot
+# Copy this to `env.sh` and edit with your real vars -- it is ignored by git
 
-export TELEGRAM_BOT_POSTGRES_URL="postgresql://postgres:postgres@localhost/origindb"
+export TELEGRAM_BOT_POSTGRES_URL="postgresql://localhost/postgres"
 
 read -r -d '' MESSAGE_BAN_PATTERNS << 'EOF'
 # ETH
@@ -25,3 +26,7 @@ export TELEGRAM_BOT_TOKEN="XXXXXXXXX:XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
 export NAME_BAN_PATTERNS="admin$"
 
 export CHAT_IDS="-250531994"
+
+# Needed to make these env vars visible to python
+export MESSAGE_BAN_PATTERNS=$MESSAGE_BAN_PATTERNS
+export MESSAGE_HIDE_PATTERNS=$MESSAGE_HIDE_PATTERNS

model.py

@@ -1,8 +1,9 @@
-from sqlalchemy import Column, DateTime, String, Integer, ForeignKey, func
+from sqlalchemy import Column, DateTime, BigInteger, String, Integer, Numeric, ForeignKey, func
 from sqlalchemy.orm import relationship, backref
 from sqlalchemy.ext.declarative import declarative_base
 import os
 
+# Localhost url: postgresql://localhost/postgres
 postgres_url = os.environ["TELEGRAM_BOT_POSTGRES_URL"]
 
 
@@ -26,9 +27,13 @@ class Message(Base):
     id = Column(Integer, primary_key=True)
     user_id = Column(Integer, ForeignKey('telegram_users.id'), nullable=False)
     message = Column(String)
+    language_code = Column(String)
+    english_message = Column(String)
+    chat_id = Column(BigInteger)
+    polarity = Column(Numeric)
+    subjectivity = Column(Numeric)
     time = Column(DateTime, default=func.now())
 
-
 class MessageHide(Base):
     __tablename__ = 'telegram_message_hides'
     id = Column(Integer, primary_key=True)

mwt.py

@@ -1,7 +1,7 @@
 import time
 
 class MWT(object):
-    """Memoize With Timeout"""
+    """Memorize With Timeout"""
     _caches = {}
     _timeouts = {}
 
@@ -26,11 +26,11 @@ class MWT(object):
             key = (args, tuple(kw))
             try:
                 v = self.cache[key]
-                print("cache")
+                # print("cache")
                 if (time.time() - v[1]) > self.timeout:
                     raise KeyError
             except KeyError:
-                print("new")
+                # print("new")
                 v = self.cache[key] = f(*args,**kwargs),time.time()
             return v[0]
         func.func_name = f.__name__

requirements.txt

@@ -1,5 +1,9 @@
-psycopg2==2.7.3.2
+psycopg2==2.8.4
 python-telegram-bot==9.0.0
-SQLAlchemy==1.2.2
+SQLAlchemy==1.3.13
 configparser==3.5.0
 Unidecode==1.0.22
+googletrans==2.4.0
+textblob==0.15.3
+ipython==5.5.0
+requests>=2.23.0

runtime.txt

@@ -0,0 +1 @@
+python-3.9.2