MrTuxedo/telegram-moderator
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: MrTuxedo:master
Branches Tags
MrTuxedo:master
MrTuxedo:dependabot/pip/ipython-7.16.3
MrTuxedo:snyk-fix-bbfa0c1ba477d9c040f55853e89b2b85
MrTuxedo:wanderingstan-patch-1
MrTuxedo:develop
..
compare: MrTuxedo:wanderingstan-patch-1
Branches Tags
MrTuxedo:dependabot/pip/ipython-7.16.3
MrTuxedo:snyk-fix-bbfa0c1ba477d9c040f55853e89b2b85
MrTuxedo:master
MrTuxedo:wanderingstan-patch-1
MrTuxedo:develop

No commits in common. "master" and "wanderingstan-patch-1" have entirely different histories.
master ... wanderings

5 changed files with 68 additions and 451 deletions
Show Stats Expand all files Collapse all files

24
README.md
View File

@ -5,11 +5,10 @@ Head to https://www.originprotocol.com/developers to learn more about what we're
# Telegram Bot
- Deletes messages matching specified patterns
- Bans users for posting messages matching specified patterns
- Bans users for posting messagses matching specified patterns
- Bans users with usernames matching specified patterns
- Records logs of conversations
- Records logs of converstations
- Logs an English translation of any foreign languages using Google Translate
- Uses textblob for basic sentiment analysis of both polarity and subjectivity
## Installation
@ -53,23 +52,6 @@ export TELEGRAM_BOT_TOKEN="4813829027:ADJFKAf0plousH2EZ2jBfxxRWFld3oK34ya"
- `DEBUG` : If set to anything except `false`, will put bot into debug mode. This means that all actions will be logged into the chat itself, and more things will be logged.
- `ADMIN_EXEMPT` : If set to anything except `false`, admin users will be exempt from monitoring. Reccomended to be set, but useful to turn off for debugging.
- `NOTIFY_CHAT` : ID of chat to report actions. Can be useful if you have an admin-only chat where you want to monitor the bot's activity. E.g. `-140532994`
- `CMC_API_KEY`: If you want the `/price` bot command to work, make sure to set a CoinMarketcap API key
## Download the corpus for Textblob
For sentiment analysis to work, you'll need to download the latest corpus file for textblob. You can do this by running:
```
python -m textblob.download_corpora
```
If you're running the bot on Heroku, set an environment variable named `NLTK_DATA` to `/app/nltk_data` by running:
```
heroku config:set NLTK_DATA='/app/nltk_data'
```
## Message ban patterns
Sample bash file to set `MESSAGE_BAN_PATTERNS`:
@ -84,7 +66,7 @@ read -r -d '' MESSAGE_BAN_PATTERNS << 'EOF'
EOF
```
## Attachments
## Attachements
By default, any attachments other than images or animations will cause the message to be hidden.

484
bot.py
View File

@ -9,203 +9,16 @@ This bot logs all messages sent in a Telegram Group to a database.
"""
#from __future__ import print_function
import os
from __future__ import print_function
import sys
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
import os
from model import User, Message, MessageHide, UserBan, session
from time import strftime
import re
import unidecode
import locale
import traceback
from time import strftime
from datetime import datetime, timedelta
import requests
import telegram
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
from model import User, Message, MessageHide, UserBan, session
from mwt import MWT
from googletrans import Translator
from textblob import TextBlob
# Used with monetary formatting
locale.setlocale(locale.LC_ALL, '')
# Price data cache duration
CACHE_DURATION = timedelta(minutes=15)
# CMC IDs can be retrived at:
# https://pro-api.coinmarketcap.com/v1/cryptocurrency/map?symbol=[SYMBOL]
CMC_SYMBOL_TO_ID = {
'OGN': 5117,
'USDT': 825,
'USDC': 3408,
'DAI': 4943,
}
CMC_API_KEY = os.environ.get('CMC_API_KEY')
CMC_USD_QUOTE_URL = 'https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest?id={}'
CMC_BTC_QUOTE_URL = 'https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest?id={}&convert=BTC'
def first_of(attr, match, it):
""" Return the first item in a set with an attribute that matches match """
if it is not None:
for i in it:
try:
if getattr(i, attr) == match:
return i
except: pass
return None
def command_from_message(message, default=None):
""" Extracts the first command from a Telegram Message """
if not message or not message.text:
return default
command = None
text = message.text
entities = message.entities
command_def = first_of('type', 'bot_command', entities)
if command_def:
command = text[command_def.offset:command_def.length]
return command or default
def cmc_get_data(jso, cmc_id, pair_symbol='USD'):
""" Pull relevant data from a response object """
if not jso:
return None
data = jso.get('data', {})
specific_data = data.get(str(cmc_id), {})
quote = specific_data.get('quote', {})
symbol_data = quote.get(pair_symbol, {})
return {
'price': symbol_data.get('price'),
'volume': symbol_data.get('volume_24h'),
'percent_change': symbol_data.get('percent_change_24h'),
'market_cap': symbol_data.get('market_cap'),
}
def decimal_format(v, decimals=2):
if not v:
v = 0
f = locale.format_string('%.{}f'.format(decimals), v, grouping=True)
return '{}'.format(f)
def monetary_format(v, decimals=2):
return '${}'.format(decimal_format(v, decimals))
def btc_format(v):
return decimal_format(v, decimals=8)
class TokenData:
def __init__(self, symbol, price=None, stamp=datetime.now()):
self.symbol = symbol
self._price = price
self._btc_price = 0
self._percent_change = 0
self._btc_percent_change = 0
self._volume = 0
self._market_cap = 0
if price is not None:
self.stamp = stamp
else:
self.stamp = None
def _fetch_from_cmc(self, url_template):
""" Get quote data for a specific known symbol """
jso = None
cmc_id = CMC_SYMBOL_TO_ID.get(self.symbol)
url = url_template.format(cmc_id)
r = requests.get(url, headers={
'X-CMC_PRO_API_KEY': CMC_API_KEY,
'Accept': 'application/json',
})
if r.status_code != 200:
print('Failed to fetch price data for id: {}'.format(cmc_id))
return None
try:
jso = r.json()
except Exception:
print('Error parsing JSON')
return None
return jso
def update(self):
""" Fetch price from binance """
jso = None
data = None
if self.stamp is None or (
self.stamp is not None
and self.stamp < datetime.now() - CACHE_DURATION
):
# CMC USD
try:
jso = self._fetch_from_cmc(CMC_USD_QUOTE_URL)
data = cmc_get_data(jso, CMC_SYMBOL_TO_ID[self.symbol])
except Exception as err:
print('Error fetching data: ', str(err))
print(traceback.format_exc())
if data is not None:
self._price = data.get('price')
self._percent_change = data.get('percent_change')
self._volume = data.get('volume')
self._market_cap = data.get('market_cap')
self.stamp = datetime.now()
# CMC BTC
try:
jso = self._fetch_from_cmc(CMC_BTC_QUOTE_URL)
data = cmc_get_data(jso, CMC_SYMBOL_TO_ID[self.symbol], 'BTC')
except Exception as err:
print('Error fetching data: ', str(err))
print(traceback.format_exc())
if data is not None:
self._btc_price = data.get('price')
self._btc_percent_change = data.get('percent_change')
@property
def price(self):
self.update()
return self._price
@property
def btc_price(self):
self.update()
return self._btc_price
@property
def btc_percent_change(self):
self.update()
return self._btc_percent_change
@property
def volume(self):
self.update()
return self._volume
@property
def percent_change(self):
self.update()
pc = str(self._percent_change)
if pc and not pc.startswith('-'):
pc = '+{}'.format(pc)
return pc
@property
def market_cap(self):
self.update()
return self._market_cap
class TelegramMonitorBot:
@ -229,17 +42,10 @@ class TelegramMonitorBot:
print("🔵 MESSAGE_BAN_PATTERNS:\n", os.environ['MESSAGE_BAN_PATTERNS'])
print("🔵 MESSAGE_HIDE_PATTERNS:\n", os.environ['MESSAGE_HIDE_PATTERNS'])
print("🔵 NAME_BAN_PATTERNS:\n", os.environ['NAME_BAN_PATTERNS'])
print("🔵 IGNORE_USER_IDS:\n", os.environ.get('IGNORE_USER_IDS'))
# Channel to notify of violoations, e.g. '@channelname'
self.notify_chat = os.environ['NOTIFY_CHAT'] if 'NOTIFY_CHAT' in os.environ else None
# Ignore these user IDs
if not os.environ.get('IGNORE_USER_IDS'):
self.ignore_user_ids = []
else:
self.ignore_user_ids = list(map(int, os.environ['IGNORE_USER_IDS'].split(',')))
# List of chat ids that bot should monitor
self.chat_ids = (
list(map(int, os.environ['CHAT_IDS'].split(',')))
@ -266,22 +72,6 @@ class TelegramMonitorBot:
re.IGNORECASE | re.VERBOSE)
if self.name_ban_patterns else None)
# Mime type document check
# NOTE: All gifs appear to be converted to video/mp4
mime_types = os.environ.get('ALLOWED_MIME_TYPES', 'video/mp4')
self.allowed_mime_types = set(map(lambda s: s.strip(), mime_types.split(',')))
# Comamnds
self.available_commands = ['flip', 'unflip']
if CMC_API_KEY is not None:
self.available_commands.append('price')
print('Available commands: {}'.format(', '.join(self.available_commands)))
# Cached token prices
self.cached_prices = {}
self.last_message_out = None
@MWT(timeout=60*60)
def get_admin_ids(self, bot, chat_id):
@ -317,8 +107,7 @@ class TelegramMonitorBot:
s.commit()
s.close()
# Notify channel
if self.notify_chat:
bot.send_message(chat_id=self.notify_chat, text=log_message)
bot.sendMessage(chat_id=self.notify_chat, text=log_message)
if self.name_ban_re and self.name_ban_re.search(update.message.from_user.username or ''):
# Logging
@ -337,8 +126,7 @@ class TelegramMonitorBot:
s.commit()
s.close()
# Notify channel
if self.notify_chat:
bot.send_message(chat_id=self.notify_chat, text=log_message)
bot.sendMessage(chat_id=self.notify_chat, text=log_message)
def security_check_message(self, bot, update):
@ -370,8 +158,7 @@ class TelegramMonitorBot:
s.commit()
s.close()
# Notify channel
if self.notify_chat:
bot.send_message(chat_id=self.notify_chat, text=log_message)
bot.sendMessage(chat_id=self.notify_chat, text=log_message)
if self.message_ban_re and self.message_ban_re.search(message):
# Logging
@ -392,8 +179,7 @@ class TelegramMonitorBot:
s.commit()
s.close()
# Notify channel
if self.notify_chat:
bot.send_message(chat_id=self.notify_chat, text=log_message)
bot.sendMessage(chat_id=self.notify_chat, text=log_message)
elif self.message_hide_re and self.message_hide_re.search(message):
# Logging
@ -412,8 +198,7 @@ class TelegramMonitorBot:
s.commit()
s.close()
# Notify channel
if self.notify_chat:
bot.send_message(chat_id=self.notify_chat, text=log_message)
bot.sendMessage(chat_id=self.notify_chat, text=log_message)
def attachment_check(self, bot, update):
@ -424,10 +209,6 @@ class TelegramMonitorBot:
update.message.voice):
# Logging
if update.message.document:
# GIFs are documents and allowed
mime_type = update.message.document.mime_type
if mime_type and mime_type in self.allowed_mime_types:
return
log_message = "❌ HIDE DOCUMENT: {}".format(update.message.document.__dict__)
else:
log_message = "❌ HIDE NON-DOCUMENT ATTACHMENT"
@ -445,95 +226,59 @@ class TelegramMonitorBot:
s.commit()
s.close()
# Notify channel
if self.notify_chat:
bot.send_message(chat_id=self.notify_chat, text=log_message)
bot.sendMessage(chat_id=self.notify_chat, text=log_message)
def logger(self, bot, update):
""" Primary Logger. Handles incoming bot messages and saves them to DB
:param bot: telegram.Bot https://python-telegram-bot.readthedocs.io/en/stable/telegram.bot.html
:param update: telegram.Update https://python-telegram-bot.readthedocs.io/en/stable/telegram.update.html
"""
if (
update.effective_user is None
or update.effective_user.id in self.ignore_user_ids
):
print("{}: Ignoring update.".format(update.update_id))
return
""" Primary Logger. Handles incoming bot messages and saves them to DB """
try:
user = update.message.from_user
message = update.message
# message is optional
if message is None:
if update.effective_message is None:
print("No message included in update")
return
message = update.effective_message
if message:
user = message.from_user
# Limit bot to monitoring certain chats
if message.chat_id not in self.chat_ids:
from_user = "UNKNOWN"
if user:
from_user = user.id
print("Message from user {} is from chat_id not being monitored: {}".format(
from_user,
message.chat_id)
)
return
if self.id_exists(user.id):
self.log_message(user.id, message.text,
message.chat_id)
else:
add_user_success = self.add_user(
user.id,
user.first_name,
user.last_name,
user.username)
if add_user_success:
self.log_message(
user.id, message.text, message.chat_id)
print("User added: {}".format(user.id))
else:
print("Something went wrong adding the user {}".format(user.id), file=sys.stderr)
user_name = (
user.username or
"{} {}".format(user.first_name, user.last_name) or
"<none>").encode('utf-8')
if message.text:
print("{} {} ({}) : {}".format(
strftime("%Y-%m-%dT%H:%M:%S"),
user.id,
user_name,
update.message.text.encode('utf-8'))
)
else:
print("{} {} ({}) : non-message".format(
strftime("%Y-%m-%dT%H:%M:%S"),
user.id,
user_name)
)
# Limit bot to monitoring certain chats
if update.message.chat_id not in self.chat_ids:
print("Message from user {} is from chat_id not being monitored: {}".format(
user.id,
update.message.chat_id)
)
return
if self.id_exists(user.id):
self.log_message(user.id, update.message.text,
update.message.chat_id)
else:
print("Update and user not logged because no message was found")
add_user_success = self.add_user(
user.id,
user.first_name,
user.last_name,
user.username)
if add_user_success:
self.log_message(
user.id, update.message.text, update.message.chat_id)
print("User added: {}".format(user.id))
else:
print("Something went wrong adding the user {}".format(user.id), file=sys.stderr)
user_name = (
user.username or
"{} {}".format(user.first_name, user.last_name) or
"<none>").encode('utf-8')
if update.message.text:
print("{} {} ({}) : {}".format(
strftime("%Y-%m-%dT%H:%M:%S"),
user.id,
user_name,
update.message.text.encode('utf-8'))
)
else:
print("{} {} ({}) : non-message".format(
strftime("%Y-%m-%dT%H:%M:%S"),
user.id,
user_name)
)
# Don't check admin activity
is_admin = False
if message:
is_admin = message.from_user.id in self.get_admin_ids(bot, message.chat_id)
is_admin = update.message.from_user.id in self.get_admin_ids(bot, update.message.chat_id)
if is_admin and self.admin_exempt:
print("👮‍♂️ Skipping checks. User is admin: {}".format(user.id))
else:
@ -543,8 +288,7 @@ class TelegramMonitorBot:
self.security_check_message(bot, update)
except Exception as e:
print("Error[521]: {}".format(e))
print(traceback.format_exc())
print("Error: {}".format(e))
print('Error on line {}'.format(sys.exc_info()[-1].tb_lineno), type(e).__name__, e)
# DB queries
@ -559,36 +303,25 @@ class TelegramMonitorBot:
return bool_set
def log_message(self, user_id, user_message, chat_id):
if user_message is None:
user_message = "[NO MESSAGE]"
try:
s = session()
language_code = english_message = ""
polarity = subjectivity = 0.0
try:
# translate to English & log the original language
translator = Translator()
translated = translator.translate(user_message)
language_code = translated.src
english_message = translated.text
# run basic sentiment analysis on the translated English string
analysis = TextBlob(english_message)
polarity = analysis.sentiment.polarity
subjectivity = analysis.sentiment.subjectivity
except Exception as e:
print("Error translating message: {}".format(e))
msg1 = Message(user_id=user_id, message=user_message, chat_id=chat_id,
language_code=language_code, english_message=english_message, polarity=polarity,
subjectivity=subjectivity)
print(e.message)
msg1 = Message(user_id=user_id, message=user_message,
chat_id=chat_id, language_code=language_code, english_message=english_message)
s.add(msg1)
s.commit()
s.close()
except Exception as e:
print("Error logging message: {}".format(e))
print(traceback.format_exc())
print("Error: {}".format(e))
def add_user(self, user_id, first_name, last_name, username):
@ -604,94 +337,8 @@ class TelegramMonitorBot:
s.close()
return self.id_exists(user_id)
except Exception as e:
print("Error[347]: {}".format(e))
print(traceback.format_exc())
print("Error: {}".format(e))
def handle_command(self, bot, update):
""" Handles commands
Note: Args reversed from docs? Maybe version differences? Docs say
cb(update, context) but we're getting cb(bot, update).
update: Update: https://python-telegram-bot.readthedocs.io/en/stable/telegram.update.html#telegram.Update
context: CallbackContext: https://python-telegram-bot.readthedocs.io/en/stable/telegram.ext.callbackcontext.html
hi: says hi
price: prints the OGN price
"""
chat_id = None
command = None
message_id = update.effective_message.message_id
command = command_from_message(update.effective_message)
if update.effective_message.chat:
chat_id = update.effective_message.chat.id
print('command: {} seen in chat_id {}'.format(command, chat_id))
if command == '/hi':
bot.send_message(chat_id, 'Yo whattup, @{}!'.format(update.effective_user.username))
elif command == '/flip':
bot.send_message(chat_id, '╯°□°）╯︵ ┻━┻')
elif command == '/unflip':
bot.send_message(chat_id, '┬──┬ ¯\\_(՞▃՞ ¯\\_)')
elif command == '/price':
""" Price, 24 hour %, 24 hour volume, and market cap """
symbol = 'OGN'
if symbol not in self.cached_prices:
self.cached_prices[symbol] = TokenData(symbol)
pdata = self.cached_prices[symbol]
message = """
*Origin Token* (OGN)
*USD Price*: {} ({}%)
*BTC Price*: {} ({}%)
*Market Cap*: {}
*Volume(24h)*: {}
@{}""".format(
monetary_format(pdata.price, decimals=5),
pdata.percent_change,
btc_format(pdata.btc_price),
pdata.btc_percent_change,
monetary_format(pdata.market_cap),
monetary_format(pdata.volume),
update.effective_user.username,
)
# If the last message we sent was price, delete it to reduce spam
if (
self.last_message_out
and self.last_message_out.get('type') == 'price'
and self.last_message_out['message'].message_id
):
try:
bot.delete_message(
chat_id,
self.last_message_out['message'].message_id
)
except Exception as err:
print('Unable to delete previous price message: ', err)
print(traceback.format_exc())
self.last_message_out = {
'type': 'price',
'message': bot.send_message(
chat_id,
message,
parse_mode=telegram.ParseMode.MARKDOWN
),
}
# Delete the command message as well
try:
bot.delete_message(chat_id, message_id)
except Exception:
# nbd if we cannot delete it
pass
def error(self, bot, update, error):
""" Log Errors caused by Updates. """
@ -710,15 +357,6 @@ class TelegramMonitorBot:
# on different commands - answer in Telegram
# on commands
dp.add_handler(
CommandHandler(
command=self.available_commands,
callback=self.handle_command,
filters=Filters.all,
)
)
# on noncommand i.e message - echo the message on Telegram
dp.add_handler(MessageHandler(
Filters.all,

2
model.py
View File

@ -1,4 +1,4 @@
from sqlalchemy import Column, DateTime, BigInteger, String, Integer, Numeric, ForeignKey, func
from sqlalchemy import Column, DateTime, BigInteger, String, Integer, ForeignKey, func
from sqlalchemy.orm import relationship, backref
from sqlalchemy.ext.declarative import declarative_base
import os

8
requirements.txt
View File

@ -1,9 +1,7 @@
psycopg2==2.8.4
psycopg2==2.7.3.2
python-telegram-bot==9.0.0
SQLAlchemy==1.3.13
SQLAlchemy==1.2.2
configparser==3.5.0
Unidecode==1.0.22
googletrans==2.4.0
textblob==0.15.3
ipython==5.5.0
requests>=2.23.0
textblob

1
runtime.txt
View File

@ -1 +0,0 @@
python-3.9.2