en2zmax revised this gist . Go to revision
1 file changed, 143 insertions
clear_old_data_in_db_voicetech.py(file created)
| @@ -0,0 +1,143 @@ | |||
| 1 | + | #!/usr/bin/env python3 | |
| 2 | + | ||
| 3 | + | import os | |
| 4 | + | import sys | |
| 5 | + | import logging | |
| 6 | + | ||
| 7 | + | import mysql.connector as con | |
| 8 | + | ||
| 9 | + | from datetime import date, timedelta, datetime | |
| 10 | + | ||
| 11 | + | # Дней хранения логов | |
| 12 | + | LOGGING_DAYS = 7 | |
| 13 | + | ||
| 14 | + | # Сохранение данных за указанное количество дней | |
| 15 | + | day_delta = 100 | |
| 16 | + | ||
| 17 | + | # Сохранение данных с конкретной даты | |
| 18 | + | # start_date_string = "2024-09-01" | |
| 19 | + | # day_delta = ( | |
| 20 | + | # date.today() - datetime.strptime(start_date_string, "%Y-%m-%d").date() | |
| 21 | + | # ).days | |
| 22 | + | ||
| 23 | + | # Данные подключения к БД | |
| 24 | + | con_data = { | |
| 25 | + | "host": "10.101.60.89", | |
| 26 | + | "port": 3306, | |
| 27 | + | "user": "root", | |
| 28 | + | "password": "MySQLP@ssw0rd1", | |
| 29 | + | "database": "voicetech", | |
| 30 | + | } | |
| 31 | + | ||
| 32 | + | logging.basicConfig( | |
| 33 | + | level=logging.INFO, | |
| 34 | + | filename=f"clear_old_data_in_database_{date.today()}.log", | |
| 35 | + | filemode="a", | |
| 36 | + | format="[%(asctime)s: %(levelname)s] %(message)s", | |
| 37 | + | ) | |
| 38 | + | logger = logging.getLogger("logger") | |
| 39 | + | try: | |
| 40 | + | os.remove( | |
| 41 | + | f"clear_old_data_in_database_{date.today() - timedelta(days=LOGGING_DAYS)}.log" | |
| 42 | + | ) | |
| 43 | + | except: | |
| 44 | + | pass | |
| 45 | + | ||
| 46 | + | ||
| 47 | + | class MySQLExecutor: | |
| 48 | + | def __init__(self): | |
| 49 | + | try: | |
| 50 | + | self._connector = con.connect(**con_data) | |
| 51 | + | except: | |
| 52 | + | logger.exception("Exception") | |
| 53 | + | sys.exit() | |
| 54 | + | ||
| 55 | + | def __del__(self): | |
| 56 | + | self._connector.close() | |
| 57 | + | ||
| 58 | + | def execute(self, query: str): | |
| 59 | + | cursor = self._connector.cursor() | |
| 60 | + | try: | |
| 61 | + | cursor.execute(query) | |
| 62 | + | except: | |
| 63 | + | logger.exception("Exception") | |
| 64 | + | sys.exit() | |
| 65 | + | cursor.close() | |
| 66 | + | ||
| 67 | + | def execute_commit(self, query: str): | |
| 68 | + | cursor = self._connector.cursor() | |
| 69 | + | try: | |
| 70 | + | cursor.execute(query) | |
| 71 | + | self._connector.commit() | |
| 72 | + | except: | |
| 73 | + | logger.exception("Exception") | |
| 74 | + | sys.exit() | |
| 75 | + | cursor.close() | |
| 76 | + | ||
| 77 | + | def select(self, query) -> list: | |
| 78 | + | cursor = self._connector.cursor() | |
| 79 | + | try: | |
| 80 | + | cursor.execute(query) | |
| 81 | + | result = cursor.fetchall() | |
| 82 | + | except: | |
| 83 | + | logger.exception("Exception") | |
| 84 | + | sys.exit() | |
| 85 | + | cursor.close() | |
| 86 | + | return result | |
| 87 | + | ||
| 88 | + | ||
| 89 | + | def optimize_table(table_name: str, where_column: str): | |
| 90 | + | logger.info(f"Start optimize table {table_name}.") | |
| 91 | + | executor = MySQLExecutor() | |
| 92 | + | ||
| 93 | + | query = f"CREATE TABLE voicetech.{table_name}_new LIKE voicetech.{table_name};" | |
| 94 | + | executor.execute(query) | |
| 95 | + | logger.info(f"Create table '{table_name}_new'.") | |
| 96 | + | ||
| 97 | + | query = f"select TRIGGER_NAME from information_schema.TRIGGERS where EVENT_OBJECT_SCHEMA = 'voicetech' and EVENT_OBJECT_TABLE = '{table_name}';" | |
| 98 | + | create_trigger_queries_list = [ | |
| 99 | + | executor.select(f"show create trigger {trigger};")[0][2] | |
| 100 | + | for trigger in [item[0] for item in executor.select(query)] | |
| 101 | + | ] | |
| 102 | + | logger.info(f"Read triggers from table '{table_name}'.") | |
| 103 | + | ||
| 104 | + | base_insert_query = ( | |
| 105 | + | f"insert into voicetech.{table_name}_new select * from voicetech.{table_name}" | |
| 106 | + | ) | |
| 107 | + | ||
| 108 | + | for i in range(day_delta + 1): | |
| 109 | + | begin_date = date.today() - timedelta(days=day_delta - i) | |
| 110 | + | end_date = begin_date + timedelta(days=1) | |
| 111 | + | query = f"{base_insert_query} where {where_column} >= '{begin_date}' and {where_column} < '{end_date}';" | |
| 112 | + | executor.execute_commit(query) | |
| 113 | + | logger.info(f"Copy data in table '{table_name}_new' for {begin_date}.") | |
| 114 | + | ||
| 115 | + | query = f"RENAME TABLE voicetech.{table_name} TO voicetech.{table_name}_old, voicetech.{table_name}_new TO voicetech.{table_name};" | |
| 116 | + | executor.execute(query) | |
| 117 | + | logger.info( | |
| 118 | + | f"Rename table '{table_name}' to '{table_name}_old' and '{table_name}_new' to '{table_name}'." | |
| 119 | + | ) | |
| 120 | + | ||
| 121 | + | query = f"DROP TABLE voicetech.{table_name}_old;" | |
| 122 | + | executor.execute(query) | |
| 123 | + | logger.info(f"Delete table '{table_name}_old'.") | |
| 124 | + | ||
| 125 | + | for query in create_trigger_queries_list: | |
| 126 | + | executor.execute(query) | |
| 127 | + | logger.info(f"Restore triggers from table '{table_name}'.") | |
| 128 | + | ||
| 129 | + | ||
| 130 | + | def main(): | |
| 131 | + | logger.info("") | |
| 132 | + | logger.info("Start script") | |
| 133 | + | logger.info("") | |
| 134 | + | optimize_table("interactions", "dtStart") | |
| 135 | + | optimize_table("phone_cdr", "start") | |
| 136 | + | optimize_table("phone_ivr_journeys", "journeyTime") | |
| 137 | + | logger.info("") | |
| 138 | + | logger.info("Stop script") | |
| 139 | + | logger.info("") | |
| 140 | + | ||
| 141 | + | ||
| 142 | + | if __name__ == "__main__": | |
| 143 | + | main() | |
Newer
Older