#!/usr/bin/env python3 # -*- coding: UTF-8 -*- import hashlib import json import logging import re import sys import time from datetime import datetime, timedelta, timezone from typing import Any, Dict, List, Optional, TypedDict, Union import requests from requests import Response sys.path.append("/opt/voicetech/integrations/online") from config import Config from io_manager import IOManager from io_manager.io_manager.manager import InputParams from logger import setup_loggers logger = logging.getLogger(__name__) DEBUG_MODE = False ALLOWED_ACTIONS = [ "waterSupply", # Горячее и холодное водоснабжение "waterWarm", # Горячее водоснабжение "waterCold", # Холодное водоснабжение "electricitySupply", # Электроснабжение "gasSupply", # Газоснабжение "heating", # Теплоснабжение ] EXCEPTION_WORDS = ["семьи", "семья"] EXCEPTION_STREETS = { "сако ванцети": "сакко ванцетти", "сако иванцети": "сакко ванцетти", "сакой ванцети": "сакко ванцетти", "3сентября": "3 сентября", "3воинская": "3 воинская", "25лет октября": "25 лет октября", "30лет октября": "30 лет октября", "40лет комсомола": "40 лет комсомола", "1905года": "1905 года", "905года": "1905 года", "каратаева": "коротаева", "3интернационала": "III Интернационала", "3интернационал": "III Интернационала", "пауза": "краузе", "кошелова": "кошевого", "том усинская": "томьусинская", "мальчишенское": "мочищенское", "головщика": "галущака", } DELETE_WORDS = [ "дзержинский", "железнодорожный", "заельцовский", "калининский", "кировский", "ленинский", "октябрьский", "первомайский", "советский", "центральный", "район", "р.", "город", "г.", "номер", "новосибирск", ] FILTER_STREETS = { "Сухарная улица 113": {"exclude": ["1-я", "2-я", "3-я"]}, "Сухарная улица 80": {"exclude": ["1-я", "2-я", "3-я"]}, } SERVICE_WORDS = { "улица": "ул", "ул.": "ул", "дом": "д", "д.": "д", "корпус": "к", "корп": "к", "корп.": "к", } class OutageResponse(TypedDict): response: Response outages: Optional[List] def normalize_spaces(text: str) -> str: return re.sub(r"\s+", " ", text).strip() def normalize_service_words(address: str) -> str: words = address.split() result = [] for word in words: result.append(SERVICE_WORDS.get(word, word)) return " ".join(result) def remove_trash_words(address: str, trash_words: List[str] = None) -> str: delete_words = DELETE_WORDS if trash_words is None else trash_words pattern = r"\b(?:" + "|".join(map(re.escape, delete_words)) + r")\b" return re.sub(pattern, "", address) def process_corpus(address: str) -> str: """ Все варианты корпуса -> кN """ patterns = [ # 43к5 (r"(\d+)к\.?\s*(\d+[а-я]?)", r"\1 к\2"), # к 5 (r"\bк\.?\s*(\d+[а-я]?)\b", r"к\1"), ] for pattern, repl in patterns: address = re.sub(pattern, repl, address) return address def glue_house_letter(address: str) -> str: """ 43 а -> 43а """ return re.sub( r"(\d+)\s+([а-яa-z])\b", r"\1\2", address, flags=re.IGNORECASE, ) def remove_flat(address: str) -> str: patterns = [ r"\bквартира\b.*", r"\bкв\.?\b.*", ] for pattern in patterns: address = re.sub(pattern, "", address) return address def replace_numbers(address: str, numbers: list[dict]) -> str: """ numbers: [ { "position": 3, "length": 2, "calculated": "43" } ] """ if not numbers: return address parts = address.split() for number in numbers: pos = number["position"] if parts[pos] in EXCEPTION_WORDS: continue parts[pos] = number["calculated"] rng = slice(pos + 1, pos + number["length"]) parts[rng] = ["$"] * (number["length"] - 1) parts = [x for x in parts if x != "$"] return " ".join(parts) def w2n(see_host: str, address) -> List[Optional[dict]]: see_url = f"{see_host}/see/entities/number?text={address}" headers = {"Accept": "application/json"} response = requests.get(see_url, headers=headers) if response.status_code != 200: return [] else: json = response.json() if not json: return [] return json["number"] def correct_address(see_host: str, address: str) -> str: address = address.lower() address = address.replace(".", ". ") address = remove_trash_words(address) address = remove_flat(address) address = normalize_service_words(address) address = process_corpus(address) numbers = w2n(see_host, address) if numbers: address = replace_numbers(address, numbers) address = glue_house_letter(address) address = remove_trash_words(address, ["ул", "д"]) address = address.replace(",", " ") address = normalize_spaces(address) return address def normalize_address(es_host: str, address: str) -> Dict[str, str]: query = { "query": { "bool": { "must": [ { "multi_match": { "query": address, "fields": ["address"], "fuzziness": "AUTO", "operator": "and", "prefix_length": 2, } } ], "should": [ { "match_phrase": { "address": { "query": address, "slop": 3, "boost": 2, } } }, ], } }, "size": 1, } es_index = "address_registry" es_url = f"{es_host}/{es_index}/_search" response = requests.post(es_url, json=query) if response.status_code != 200: return None else: if response.json()["hits"]["total"]["value"] == 0: return {"address": "", "district_name": ""} else: return response.json()["hits"]["hits"][0]["_source"] def build_sql_where(address: Dict[str, str]) -> str: address_parts = address["address"].split() for i, part in enumerate(address_parts): for char in part: if char.isdigit(): address_parts[i] = f" {part} " break return "%" + "%".join(address_parts) + "%" def get_outages(address: Dict[str, str]) -> List[dict]: where = build_sql_where(address) params = { "where": f"address like '{where}' and (start_date::date <= CURRENT_DATE and end_date::date >= CURRENT_DATE)", "text": "", "geometry": "", "spatialRel": " ", "objectIds": "", "orderByFields": "", "outFields": "", "f": "json", "geometryType": " ", "inSR": "", "groupByFieldsForStatistics": "", "outSR": "", "outStatistics": "", "resultOffset": "", "resultRecordCount": "", "time": "", } response = requests.get( "https://map.novo-sibirsk.ru/elitegis/rest/services/maps/disconnections/MapServer/101/query", params=params, timeout=10, verify=False ) response.raise_for_status() result = [] data = response.json() for feature in data["features"]: feature_address = feature["value"].replace(",", "") if address["address"] in feature_address: result.append(feature["attributes"]) if not result: return [] type_map = {} system_map = {} time_map = {} def get_map(code_values: Dict[str, Any]): return {value["code"]: value["name"] for value in code_values} for field in data["fields"]: if field["name"] == "type_id": type_map = get_map(field["domain"]["codedValues"]) if field["name"] == "system_id": system_map = get_map(field["domain"]["codedValues"]) if field["name"] == "time_category": time_map = get_map(field["domain"]["codedValues"]) actions_map = { "Горячее водоснабжение": "waterWarm", "Холодное водоснабжение и канализация": "waterCold", "Электроснабжение": "electricitySupply", "Газоснабжение": "gasSupply", "Теплоснабжение": "heating", } for record in result: record["address"] = ( record["address"] .replace(",", "") .replace(f"({record['district_name']})", "") ) record["type"] = type_map[record["type_id"]] record["system"] = system_map[record["system_id"]] record["time"] = time_map[record["time_category"]] record["actionS"] = actions_map[system_map[record["system_id"]]] record.pop("oid", "") record.pop("type_id", "") record.pop("system_id", "") record.pop("1", "") return result def get_actions_list(actionS: str, replace_water_supply: bool = True) -> List[str]: actionS_list: List[str] = re.split(r"[,\s;]+", actionS) if replace_water_supply: if "waterSupply" in actionS_list: actionS_list.remove("waterSupply") if "waterCold" not in actionS_list: actionS_list.append("waterCold") if "waterWarm" not in actionS_list: actionS_list.append("waterWarm") return actionS_list def culling_unworthy(outages: List[dict], address: Dict[str, str]) -> List[dict]: worthy = [] pattern = re.compile( r""" ^.*? # Имя улицы (любые символы до номера дома) (?P # Группа для номера дома \d+ # Обязательные цифры номера [а-яА-Я]? # Опциональный литер дома (1 буква) (?:/\d+[а-яА-Я]?)? # Опциональная дробь с литером ) (?:\s+к(?P # Группа для корпуса \d+ # Обязательные цифры корпуса [а-яА-Я]? # Опциональный литер корпуса (?:/\d+[а-яА-Я]?)? # Опциональная дробь корпуса )?)?$ # Вся группа корпуса необязательна """, re.VERBOSE | re.IGNORECASE, ) match = pattern.search(address["address"]) if match: house = match.group("house") building = match.group("building") for outage in outages: o_address = outage["address"] if o_address[-1] == " ": o_address = o_address[:-1] o_match = pattern.search(o_address) if o_match: o_house = o_match.group("house") o_building = o_match.group("building") if o_house == house and o_building == building: worthy.append(outage) else: continue if len(worthy) == 0: numbers = re.findall(r"\d+", address["address"]) for outage in outages: outage_numbers = re.findall(r"\d+", outage["address"]) if len(outage_numbers) == len(numbers) and set(outage_numbers) == set( numbers ): worthy.append(outage) worthy = [out for out in outages if out["time"] == "Текущие"] if address["address"] in FILTER_STREETS: for exc in FILTER_STREETS[address["address"]]["exclude"]: worthy = [el for el in worthy if exc not in el["address"]] return worthy def rename_waterCold(outages: List[dict]) -> List[dict]: for outage in outages: if outage["actionS"] == "waterCold": outage["system"] = "Холодное водоснабжение" return outages def build_info(outages: List[dict]) -> str: # system type date_range # 0 0 0 -> одно отключение # 1 0 0 -> два отключения # 1 1 0 -> два отключения с разными типами # 1 0 1 -> два отключения с разным временем # 1 1 1 -> два отключения с разным типом и временем diff_type = False diff_range_date = False o_type = outages[0]["type"] o_start_date = outages[0]["start_date"] o_end_date = outages[0]["end_date"] for outage in outages: if outage["start_date"] != o_start_date or outage["end_date"] != o_end_date: diff_range_date = True if outage["type"] != o_type: diff_type = True def get_data_range(start_date: int, end_date: int) -> str: months_genitive = { 1: "января", 2: "февраля", 3: "марта", 4: "апреля", 5: "мая", 6: "июня", 7: "июля", 8: "августа", 9: "сентября", 10: "октября", 11: "ноября", 12: "декабря", } start_date = datetime.fromtimestamp(int(str(start_date)[:10])) end_date = datetime.fromtimestamp(int(str(end_date)[:10])) start_month = months_genitive[start_date.month] end_month = months_genitive[end_date.month] if start_date.hour == 0 and start_date.minute == 0: start_date_str = start_date.strftime( f"%d {start_month} с двенадцати часов ночи" ) else: start_date_str = start_date.strftime(f"%d {start_month} с %H:%M") if end_date.hour == 0 and end_date.minute == 0: end_date_str = end_date.strftime( f"по %d {end_month} до двенадцати часов ночи" ) else: end_date_str = end_date.strftime(f"по %d {end_month} до %H:%M") return f"{start_date_str} {end_date_str}" if len(outages) == 1 and not diff_type and not diff_range_date: # {type} {system} {date_range}. result = f"{o_type} {outages[0]['system']} {get_data_range(o_start_date, o_end_date)}" elif len(outages) > 1 and not diff_type and not diff_range_date: # если оотключения два # {type} {system1} и {system2} {date_range}. # если отключений больше # {type} {system1}, {system2}, {system3} {date_range}. separator = " и " if len(outages) == 2 else ", " result = f"{o_type} {separator.join([o['system'] for o in outages])} {get_data_range(o_start_date, o_end_date)}" elif len(outages) > 1 and diff_type and not diff_range_date: # {type1} {system1}, {type2} {system2} {date_range} ... result = f"""{", ".join([f"{o['type']} {o['system']}" for o in outages])} {get_data_range(o_start_date, o_end_date)}""" elif len(outages) > 1: # {type1} {system1} {date_range1}, а также {type2} {system2} {date_range2} ... result = ", а также ".join([ f"{o['type']} {o['system']} {get_data_range(o['start_date'], o['end_date'])}" for o in outages ]) return result def check_outage(outages: List[dict], actionS_list: str) -> Dict[str, str]: target_outages = [] other_outages = [] for outage in outages: if outage["actionS"] in actionS_list: target_outages.append(outage) else: other_outages.append(outage) result = {} if target_outages: result["result"] = "info_yes" result["info"] = build_info(target_outages) else: result["result"] = "info_no" info_map = { "waterWarm": "Горячее водоснабжение", "waterCold": "Холодное водоснабжение", "electricitySupply": "Электроснабжение", "gasSupply": "Газоснабжение", "heating": "Теплоснабжение", } result["info"] = ", ".join([info_map[a] for a in actionS_list]) if other_outages: result["result"] = result["result"] + "_other_yes" result["info_other"] = build_info(other_outages) else: result["result"] = result["result"] + "_other_no" return result def push_elastic_log( es_host: str, log_data: InputParams, outage_response: Optional[Union[Response, dict]] = None, redis_data: Optional[Dict[str, Any]] = None, ) -> None: if not redis_data: redis_data = {} channel_type = "telephony" if log_data.phone else "text" nsk_offset = timezone(timedelta(hours=7)) date = datetime.now(nsk_offset).isoformat() if isinstance(outage_response, Response) and outage_response: request = json.loads(outage_response.request.body.decode("utf-8")) elif isinstance(outage_response, dict): request = outage_response else: request = {} actions_map = { "waterWarm": "Горячее водоснабжение", "waterCold": "Холодное водоснабжение", "electricitySupply": "Электроснабжение", "gasSupply": "Газоснабжение", "heating": "Теплоснабжение", "waterSupply": "Нет воды", } actions_list = get_actions_list( actionS=log_data.data["actionS"], replace_water_supply=False ) log = { "dialog_id": log_data.interaction_id, "start_date": date, "end_date": date, "phone": log_data.phone, "actionS": ", ".join([ actions_map[a] if a in actions_map else a for a in actions_list ]), "address": log_data.data["address"], "request": json.dumps(request, ensure_ascii=False), "info": log_data.data.get("info", ""), "result": log_data.data["result"], "info_other": redis_data.get("info_other", ""), "channel_type": channel_type, } id = hashlib.md5(json.dumps(log).encode("utf-8")).hexdigest() requests.post( url=es_host + "/integrations/events/" + id, headers={"Content-Type": "application/json"}, json=log, timeout=5, ) def proccessing_ending_word(info: str) -> str: exception_words = ["Плановое", "Аварийное", "отключение"] for exception_word in exception_words: bad_word = exception_word[: len(exception_word) - 1] + "$" + exception_word[-1] info = info.replace(exception_word, bad_word) result_info = ( info .replace("ие", "ия") .replace("ое", "ого") .replace("ее", "его") .replace("$", "") ) return result_info def move_word_to_beginning(address: str, word: str) -> str: filtered_address = [w for w in address.split() if w.lower() != word.lower()] result = " ".join(filtered_address) return f"{word} {result}".strip() def main() -> None: integrations_config = Config("integrations.conf") es_host = integrations_config.get_param("elasticsearch_host") see_host = integrations_config.get_param("see_host") logpath = integrations_config.get_param("integrations_logpath_mask") if DEBUG_MODE: logpath = "./edds_debug.log" setup_loggers(logger, level=logging.WARNING, filename=logpath, log_by_day=True) io_manager = IOManager(logger=logger) input_params = io_manager.input() # second round input_params.data.pop("info_other", "") c_address = correct_address(see_host=see_host, address=input_params.data["address"]) # Может прийти только число, без улицы if c_address.isdigit(): push_elastic_log( log_data=input_params, redis_data=input_params.data.copy(), outage_response={"address_before_correct": c_address}, es_host=es_host, ) return io_manager.output( is_error=True, goto=False, data=input_params.data_from_args ) for exception_street in EXCEPTION_STREETS: if exception_street in c_address: c_address = c_address.replace( exception_street, EXCEPTION_STREETS[exception_street] ) address = normalize_address(es_host=es_host, address=c_address) # Пробуем достать адрес из addressS if not address["address"]: c_address = correct_address( see_host=see_host, address=input_params.data["addressS"] ) # Может прийти только число, без улицы if c_address.isdigit(): push_elastic_log( log_data=input_params, redis_data=input_params.data.copy(), outage_response={"address_before_correct": c_address}, es_host=es_host, ) return io_manager.output( is_error=True, goto=False, data=input_params.data_from_args ) for exception_street in EXCEPTION_STREETS: if exception_street in c_address: c_address = c_address.replace( exception_street, EXCEPTION_STREETS[exception_street] ) address = normalize_address(es_host=es_host, address=c_address) if not address["address"]: input_params.data["result"] = "address_not_found" push_elastic_log( log_data=input_params, redis_data=input_params.data.copy(), outage_response={"address_before_correct": c_address}, es_host=es_host, ) return io_manager.output( is_error=True, goto=False, data=input_params.data_from_args ) # Адрес для озвучки address_for_voice = address["address"] WORDS_TO_MOVE = ["бульвар", "площадь", "площадь", "поселок", "улица", "территория"] for word in WORDS_TO_MOVE: if word in address_for_voice: address_for_voice = move_word_to_beginning(address_for_voice, word) input_params.data["address"] = address_for_voice input_params.data_from_args["address"] = address_for_voice try: outages = get_outages(address=address) except TimeoutError: input_params.data["result"] = "portal_timeout" return io_manager.output(is_error=True, goto=False, data=input_params.data) if len(outages) == 0: input_params.data_from_args["result"] = input_params.data["result"] = ( "info_no_other_no" ) push_elastic_log( log_data=input_params, redis_data=input_params.data.copy(), outage_response={"address": address["address"]}, es_host=es_host, ) return io_manager.output( is_error=False, goto=True, data=input_params.data_from_args ) actionS_list = get_actions_list(input_params.data["actionS"]) actionS_list = [actionS for actionS in actionS_list if actionS in ALLOWED_ACTIONS] outages = culling_unworthy(outages=outages, address=address) outages = rename_waterCold(outages=outages) result = check_outage(outages=outages, actionS_list=actionS_list) input_params.data["result"] = result["result"] target_info = proccessing_ending_word(result["info"]) input_params.data["info"] = target_info if "info_other" in result: other_info = proccessing_ending_word(result["info_other"]) input_params.data["info_other"] = other_info push_elastic_log( log_data=input_params, redis_data=input_params.data, outage_response={"address": address["address"]}, es_host=es_host, ) return io_manager.output(is_error=False, goto=True, data=input_params.data) main()