Last active 1790752416

script-ui.py Raw
1#!/usr/bin/env python3
2# -*- coding: UTF-8 -*-
3import hashlib
4import json
5import logging
6import re
7import sys
8import time
9from datetime import datetime, timedelta, timezone
10from typing import Any, Dict, List, Optional, TypedDict, Union
11
12import requests
13from requests import Response
14
15sys.path.append("/opt/voicetech/integrations/online")
16from config import Config
17from io_manager import IOManager
18from io_manager.io_manager.manager import InputParams
19from logger import setup_loggers
20
21logger = logging.getLogger(__name__)
22DEBUG_MODE = False
23
24ALLOWED_ACTIONS = [
25 "waterSupply", # Горячее и холодное водоснабжение
26 "waterWarm", # Горячее водоснабжение
27 "waterCold", # Холодное водоснабжение
28 "electricitySupply", # Электроснабжение
29 "gasSupply", # Газоснабжение
30 "heating", # Теплоснабжение
31]
32EXCEPTION_WORDS = ["семьи", "семья"]
33EXCEPTION_STREETS = {
34 "сако ванцети": "сакко ванцетти",
35 "сако иванцети": "сакко ванцетти",
36 "сакой ванцети": "сакко ванцетти",
37 "3сентября": "3 сентября",
38 "3воинская": "3 воинская",
39 "25лет октября": "25 лет октября",
40 "30лет октября": "30 лет октября",
41 "40лет комсомола": "40 лет комсомола",
42 "1905года": "1905 года",
43 "905года": "1905 года",
44 "каратаева": "коротаева",
45 "3интернационала": "III Интернационала",
46 "3интернационал": "III Интернационала",
47 "пауза": "краузе",
48 "кошелова": "кошевого",
49 "том усинская": "томьусинская",
50 "мальчишенское": "мочищенское",
51 "головщика": "галущака",
52}
53
54DELETE_WORDS = [
55 "дзержинский",
56 "железнодорожный",
57 "заельцовский",
58 "калининский",
59 "кировский",
60 "ленинский",
61 "октябрьский",
62 "первомайский",
63 "советский",
64 "центральный",
65 "район",
66 "р.",
67 "город",
68 "г.",
69 "номер",
70 "новосибирск",
71]
72FILTER_STREETS = {
73 "Сухарная улица 113": {"exclude": ["1-я", "2-я", "3-я"]},
74 "Сухарная улица 80": {"exclude": ["1-я", "2-я", "3-я"]},
75}
76
77SERVICE_WORDS = {
78 "улица": "ул",
79 "ул.": "ул",
80 "дом": "д",
81 "д.": "д",
82 "корпус": "к",
83 "корп": "к",
84 "корп.": "к",
85}
86
87
88class OutageResponse(TypedDict):
89 response: Response
90 outages: Optional[List]
91
92
93def normalize_spaces(text: str) -> str:
94 return re.sub(r"\s+", " ", text).strip()
95
96
97def normalize_service_words(address: str) -> str:
98 words = address.split()
99
100 result = []
101
102 for word in words:
103 result.append(SERVICE_WORDS.get(word, word))
104
105 return " ".join(result)
106
107
108def remove_trash_words(address: str, trash_words: List[str] = None) -> str:
109 delete_words = DELETE_WORDS if trash_words is None else trash_words
110 pattern = r"\b(?:" + "|".join(map(re.escape, delete_words)) + r")\b"
111 return re.sub(pattern, "", address)
112
113
114def process_corpus(address: str) -> str:
115 """
116 Все варианты корпуса -> кN
117 """
118
119 patterns = [
120 # 43к5
121 (r"(\d+)к\.?\s*(\d+[а-я]?)", r"\1 к\2"),
122 # к 5
123 (r"\bк\.?\s*(\d+[а-я]?)\b", r"к\1"),
124 ]
125
126 for pattern, repl in patterns:
127 address = re.sub(pattern, repl, address)
128
129 return address
130
131
132def glue_house_letter(address: str) -> str:
133 """
134 43 а -> 43а
135 """
136
137 return re.sub(
138 r"(\d+)\s+([а-яa-z])\b",
139 r"\1\2",
140 address,
141 flags=re.IGNORECASE,
142 )
143
144
145def remove_flat(address: str) -> str:
146 patterns = [
147 r"\bквартира\b.*",
148 r"\bкв\.?\b.*",
149 ]
150
151 for pattern in patterns:
152 address = re.sub(pattern, "", address)
153
154 return address
155
156
157def replace_numbers(address: str, numbers: list[dict]) -> str:
158 """
159 numbers:
160 [
161 {
162 "position": 3,
163 "length": 2,
164 "calculated": "43"
165 }
166 ]
167 """
168
169 if not numbers:
170 return address
171
172 parts = address.split()
173
174 for number in numbers:
175 pos = number["position"]
176
177 if parts[pos] in EXCEPTION_WORDS:
178 continue
179
180 parts[pos] = number["calculated"]
181
182 rng = slice(pos + 1, pos + number["length"])
183 parts[rng] = ["$"] * (number["length"] - 1)
184
185 parts = [x for x in parts if x != "$"]
186
187 return " ".join(parts)
188
189
190def w2n(see_host: str, address) -> List[Optional[dict]]:
191 see_url = f"{see_host}/see/entities/number?text={address}"
192 headers = {"Accept": "application/json"}
193 response = requests.get(see_url, headers=headers)
194 if response.status_code != 200:
195 return []
196 else:
197 json = response.json()
198
199 if not json:
200 return []
201
202 return json["number"]
203
204
205def correct_address(see_host: str, address: str) -> str:
206
207 address = address.lower()
208
209 address = address.replace(".", ". ")
210
211 address = remove_trash_words(address)
212
213 address = remove_flat(address)
214
215 address = normalize_service_words(address)
216
217 address = process_corpus(address)
218
219 numbers = w2n(see_host, address)
220
221 if numbers:
222 address = replace_numbers(address, numbers)
223
224 address = glue_house_letter(address)
225
226 address = remove_trash_words(address, ["ул", "д"])
227
228 address = address.replace(",", " ")
229
230 address = normalize_spaces(address)
231
232 return address
233
234
235def normalize_address(es_host: str, address: str) -> Dict[str, str]:
236 query = {
237 "query": {
238 "bool": {
239 "must": [
240 {
241 "multi_match": {
242 "query": address,
243 "fields": ["address"],
244 "fuzziness": "AUTO",
245 "operator": "and",
246 "prefix_length": 2,
247 }
248 }
249 ],
250 "should": [
251 {
252 "match_phrase": {
253 "address": {
254 "query": address,
255 "slop": 3,
256 "boost": 2,
257 }
258 }
259 },
260 ],
261 }
262 },
263 "size": 1,
264 }
265
266 es_index = "address_registry"
267
268 es_url = f"{es_host}/{es_index}/_search"
269
270 response = requests.post(es_url, json=query)
271
272 if response.status_code != 200:
273 return None
274 else:
275 if response.json()["hits"]["total"]["value"] == 0:
276 return {"address": "", "district_name": ""}
277 else:
278 return response.json()["hits"]["hits"][0]["_source"]
279
280
281def build_sql_where(address: Dict[str, str]) -> str:
282 address_parts = address["address"].split()
283 for i, part in enumerate(address_parts):
284 for char in part:
285 if char.isdigit():
286 address_parts[i] = f" {part} "
287 break
288
289 return "%" + "%".join(address_parts) + "%"
290
291
292def get_outages(address: Dict[str, str]) -> List[dict]:
293 where = build_sql_where(address)
294 params = {
295 "where": f"address like '{where}' and (start_date::date <= CURRENT_DATE and end_date::date >= CURRENT_DATE)",
296 "text": "",
297 "geometry": "",
298 "spatialRel": " ",
299 "objectIds": "",
300 "orderByFields": "",
301 "outFields": "",
302 "f": "json",
303 "geometryType": " ",
304 "inSR": "",
305 "groupByFieldsForStatistics": "",
306 "outSR": "",
307 "outStatistics": "",
308 "resultOffset": "",
309 "resultRecordCount": "",
310 "time": "",
311 }
312 response = requests.get(
313 "https://map.novo-sibirsk.ru/elitegis/rest/services/maps/disconnections/MapServer/101/query",
314 params=params,
315 timeout=10,
316 verify=False
317 )
318 response.raise_for_status()
319
320 result = []
321
322 data = response.json()
323
324 for feature in data["features"]:
325 feature_address = feature["value"].replace(",", "")
326 if address["address"] in feature_address:
327 result.append(feature["attributes"])
328
329 if not result:
330 return []
331
332 type_map = {}
333 system_map = {}
334 time_map = {}
335
336 def get_map(code_values: Dict[str, Any]):
337 return {value["code"]: value["name"] for value in code_values}
338
339 for field in data["fields"]:
340 if field["name"] == "type_id":
341 type_map = get_map(field["domain"]["codedValues"])
342 if field["name"] == "system_id":
343 system_map = get_map(field["domain"]["codedValues"])
344 if field["name"] == "time_category":
345 time_map = get_map(field["domain"]["codedValues"])
346
347 actions_map = {
348 "Горячее водоснабжение": "waterWarm",
349 "Холодное водоснабжение и канализация": "waterCold",
350 "Электроснабжение": "electricitySupply",
351 "Газоснабжение": "gasSupply",
352 "Теплоснабжение": "heating",
353 }
354
355 for record in result:
356 record["address"] = (
357 record["address"]
358 .replace(",", "")
359 .replace(f"({record['district_name']})", "")
360 )
361 record["type"] = type_map[record["type_id"]]
362 record["system"] = system_map[record["system_id"]]
363 record["time"] = time_map[record["time_category"]]
364 record["actionS"] = actions_map[system_map[record["system_id"]]]
365 record.pop("oid", "")
366 record.pop("type_id", "")
367 record.pop("system_id", "")
368 record.pop("1", "")
369
370 return result
371
372
373def get_actions_list(actionS: str, replace_water_supply: bool = True) -> List[str]:
374 actionS_list: List[str] = re.split(r"[,\s;]+", actionS)
375
376 if replace_water_supply:
377 if "waterSupply" in actionS_list:
378 actionS_list.remove("waterSupply")
379 if "waterCold" not in actionS_list:
380 actionS_list.append("waterCold")
381 if "waterWarm" not in actionS_list:
382 actionS_list.append("waterWarm")
383
384 return actionS_list
385
386
387def culling_unworthy(outages: List[dict], address: Dict[str, str]) -> List[dict]:
388 worthy = []
389 pattern = re.compile(
390 r"""
391 ^.*? # Имя улицы (любые символы до номера дома)
392 (?P<house> # Группа для номера дома
393 \d+ # Обязательные цифры номера
394 [а-яА-Я]? # Опциональный литер дома (1 буква)
395 (?:/\d+[а-яА-Я]?)? # Опциональная дробь с литером
396 )
397 (?:\s+к(?P<building> # Группа для корпуса
398 \d+ # Обязательные цифры корпуса
399 [а-яА-Я]? # Опциональный литер корпуса
400 (?:/\d+[а-яА-Я]?)? # Опциональная дробь корпуса
401 )?)?$ # Вся группа корпуса необязательна
402 """,
403 re.VERBOSE | re.IGNORECASE,
404 )
405
406 match = pattern.search(address["address"])
407 if match:
408 house = match.group("house")
409 building = match.group("building")
410
411 for outage in outages:
412 o_address = outage["address"]
413 if o_address[-1] == " ":
414 o_address = o_address[:-1]
415 o_match = pattern.search(o_address)
416 if o_match:
417 o_house = o_match.group("house")
418 o_building = o_match.group("building")
419 if o_house == house and o_building == building:
420 worthy.append(outage)
421 else:
422 continue
423
424 if len(worthy) == 0:
425 numbers = re.findall(r"\d+", address["address"])
426 for outage in outages:
427 outage_numbers = re.findall(r"\d+", outage["address"])
428 if len(outage_numbers) == len(numbers) and set(outage_numbers) == set(
429 numbers
430 ):
431 worthy.append(outage)
432
433 worthy = [out for out in outages if out["time"] == "Текущие"]
434
435 if address["address"] in FILTER_STREETS:
436 for exc in FILTER_STREETS[address["address"]]["exclude"]:
437 worthy = [el for el in worthy if exc not in el["address"]]
438
439 return worthy
440
441
442def rename_waterCold(outages: List[dict]) -> List[dict]:
443 for outage in outages:
444 if outage["actionS"] == "waterCold":
445 outage["system"] = "Холодное водоснабжение"
446 return outages
447
448
449def build_info(outages: List[dict]) -> str:
450 # system type date_range
451 # 0 0 0 -> одно отключение
452 # 1 0 0 -> два отключения
453 # 1 1 0 -> два отключения с разными типами
454 # 1 0 1 -> два отключения с разным временем
455 # 1 1 1 -> два отключения с разным типом и временем
456 diff_type = False
457 diff_range_date = False
458
459 o_type = outages[0]["type"]
460 o_start_date = outages[0]["start_date"]
461 o_end_date = outages[0]["end_date"]
462
463 for outage in outages:
464 if outage["start_date"] != o_start_date or outage["end_date"] != o_end_date:
465 diff_range_date = True
466 if outage["type"] != o_type:
467 diff_type = True
468
469 def get_data_range(start_date: int, end_date: int) -> str:
470 months_genitive = {
471 1: "января",
472 2: "февраля",
473 3: "марта",
474 4: "апреля",
475 5: "мая",
476 6: "июня",
477 7: "июля",
478 8: "августа",
479 9: "сентября",
480 10: "октября",
481 11: "ноября",
482 12: "декабря",
483 }
484 start_date = datetime.fromtimestamp(int(str(start_date)[:10]))
485 end_date = datetime.fromtimestamp(int(str(end_date)[:10]))
486 start_month = months_genitive[start_date.month]
487 end_month = months_genitive[end_date.month]
488
489 if start_date.hour == 0 and start_date.minute == 0:
490 start_date_str = start_date.strftime(
491 f"%d {start_month} с двенадцати часов ночи"
492 )
493 else:
494 start_date_str = start_date.strftime(f"%d {start_month} с %H:%M")
495
496 if end_date.hour == 0 and end_date.minute == 0:
497 end_date_str = end_date.strftime(
498 f"по %d {end_month} до двенадцати часов ночи"
499 )
500 else:
501 end_date_str = end_date.strftime(f"по %d {end_month} до %H:%M")
502
503 return f"{start_date_str} {end_date_str}"
504
505 if len(outages) == 1 and not diff_type and not diff_range_date:
506 # {type} {system} {date_range}.
507 result = f"{o_type} {outages[0]['system']} {get_data_range(o_start_date, o_end_date)}"
508
509 elif len(outages) > 1 and not diff_type and not diff_range_date:
510 # если оотключения два
511 # {type} {system1} и {system2} {date_range}.
512 # если отключений больше
513 # {type} {system1}, {system2}, {system3} {date_range}.
514 separator = " и " if len(outages) == 2 else ", "
515 result = f"{o_type} {separator.join([o['system'] for o in outages])} {get_data_range(o_start_date, o_end_date)}"
516
517 elif len(outages) > 1 and diff_type and not diff_range_date:
518 # {type1} {system1}, {type2} {system2} {date_range} ...
519 result = f"""{", ".join([f"{o['type']} {o['system']}" for o in outages])} {get_data_range(o_start_date, o_end_date)}"""
520
521 elif len(outages) > 1:
522 # {type1} {system1} {date_range1}, а также {type2} {system2} {date_range2} ...
523 result = ", а также ".join([
524 f"{o['type']} {o['system']} {get_data_range(o['start_date'], o['end_date'])}"
525 for o in outages
526 ])
527
528 return result
529
530
531def check_outage(outages: List[dict], actionS_list: str) -> Dict[str, str]:
532 target_outages = []
533 other_outages = []
534
535 for outage in outages:
536 if outage["actionS"] in actionS_list:
537 target_outages.append(outage)
538 else:
539 other_outages.append(outage)
540
541 result = {}
542
543 if target_outages:
544 result["result"] = "info_yes"
545 result["info"] = build_info(target_outages)
546 else:
547 result["result"] = "info_no"
548 info_map = {
549 "waterWarm": "Горячее водоснабжение",
550 "waterCold": "Холодное водоснабжение",
551 "electricitySupply": "Электроснабжение",
552 "gasSupply": "Газоснабжение",
553 "heating": "Теплоснабжение",
554 }
555 result["info"] = ", ".join([info_map[a] for a in actionS_list])
556
557 if other_outages:
558 result["result"] = result["result"] + "_other_yes"
559 result["info_other"] = build_info(other_outages)
560 else:
561 result["result"] = result["result"] + "_other_no"
562
563 return result
564
565
566def push_elastic_log(
567 es_host: str,
568 log_data: InputParams,
569 outage_response: Optional[Union[Response, dict]] = None,
570 redis_data: Optional[Dict[str, Any]] = None,
571) -> None:
572 if not redis_data:
573 redis_data = {}
574
575 channel_type = "telephony" if log_data.phone else "text"
576 nsk_offset = timezone(timedelta(hours=7))
577 date = datetime.now(nsk_offset).isoformat()
578
579 if isinstance(outage_response, Response) and outage_response:
580 request = json.loads(outage_response.request.body.decode("utf-8"))
581 elif isinstance(outage_response, dict):
582 request = outage_response
583 else:
584 request = {}
585
586 actions_map = {
587 "waterWarm": "Горячее водоснабжение",
588 "waterCold": "Холодное водоснабжение",
589 "electricitySupply": "Электроснабжение",
590 "gasSupply": "Газоснабжение",
591 "heating": "Теплоснабжение",
592 "waterSupply": "Нет воды",
593 }
594 actions_list = get_actions_list(
595 actionS=log_data.data["actionS"], replace_water_supply=False
596 )
597
598 log = {
599 "dialog_id": log_data.interaction_id,
600 "start_date": date,
601 "end_date": date,
602 "phone": log_data.phone,
603 "actionS": ", ".join([
604 actions_map[a] if a in actions_map else a for a in actions_list
605 ]),
606 "address": log_data.data["address"],
607 "request": json.dumps(request, ensure_ascii=False),
608 "info": log_data.data.get("info", ""),
609 "result": log_data.data["result"],
610 "info_other": redis_data.get("info_other", ""),
611 "channel_type": channel_type,
612 }
613
614 id = hashlib.md5(json.dumps(log).encode("utf-8")).hexdigest()
615
616 requests.post(
617 url=es_host + "/integrations/events/" + id,
618 headers={"Content-Type": "application/json"},
619 json=log,
620 timeout=5,
621 )
622
623
624def proccessing_ending_word(info: str) -> str:
625 exception_words = ["Плановое", "Аварийное", "отключение"]
626 for exception_word in exception_words:
627 bad_word = exception_word[: len(exception_word) - 1] + "$" + exception_word[-1]
628 info = info.replace(exception_word, bad_word)
629 result_info = (
630 info
631 .replace("ие", "ия")
632 .replace("ое", "ого")
633 .replace("ее", "его")
634 .replace("$", "")
635 )
636
637 return result_info
638
639
640def move_word_to_beginning(address: str, word: str) -> str:
641 filtered_address = [w for w in address.split() if w.lower() != word.lower()]
642 result = " ".join(filtered_address)
643
644 return f"{word} {result}".strip()
645
646
647def main() -> None:
648 integrations_config = Config("integrations.conf")
649 es_host = integrations_config.get_param("elasticsearch_host")
650 see_host = integrations_config.get_param("see_host")
651 logpath = integrations_config.get_param("integrations_logpath_mask")
652
653 if DEBUG_MODE:
654 logpath = "./edds_debug.log"
655
656 setup_loggers(logger, level=logging.WARNING, filename=logpath, log_by_day=True)
657
658 io_manager = IOManager(logger=logger)
659 input_params = io_manager.input()
660
661 # second round
662 input_params.data.pop("info_other", "")
663
664 c_address = correct_address(see_host=see_host, address=input_params.data["address"])
665
666 # Может прийти только число, без улицы
667 if c_address.isdigit():
668 push_elastic_log(
669 log_data=input_params,
670 redis_data=input_params.data.copy(),
671 outage_response={"address_before_correct": c_address},
672 es_host=es_host,
673 )
674 return io_manager.output(
675 is_error=True, goto=False, data=input_params.data_from_args
676 )
677
678 for exception_street in EXCEPTION_STREETS:
679 if exception_street in c_address:
680 c_address = c_address.replace(
681 exception_street, EXCEPTION_STREETS[exception_street]
682 )
683 address = normalize_address(es_host=es_host, address=c_address)
684
685 # Пробуем достать адрес из addressS
686 if not address["address"]:
687 c_address = correct_address(
688 see_host=see_host, address=input_params.data["addressS"]
689 )
690
691 # Может прийти только число, без улицы
692 if c_address.isdigit():
693 push_elastic_log(
694 log_data=input_params,
695 redis_data=input_params.data.copy(),
696 outage_response={"address_before_correct": c_address},
697 es_host=es_host,
698 )
699 return io_manager.output(
700 is_error=True, goto=False, data=input_params.data_from_args
701 )
702
703 for exception_street in EXCEPTION_STREETS:
704 if exception_street in c_address:
705 c_address = c_address.replace(
706 exception_street, EXCEPTION_STREETS[exception_street]
707 )
708 address = normalize_address(es_host=es_host, address=c_address)
709
710 if not address["address"]:
711 input_params.data["result"] = "address_not_found"
712 push_elastic_log(
713 log_data=input_params,
714 redis_data=input_params.data.copy(),
715 outage_response={"address_before_correct": c_address},
716 es_host=es_host,
717 )
718 return io_manager.output(
719 is_error=True, goto=False, data=input_params.data_from_args
720 )
721
722 # Адрес для озвучки
723 address_for_voice = address["address"]
724 WORDS_TO_MOVE = ["бульвар", "площадь", "площадь", "поселок", "улица", "территория"]
725
726 for word in WORDS_TO_MOVE:
727 if word in address_for_voice:
728 address_for_voice = move_word_to_beginning(address_for_voice, word)
729
730 input_params.data["address"] = address_for_voice
731 input_params.data_from_args["address"] = address_for_voice
732
733 try:
734 outages = get_outages(address=address)
735 except TimeoutError:
736 input_params.data["result"] = "portal_timeout"
737 return io_manager.output(is_error=True, goto=False, data=input_params.data)
738
739 if len(outages) == 0:
740 input_params.data_from_args["result"] = input_params.data["result"] = (
741 "info_no_other_no"
742 )
743
744 push_elastic_log(
745 log_data=input_params,
746 redis_data=input_params.data.copy(),
747 outage_response={"address": address["address"]},
748 es_host=es_host,
749 )
750
751 return io_manager.output(
752 is_error=False, goto=True, data=input_params.data_from_args
753 )
754
755 actionS_list = get_actions_list(input_params.data["actionS"])
756 actionS_list = [actionS for actionS in actionS_list if actionS in ALLOWED_ACTIONS]
757 outages = culling_unworthy(outages=outages, address=address)
758 outages = rename_waterCold(outages=outages)
759 result = check_outage(outages=outages, actionS_list=actionS_list)
760
761 input_params.data["result"] = result["result"]
762 target_info = proccessing_ending_word(result["info"])
763 input_params.data["info"] = target_info
764
765 if "info_other" in result:
766 other_info = proccessing_ending_word(result["info_other"])
767 input_params.data["info_other"] = other_info
768
769 push_elastic_log(
770 log_data=input_params,
771 redis_data=input_params.data,
772 outage_response={"address": address["address"]},
773 es_host=es_host,
774 )
775
776 return io_manager.output(is_error=False, goto=True, data=input_params.data)
777
778
779main()