Just Save Work

.gitattributes

@@ -0,0 +1 @@
+*.pt filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -76,6 +76,9 @@ target/
 # Jupyter Notebook
 .ipynb_checkpoints
 
+# vscode
+.vscode
+
 # IPython
 profile_default/
 ipython_config.py

checkpoint_model_v0.1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e7a1c0874c88b437e336385e8725be94a58e12d00b8dd8d7860ef48c4ba4f98
+size 255517357

fits_parse.py

@@ -0,0 +1,631 @@
+import re
+import csv
+import logging
+import math
+import glob
+# import argparse
+# import numpy as np
+import os
+import pandas as pd
+import time
+import datetime
+import drms
+import urllib
+# import json
+import matplotlib.pyplot as plt
+import matplotlib.patches as patches
+import astropy.units as u
+import telegram_handler
+# import warnings
+import sunpy.wcs
+import sunpy.map
+import pickle
+import telepot
+from colorlog import ColoredFormatter
+from astropy.coordinates import SkyCoord
+# from astropy.io import fits
+# from astropy.time import Time
+# from datetime import timedelta
+# from sunpy.coordinates import frames
+# from astropy.coordinates import SkyCoord
+from tg_tqdm import tg_tqdm
+# from tqdm import tqdm
+# warnings.filterwarnings("ignore")
+
+
+# define constants
+EMAIL = 'iknyazeva@gmail.com'
+# EMAIL = 'metya.tm@gmail.com'
+SAVE_PATH = 'dataset'
+tg_bot_token = '831964163:AAH7SoaoqWzWIcHaS3yfdmMu-H46hhtUaXw'
+tm_chat_id = 1147194
+ik_chat_id = 94616973
+sun_group_id = -321681009
+DATE_DELIMIT = '2010-06-28'
+TG_LOGGER = False
+FILE_DELETE = False
+LOGGER_LEVEL = logging.WARNING
+# LOGGER_LEVEL = logging.DEBUG
+VERBOSE = True
+PERIOD = 300
+START_DATE = '2018-01-01'
+CROP_DATE = '2017-11-01'
+SLEEP = 0.1
+PROGRESS = 10
+
+
+# logging.basicConfig(filename='futs_parse.log', level=logging.INFO)
+
+
+def set_logger(level=logging.WARNING, name='logger', telegram=False):
+    """Return a logger with a default ColoredFormatter."""
+    file_formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(funcName)s - %(message)s")
+    stream_formatter = ColoredFormatter(
+        "%(asctime)s [%(log_color)s%(levelname)-8s%(reset)s: %(funcName)s] %(white)s%(message)s",
+        datefmt=None,
+        reset=True,
+        log_colors={
+            'DEBUG': 'cyan',
+            'INFO': 'green',
+            'WARNING': 'yellow',
+            'ERROR': 'red',
+            'CRITICAL': 'red',
+        }
+    )
+
+    logger = logging.getLogger(name)
+    stream_handler = logging.StreamHandler()
+    stream_handler.setFormatter(stream_formatter)
+    log_handler = logging.FileHandler("fits_parse.log")
+    log_handler.setFormatter(file_formatter)
+    logger.addHandler(stream_handler)
+    logger.addHandler(log_handler)
+
+    if telegram:
+        tg_handler = telegram_handler.TelegramHandler(tg_bot_token, sun_group_id)
+        tg_formatter = telegram_handler.HtmlFormatter()
+        tg_handler.setFormatter(tg_formatter)
+        logger.addHandler(tg_handler)
+
+    logger.setLevel(level)
+
+    return logger
+
+
+logger = set_logger(level=LOGGER_LEVEL, name='sun_logger', telegram=TG_LOGGER)
+
+
+def check_dataset_directory():
+
+    if not os.path.exists('HMIdataset/fragments'):
+        logger.warning('HMIdataset folders not exist, create them')
+        os.makedirs('HMIdataset/fragments')
+
+    if not os.path.exists('MDIdataset/fragments'):
+        logger.warning('MDIdataset folders not exist, create them')
+        os.makedirs('MDIdataset/fragments')
+
+    return True
+
+
+def clean_folder(path):
+    for file in os.listdir(path):
+        file_path = os.path.join(path, file)
+    if os.path.isfile(file_path):
+        os.remove(file_path)
+
+    return True
+
+
+def message_of_start(token=tg_bot_token, id=sun_group_id):
+    bot = telepot.Bot(token)
+    bot.sendMessage(id, 'Start parsing fits on remote server')
+
+
+def message_of_start_cropping(token=tg_bot_token, id=sun_group_id):
+    bot = telepot.Bot(token)
+    bot.sendMessage(id, '-' * 30)
+    bot.sendMessage(id, 'Start cropping regions')
+    bot.sendMessage(id, '-' * 30)
+
+
+def hook_for_download_fits(t):
+    """Wraps tqdm instance.
+    Don't forget to close() or __exit__()
+    the tqdm instance once you're done with it (easiest using `with` syntax).
+    Example
+    -------
+    >>> with tqdm(...) as t:
+    ...     reporthook = my_hook(t)
+    ...     urllib.urlretrieve(..., reporthook=reporthook)
+    """
+    last_b = [0]
+
+    def update_to(b=1, bsize=1, tsize=None):
+        """
+        b  : int, optional
+            Number of blocks transferred so far [default: 1].
+        bsize  : int, optional
+            Size of each block (in tqdm units) [default: 1].
+        tsize  : int, optional
+            Total size (in tqdm units). If [default: None] remains unchanged.
+        """
+        if tsize is not None:
+            t.total = tsize
+        t.update((b - last_b[0]) * bsize)
+        last_b[0] = b
+
+    return update_to
+
+
+def request_mfits_by_date_MDI(moment, email=EMAIL, path_to_save='MDIdataset', verbose=False):
+    """
+    Function for request fits from JSOC database
+    moment: pd.datetime object
+    return: filepath to the magnetogram
+    """
+
+    filename = 'mdi.fd_m_96m_lev182.' + moment.strftime('%Y%m%d_%H%M%S_TAI.data.fits')
+    filepath = os.path.join(path_to_save, filename)
+
+    if os.path.exists(filepath):
+        pass
+    else:
+
+        c = drms.Client(email=email, verbose=verbose)
+        str_for_query = 'mdi.fd_M_96m_lev182' + moment.strftime('[%Y.%m.%d_%H:%M:%S_TAI]')
+        logger.info('Magnetogram: {} will be downloaded ... '.format(str_for_query))
+        r = c.export(str_for_query, method='url', protocol='fits')
+        logger.debug(r)
+
+        try:
+            r.wait()
+            logger.info(r.request_url)
+        except Exception as e:
+            logger.warning('Can not wait anymore, skip this. Get Exception: {}'.format(e))
+
+        try:
+            logger.info("Download data and save to path {}".format(filepath))
+            r.download(path_to_save, verbose=verbose)
+        except Exception as e:
+            logger.error('Get error while trying download: {}'.format(e))
+            logger.warning('Skip this date')
+
+    return filepath
+
+
+def request_batch_mfits_by_date(moment,
+                                period_of_days=30, email=EMAIL,
+                                path_to_save='dataset',
+                                verbose=False,
+                                type_mag='MDI',
+                                token=tg_bot_token,
+                                chat_id=sun_group_id):
+    '''Request batch fits for a period of days and return:
+    request url
+    period of days that was apply
+    first date of butch
+    last date of batch
+    '''
+
+    c = drms.Client(email=email, verbose=verbose)
+
+    def set_str_for_query(period_of_days=period_of_days):
+        if type_mag == 'MDI':
+            str_for_query = 'mdi.fd_M_96m_lev182' + moment.strftime('[%Y.%m.%d_%H:%M:%S_TAI/{}d@24h]'.format(period_of_days))
+            filename_to_check = 'mdi.fd_m_96m_lev182.' + moment.strftime('%Y%m%d_%H%M%S_TAI.data.fits')
+            path_to_save = 'MDIdataset'
+        if type_mag == 'HMI':
+            str_for_query = 'hmi.m_720s' + moment.strftime('[%Y.%m.%d_%H:%M:%S_TAI/{}d@24h]'.format(period_of_days))
+            path_to_save = 'HMIdataset'
+            filename_to_check = 'hmi.m_720s.' + moment.strftime('%Y%m%d_%H%M%S_TAI.magnetogram.fits')
+
+        return str_for_query, path_to_save, filename_to_check
+
+    str_for_query, path_to_save, filename_to_check = set_str_for_query()
+    logger.debug('{}\n{}\n{}'.format(str_for_query, path_to_save, filename_to_check))
+    if os.path.exists(os.path.join(path_to_save, filename_to_check)):
+        period_of_days = 10
+        logger.info('Files already exists. Skip downloads this batch size of {}'.format(period_of_days))
+        return None, period_of_days, moment, moment + datetime.timedelta(days=period_of_days), period_of_days
+
+    logger.info('Magnetogram: {} will be downloaded ... '.format(str_for_query))
+
+    r = c.export(str_for_query, protocol='fits')
+    logger.debug(r)
+    logger.debug(r.has_failed())
+
+    treshold = round(math.log(period_of_days) ** 2 / 2)
+    while r.has_failed():
+        period_of_days -= round(treshold)
+        if period_of_days < round(treshold / 2):
+            logger.warning('Period of days is too small, skip this request to 10 days')
+            logger.warning('Export request was {}: '.format(str_for_query))
+            period_of_days = 10
+            return None, period_of_days, moment, moment + datetime.timedelta(days=period_of_days), period_of_days
+        time.sleep(1)
+        logger.info('Export request has failed. Reduce number of days in it on {}. Now days in request {}'.format(int(treshold), period_of_days))
+        str_for_query, _, _ = set_str_for_query(period_of_days=period_of_days)
+        logger.debug('Request string: {}'.format(str_for_query))
+        r = c.export(str_for_query, protocol='fits')
+
+    logger.debug(r)
+    logger.debug(len(r.data))
+
+    try:
+        r.wait(sleep=10, retries_notfound=10)
+    except Exception as e:
+        logger.error('Can not wait anymore, skip this. Get Exception: {}'.format(e))
+
+    logger.info("Download data and save to path {}".format(path_to_save))
+
+    first_date_batch = r.urls[0:]['record'].values[0].replace('[', ' ').split()[1].split('_')[0].replace('.', '-')
+    last_date_batch = r.urls[-1:]['record'].values[0].replace('[', ' ').split()[1].split('_')[0].replace('.', '-')
+
+    with tg_tqdm(r.urls.index, token=token, chat_id=chat_id, desc='DOWNLOAD BATCH',
+                 postfix='start_date = {}, end_date = {}'.format(first_date_batch, last_date_batch)) as batch_d:
+        for ind in batch_d:
+            try:
+                # file_name = '.'.join(r.urls.filename[ind].split('.')[:3] + r.urls.filename[ind].split('.')[4:])
+                urllib.request.urlretrieve(r.urls.url[ind], os.path.join(path_to_save, r.urls.filename[ind]))
+            except Exception as e:
+                logger.error('Get error while trying download {}: {}'.format(r.urls.url[ind], repr(e)))
+                logger.warning('Skip this file')
+
+    len_batch = len(r.urls)
+
+    return r.request_url, period_of_days, first_date_batch, last_date_batch, len_batch
+
+
+def request_mfits_by_date_HMI(moment, email=EMAIL, path_to_save='HMIdataset', verbose=False):
+    """
+    Function for request fits from JSOC database
+    moment: pd.datetime object
+    return: filepath to the magnetogram
+    """
+
+    filename = 'hmi.m_720s.' + moment.strftime('%Y%m%d_%H%M%S_TAI.magnetogram.fits')
+    filepath = os.path.join(path_to_save, filename)
+
+    if os.path.exists(filepath):
+        pass
+    else:
+
+        c = drms.Client(email=email, verbose=verbose)
+        str_for_query = 'hmi.m_720s' + moment.strftime('[%Y.%m.%d_%H:%M:%S_TAI]{magnetogram}')
+        logger.info('Magnetogram: {} will be downloaded ... '.format(str_for_query))
+        r = c.export(str_for_query, method='url', protocol='fits')
+        logger.debug(r)
+
+        try:
+            r.wait()
+            logger.info(r.request_url)
+        except Exception as e:
+            logger.warning('Can not wait anymore, skip this. Get Exception: {}'.format(e))
+
+        try:
+            logger.info("Download data and save to path {}".format(filepath))
+            r.download(path_to_save, verbose=verbose)
+        except Exception as e:
+            logger.error('Get error while trying download: {}'.format(e))
+            logger.warning('Skip this date')
+
+    return filepath
+
+
+def read_fits_to_map(filepath, plot_show=False):
+    """
+    read fits to sunpy object and plot in logariphmic scale
+    return
+    mymap: sunpy object
+    """
+
+    mymap = sunpy.map.Map(filepath)
+
+    if plot_show:
+        plt.figure(figsize=(12, 12))
+
+        # data = np.sign(mymap.data)*np.log1p(np.abs(mymap.data))
+        data = mymap.data
+        plt.imshow(data, cmap='gray')
+
+    return mymap
+
+
+def region_coord_list(datestr, sunspots_df, limit_deg=45):
+    """
+    Function for working with sunspot_1996_2017.pkl dataframe,
+    return list of tuples: (datestr, NOAA number, location)
+    used in cropping
+
+    args:
+    datestr: string for date in the format used in dataframe '2001-04-30'
+    sunspots_df: dataframe from file sunspot_1996_2017.pkl
+
+    return: list of tuples
+    """
+
+    date_df = sunspots_df.loc[datestr]
+    date_df.index = date_df.index.droplevel()
+    rc_list = []
+    for index, row in date_df.iterrows():
+        try:
+            restriction_degree = (abs(float(row.location[1:3]) <= limit_deg)) and (abs(float(row.location[4:])) <= limit_deg)
+            if restriction_degree:
+                rc_list.append((pd.to_datetime(datestr, format='%Y-%m-%d'), index, row.location))
+        except ValueError as e:
+            if TG_LOGGER:
+                time.sleep(SLEEP)
+            logger.warning('Some error with read location {} in degree for date {}: {}'.format(row.location, datestr, e))
+        except Exception as e:
+            if TG_LOGGER:
+                time.sleep(SLEEP)
+            logger.error('Some error with read location {} in degree for date {}: {}'.format(row.location, datestr, e))
+
+    return rc_list
+
+
+def return_pixel_from_map(mag_map, record, limit_deg=45):
+    '''
+    convert lon lat coordinate to coordinate in pixel in sun map and return it
+    '''
+
+    pattern = re.compile("[NS]\d{2}[EW]\d{2}")
+    assert bool(pattern.match(record)), 'Pattern should be in the same format as N20E18'
+    assert (abs(float(record[1:3]) <= limit_deg)) and (abs(float(record[4:])) <= limit_deg), 'Consider only regions between -{}, +{} degree'.format(limit_deg)
+    if record[0] == 'N':
+        lat = float(record[1:3])
+    else:
+        lat = -float(record[1:3])
+    if record[3] == 'W':
+        lon = float(record[4:])
+    else:
+        lon = -float(record[4:])
+
+    hpc_coord = sunpy.wcs.convert_hg_hpc(lon, lat, b0_deg=mag_map.meta['crlt_obs'])
+    coord = SkyCoord(hpc_coord[0] * u.arcsec, hpc_coord[1] * u.arcsec, frame=mag_map.coordinate_frame)
+    # pixel_pos = mag_map.world_to_pixel(coord)
+    pixel_pos = mag_map.world_to_pixel(coord) * u.pixel
+    # pixel_pos = pixel_pos.to_value()
+
+    return pixel_pos
+
+
+def crop_regions(mag_map, rc_list, type_mag, delta=100, plot_rec=False, plot_crop=False, limit_deg=45):
+    '''
+    Crop region by size delta and save it to disk,
+    if plot_rec, plot rectangle of regions on disk,
+    if plot_crop, plot only crop regions
+    '''
+
+    # data = np.sign(mag_map.data)*np.log1p(np.abs(mag_map.data))
+    data = mag_map.data
+
+    if type_mag == 'MDI':
+        delta = 100
+    if type_mag == 'HMI':
+        delta = 200
+
+    if plot_rec:
+        fig, ax = plt.subplots(1, figsize=(12, 12))
+        ax.matshow(data)
+        plt.gray()
+        ax.set_title('{} magnetogram at '.format(type_mag) + rc_list[0][0].strftime('%Y-%m-%d %H:%M'))
+
+        for record in rc_list:
+            try:
+                pxs = return_pixel_from_map(mag_map, record[2], limit_deg).to_value()
+            except Exception as e:
+                logger.error('Some error with get pixel coordinates from map: {}. Skip it'.format(e))
+                continue
+            rect = patches.Rectangle((pxs[0] - 1.25 * delta, pxs[1] - delta), 2.5 * delta, 2 * delta, linewidth=3, edgecolor='r', facecolor='none')
+            ax.add_patch(rect)
+            ax.annotate('{}.AR'.format(type_mag) + str(record[1]), xy=(pxs[0], pxs[1]), xytext=(pxs[0], pxs[1] - 50), color='yellow', fontsize='xx-large')
+
+        plt.show()
+
+    submaps = []
+    for record in rc_list:
+
+        filename = '{}.{}.AR{}.fits'.format(type_mag, record[0].strftime('%Y-%m-%d_%H%M%S'), record[1])
+        filepath = os.path.join('{}dataset/fragments'.format(type_mag), filename)
+        try:
+            pxs = return_pixel_from_map(mag_map, record[2], limit_deg)
+        except Exception as e:
+            logger.error('Some error with get pixel coordinates from map: {}. Skip it'.format(e))
+            continue
+        bot_l = [pxs[0] - delta * 1.25 * u.pixel, pxs[1] - delta * u.pixel]
+        top_r = [pxs[0] + delta * 1.25 * u.pixel, pxs[1] + delta * u.pixel]
+
+        submap = mag_map.submap(bot_l * u.pixel, top_r * u.pixel)
+
+        if plot_crop:
+            submap.peek()
+
+        try:
+            submap.save(filepath)
+        except Exception as e:
+            if TG_LOGGER:
+                time.sleep(SLEEP)
+            logger.info('Could not save fits {} cause: {}. Skip it'.format(filename, e))
+
+        submaps.append(submap)
+
+    return submaps
+
+
+def date_compare(date):
+    return date < datetime.datetime.fromtimestamp(time.mktime(time.strptime(DATE_DELIMIT, '%Y-%m-%d')))
+
+
+if __name__ == '__main__':
+
+    check_dataset_directory()
+    message_of_start()
+
+    try:
+        sunspots = pickle.load(urllib.request.urlopen('https://raw.githubusercontent.com/iknyazeva/FitsProcessing/master/sunspot_1996_2017.pkl'))
+        logger.info('Load sunspot dataframe is successful!')
+    except Exception as e:
+        logger.error('Can not load sunspot dataframe, halt parsing! Get Exception: {}'.format(e))
+        raise(e)
+
+    requests_urls = []
+    if START_DATE:
+        try:
+            start_moment = sunspots[(sunspots.index.get_level_values(0) > START_DATE)].index.get_level_values(0)[0]
+        except IndexError as e:
+            logger.info('Index out of bound. Possibly the table is ended: {}'.format(e))
+            start_moment = START_DATE
+        except Exception as e:
+            logger.error('Some error then get start_moment for first iteration: {}'.format(e))
+    else:
+        start_moment = sunspots.index.get_level_values(0)[0]
+    logger.debug(start_moment)
+    count_of_days_left = len(sunspots[(sunspots.index.get_level_values(0) >= start_moment)].groupby(level=0))
+    logger.debug(count_of_days_left)
+
+    with tg_tqdm(sunspots[(sunspots.index.get_level_values(0) > start_moment)].groupby(level=0),
+                 token=tg_bot_token, chat_id=sun_group_id, desc='MAIN PROGRESS DOWNLOAD') as tgm:
+        number_batch = 1
+        while count_of_days_left > 0:
+            tgm.set_postfix(batch=number_batch)
+            if date_compare(start_moment):
+                request_url,\
+                    period_of_days,\
+                    first_date_batch,\
+                    last_date_batch,\
+                    len_batch = request_batch_mfits_by_date(start_moment, period_of_days=PERIOD,
+                                                            email=EMAIL, type_mag='MDI', verbose=VERBOSE)
+            else:
+                request_url,\
+                    period_of_days,\
+                    first_date_batch,\
+                    last_date_batch,\
+                    len_batch = request_batch_mfits_by_date(start_moment, period_of_days=PERIOD,
+                                                            email=EMAIL, type_mag='HMI', verbose=VERBOSE)
+
+            logger.debug('Returned period of days {}'.format(period_of_days))
+            # requests_urls.append(request_url)
+            try:
+                start_moment = sunspots[(sunspots.index.get_level_values(0) > last_date_batch)].index.get_level_values(0)[0]
+            except IndexError as e:
+                logger.info('Index out of bound. Possibly the table is ended: {}'.format(e))
+            except Exception as e:
+                logger.error('Some error then get start_moment for next iteration: {}'.format(e))
+            count_of_days_left = len(sunspots[(sunspots.index.get_level_values(0) >= start_moment)])
+            number_batch += 1
+
+            with open('requests_urls.csv', 'a', newline='') as file:
+                csv.writer(file).writerow(request_url)
+
+            tgm.update(len_batch)
+
+    # with open('requests_urls.csv', 'w') as file:
+    #     csv.writer(file, delimiter='\n').writerow(requests_urls)
+
+    message_of_start_cropping()
+
+    if CROP_DATE:
+        crop_df = sunspots[(sunspots.index.get_level_values(0) > CROP_DATE)]
+    else:
+        crop_df = sunspots
+    with tg_tqdm(range(1), tg_bot_token, sun_group_id,
+                 total=len(crop_df.groupby(level=0)), desc='CROPPING PROGRESS') as tgt:
+
+        def is_progress(acc, total, progress=PROGRESS, tqdm_instanse=tgt):
+            if (acc % PROGRESS == 0):
+                logger.debug('In if acc = {}'.format(acc))
+                time.sleep(SLEEP)
+                tgt.update(PROGRESS)
+            elif (acc >= total):
+                logger.debug('In if acc = {}'.format(acc))
+                time.sleep(SLEEP)
+                tgt.update(total % PROGRESS)
+
+            return True
+
+        acc = 0
+        total = len(crop_df.groupby(level=0))
+        logger.debug(total)
+        for date, df in crop_df.groupby(level=0):
+
+            rc_list = region_coord_list(str(date), df, limit_deg=45)
+
+            if not rc_list:
+                acc += 1
+                time.sleep(SLEEP)
+                is_progress(acc, total)
+                logger.debug('rc_list is empty - {}, acc = {}'.format(rc_list, acc))
+                continue
+
+            if date_compare(date):
+                filename = 'mdi.fd_m_96m_lev182.' + date.strftime('%Y%m%d_%H%M%S_TAI') + '*.fits'
+                path = 'MDIdataset/'
+                try:
+                    filepath = glob.glob(path + filename)[0]
+                    if TG_LOGGER:
+                        time.sleep(SLEEP)
+                    logger.debug('filepath: {}'.format(filepath))
+                except IndexError as e:
+                    logger.info('File with this date {} is not exist'.format(str(date)))
+                    acc += 1
+                    is_progress(acc, total)
+                    continue
+                except Exception as e:
+                    logger.error('Some error with glob:'.format(e))
+                    acc += 1
+                    is_progress(acc, total)
+                    continue
+                type_mag = 'MDI'
+
+            else:
+                filename = 'hmi.m_720s.' + date.strftime('%Y%m%d_%H%M%S_TAI') + '*.fits'
+                path = 'HMIdataset/'
+                try:
+                    filepath = glob.glob(path + filename)[0]
+                    if TG_LOGGER:
+                        time.sleep(SLEEP)
+                    logger.debug('filepath: {}'.format(filepath))
+                except IndexError as e:
+                    if TG_LOGGER:
+                        time.sleep(SLEEP)
+                    logger.info('File with this date {} is not exist'.format(str(date)))
+                    acc += 1
+                    is_progress(acc, total)
+                    continue
+                except Exception as e:
+                    if TG_LOGGER:
+                        time.sleep(SLEEP)
+                    logger.error('Some error with glob:'.format(e))
+                    acc += 1
+                    is_progress(acc, total)
+                    continue
+                type_mag = 'HMI'
+
+            try:
+                sun_map = read_fits_to_map(filepath, plot_show=False)
+                crop_regions(sun_map, rc_list, plot_rec=False, plot_crop=False, type_mag=type_mag)
+            except ValueError as e:
+                if TG_LOGGER:
+                    time.sleep(SLEEP)
+                logger.info('Get Exception while reading: {}'.format(e))
+                logger.info('Doing active farther, skip it.')
+                # acc += 1
+                # continue
+            except Exception as e:
+                if TG_LOGGER:
+                    time.sleep(SLEEP)
+                logger.error('Get Exception while reading: {}'.format(e))
+                logger.warning('Doing active farther, skip it.')
+                # acc += 1
+                # continue
+
+            # tgt.update()
+            acc += 1
+            logger.debug('acc = {}'.format(acc))
+            is_progress(acc, total)
+
+    if FILE_DELETE:
+        clean_folder('MDIdataset')
+        clean_folder('HMIdataset')

telega.ipynb

@@ -0,0 +1,1027 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 83,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import tqdm\n",
+    "import telepot\n",
+    "from tg_tqdm import tg_tqdm\n",
+    "import time\n",
+    "import requests"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "tg_token = '831964163:AAH7SoaoqWzWIcHaS3yfdmMu-H46hhtUaXw'\n",
+    "tg_chat_id = 1147194\n",
+    "ik_chat_id = 94616973\n",
+    "sun_group_id = -321681009"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "bot = telepot.Bot(tg_token)\n",
+    "# bot.getUpdates()\n",
+    "# message = bot.sendMessage(chat_id=sun_group_id, text='Я счас запиню этот чат, попробуем!')\n",
+    "bot\n",
+    "send_message_url = \"https://api.telegram.org/bot{}/sendMessage\".format(tg_token)\n",
+    "pin_message_url = \"https://api.telegram.org/bot{}/pinChatMessage\".format(tg_token)\n",
+    "media_message_url = \"https://api.telegram.org/bot{}/sendPhoto\".format(tg_token)\n",
+    "change_media_url = \"https://api.telegram.org/bot{}/editMessageMedia\".format(tg_token)\n",
+    "\n",
+    "# send_responce = requests.get(send_message_url, params={'chat_id':tg_chat_id, 'text':'THIS IS WILL BE PIN'})\n",
+    "# send_responce\n",
+    "\n",
+    "# pin_responce = requests.get(pin_message_url, params={'chat_id':send_responce.json()['result']['chat']['id'],\n",
+    "#                                                           'message_id':send_responce.json()['result']['message_id']}).text\n",
+    "# pin_responce\n",
+    "\n",
+    "# with open('image.png', 'rb') as photo:\n",
+    "#     media_responce = requests.post(media_message_url, data={'chat_id':tg_chat_id}, files={'photo':photo})\n",
+    "# media_responce\n",
+    "\n",
+    "with open('image2.png', 'rb') as photo:\n",
+    "    change_responce = requests.post(change_media_url,\n",
+    "                                    data={\n",
+    "                                        'chat_id': media_responce.json()['result']['chat']['id'],\n",
+    "                                        'message_id': media_responce.json()['result']['message_id'],\n",
+    "                                        'media': '{\"type\": \"photo\", \"media\": \"attach://image2.png\"}'\n",
+    "                                    }, headers={\"Content-Type\": \"multipart/form-data\"})\n",
+    "                                        \n",
+    "change_responce\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 338,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "6445"
+      ]
+     },
+     "execution_count": 338,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "args = {'data':{'media':None}}\n",
+    "from json import JSONEncoder\n",
+    "import json\n",
+    "media = {}\n",
+    "# file = ['image.png']\n",
+    "# type = 'photo'\n",
+    "media = {'type':'photo', 'media': 'attach://image2.png'}\n",
+    "# JSONEncoder().encode(media)\n",
+    "# args['data']['media'] = media\n",
+    "# args\n",
+    "media = JSONEncoder().encode(media)\n",
+    "media\n",
+    "media_responce.json()['result']['message_id']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 257,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'ok': False,\n",
+       " 'error_code': 400,\n",
+       " 'description': \"Bad Request: can't parse InputMedia: Can't parse media JSON object\"}"
+      ]
+     },
+     "execution_count": 257,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# media_responce.json()\n",
+    "change_responce.json()\n",
+    "# ??requests.Session"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# telepot.Bot.\n",
+    "bot.sendMessage(ik_chat_id, \"Hi! Why in English? Let's talk in Russian. Потому что я ж русский бот, пусть и курва!\")\n",
+    "bot.sendMessage(ik_chat_id, \"Я бот Тимура, который создан для того, чтобы следить за обученим и всякими длительными процессами.\\\n",
+    "                            Так что сюда буду присылать всякие обновления, что Тимур посчитатет нужным вам прислать.\")\n",
+    "bot.sendMessage(chat_id=ik_chat_id, text=\"Потому что curve занято и скучно)\", reply_to_message_id=41)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from telepot.loop import MessageLoop\n",
+    "import telepot \n",
+    "bot = telepot.Bot(tg_token)\n",
+    "# msg = bot.sendMessage(tg_chat_id, 'pidor', disable_notification=True)\n",
+    "# telepot.Bot(tg_token).editMessageText((tg_chat_id, id), 'suka')\n",
+    "# telepot.telepot.loop.MessageLoop.cancel()\n",
+    "# with open('im.png', 'rb') as file:\n",
+    "#     pic_msg = bot.sendPhoto(tg_chat_id,  file)\n",
+    "pic_msg\n",
+    "bot.ed"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# bot._api_request('edit')\n",
+    "# bot._api_request('sendMessage', params={'chat_id':tg_chat_id, 'text':'test'})\n",
+    "# msg_id = 37\n",
+    "# with open('image.png.png', 'rb') as photo:\n",
+    "#     pic_msg = bot._api_request_with_file('sendPhoto', {'chat_id':tg_chat_id}, 'photo', photo)\n",
+    "# pic_msg\n",
+    "# bot._api_request('editMessageText', {'chat_id':tg_chat_id, 'message_id': msg_id, 'text':'test_edited'})\n",
+    "with open('image2.png') as file:\n",
+    "    bot._api_request_with_file('editMessageMedia', {'chat_id':media_responce.json()['result']['chat']['id'], \n",
+    "                                                    'message_id': media_responce.json()['result']['message_id']}, \n",
+    "                               'media', {'type': 'photo', 'media':'image2.png'})\n",
+    "# bot._api_request_with_file('editMessageMedia', {'chat_id':tg_chat_id, 'message_id': msg_id}, 'media', 'attach://imf.png')\n",
+    "# with open('im.png', 'rb') as f1: \n",
+    "#     with open('imf.png', 'rb') as f2:\n",
+    "#         bot.sendMediaGroup(tg_chat_id, ('', f1))\n",
+    "def editMessageMedia(bot, msg_identifier, media,\n",
+    "                     reply_markup=None):\n",
+    "    \"\"\"\n",
+    "    See: https://core.telegram.org/bots/api#editmessagemedia\n",
+    "\n",
+    "    :param msg_identifier: Same as ``msg_identifier`` in :meth:`telepot.Bot.editMessageText`\n",
+    "\n",
+    "    :param media:\n",
+    "        Same as ``media`` in :meth:`telepot.Bot.sendMediaGroup`, except that here is a single\n",
+    "        `InputMedia <https://core.telegram.org/bots/api#inputmedia>`_ object, not an array.\n",
+    "    \"\"\"\n",
+    "    p,f = bot._split(locals(), discard=['msg_identifier', 'media'])\n",
+    "    p.update(bot._dismantle_message_identifier(msg_identifier))\n",
+    "\n",
+    "    legal_media, files_to_attach = bot._split_input_media_array([media])\n",
+    "    p['media'] = legal_media[0]\n",
+    "\n",
+    "    return bot._api_request('editMessageMedia', bot._rectify(p), files_to_attach)\n",
+    "editMessageMedia(bot, (:media_responce.json()['result']['chat']['id'], media_responce.json()['result']['message_id'] ))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import time\n",
+    "from tg_tqdm import tg_tqdm\n",
+    "\n",
+    "for _ in tqdm.tqdm(range(5), leave=True, unit_scale = True, desc='MAIN PROGRESS'):\n",
+    "    time.sleep(.5)\n",
+    "#     tqdm.tqdm.write('pidor')\n",
+    "#     print('pidor')\n",
+    "\n",
+    "# for i in tqdm.tqdm(desc=)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "del(tg_tqdm)\n",
+    "from tg_tqdm import tg_tqdm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for _ in tg_tqdm(range(5),tg_token, [ik_chat_id, tg_chat_id], decs='MAIN PROGRESS'):\n",
+    "    time.sleep(1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 277,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "ValueError",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
+      "\u001b[1;32m<ipython-input-277-21f5887b43a5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'image.png'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'rb'\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mphoto\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[0mbot\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msendMediaGroup\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmedia_responce\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjson\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'result'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'chat'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'id'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmedia\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'photo'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mphoto\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
+      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\telepot\\__init__.py\u001b[0m in \u001b[0;36msendMediaGroup\u001b[1;34m(self, chat_id, media, disable_notification, reply_to_message_id)\u001b[0m\n\u001b[0;32m    645\u001b[0m         \"\"\"\n\u001b[0;32m    646\u001b[0m         \u001b[0mp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_strip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlocals\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmore\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'media'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 647\u001b[1;33m         \u001b[0mlegal_media\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfiles_to_attach\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_split_input_media_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmedia\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    648\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    649\u001b[0m         \u001b[0mp\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'media'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlegal_media\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\telepot\\__init__.py\u001b[0m in \u001b[0;36m_split_input_media_array\u001b[1;34m(media_array)\u001b[0m\n\u001b[0;32m    281\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mm\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    282\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 283\u001b[1;33m     \u001b[0mms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mensure_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mm\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mm\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mmedia_array\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    284\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    285\u001b[0m     \u001b[0mused_names\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mgiven_attach_name\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mm\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mm\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mms\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mgiven_attach_name\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mm\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\telepot\\__init__.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[1;34m(.0)\u001b[0m\n\u001b[0;32m    281\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mm\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    282\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 283\u001b[1;33m     \u001b[0mms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mensure_dict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mm\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mm\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mmedia_array\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    284\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    285\u001b[0m     \u001b[0mused_names\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0mgiven_attach_name\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mm\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mm\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mms\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0mgiven_attach_name\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mm\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\telepot\\__init__.py\u001b[0m in \u001b[0;36mensure_dict\u001b[1;34m(input_media)\u001b[0m\n\u001b[0;32m    244\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0minput_media\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    245\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 246\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    247\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    248\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mgiven_attach_name\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minput_media\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;31mValueError\u001b[0m: "
+     ]
+    }
+   ],
+   "source": [
+    "with open('image.png', 'rb') as photo:\n",
+    "    bot.sendMediaGroup(media_responce.json()['result']['chat']['id'], media=('photo', photo))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with tqdm.tqdm_notebook(total=5) as t:\n",
+    "    for i in [1,2,3,4,5,6]:\n",
+    "        time.sleep(1)\n",
+    "        t.update(3)\n",
+    "        t.write('pidor')\n",
+    "        \n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import random\n",
+    "with tqdm.tnrange(10) as t:\n",
+    "    for i in t:\n",
+    "        # Description will be displayed on the left\n",
+    "        t.set_description('GEN %i' % i)\n",
+    "        # Postfix will be displayed on the right,\n",
+    "        # formatted automatically based on argument's datatype\n",
+    "        t.set_postfix(loss=random.random(), gen=random.randint(1,999), str='h',\n",
+    "                      lst=[1, 2])\n",
+    "        time.sleep(0.1)\n",
+    "\n",
+    "with tqdm.tqdm_notebook(total=10, bar_format=\"{postfix[0]} {postfix[1][value]:>8.2g}\",\n",
+    "          postfix=[\"Batch\", dict(value=0)]) as t:\n",
+    "    for i in range(10):\n",
+    "        time.sleep(0.1)\n",
+    "        t.postfix[1][\"value\"] = i / 2\n",
+    "        t.update()\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "bot.sendMessage(chat_id=ik_chat_id, chat_id=tg_chat_id, 'test')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 110,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from tg_tqdm import tg_tqdm\n",
+    "with tg_tqdm(range(5), token=tg_token,chat_id=tg_chat_id, desc='DOWNLOAD', dynamic_ncols=False) as t:\n",
+    "    for i in t:\n",
+    "        time.sleep(1)\n",
+    "        t.set_postfix(dom='fon')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 114,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "71c982004313486baf488d8cfea222e4",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, description='Download', max=5, style=ProgressStyle(description_width='init…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "with tqdm.tqdm_notebook(desc='Download', total=5, postfix='pidor', dynamic_ncols=0) as t:\n",
+    "    for i in range(3):\n",
+    "        \n",
+    "#         t.display(msg='display!!!', pos=30)\n",
+    "        time.sleep(1)\n",
+    "        t.set_postfix(govno='pidor')\n",
+    "        t.update(i)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "e80ec189f8134916a7c1798a2f410811",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "import tqdm\n",
+    "t = tqdm.tqdm_notebook(range(10))\n",
+    "\n",
+    "for i in t:\n",
+    "    time.sleep(.1)\n",
+    "    t.set_postfix_str('podor {}'.format(i*i))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 40,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "GEN 10: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 10/10 [00:30<00:00,  3.02s/it, gen=301, loss=0.199, lst=[1, 2], str=h]\n"
+     ]
+    }
+   ],
+   "source": [
+    "import random\n",
+    "del(t)\n",
+    "with tqdm.tqdm(range(10)) as t:\n",
+    "    for i in t:\n",
+    "        # Description will be displayed on the left\n",
+    "        t.set_description('GEN {}'.format(i+1))\n",
+    "        # Postfix will be displayed on the right,\n",
+    "        # formatted automatically based on argument's datatype\n",
+    "        t.set_postfix(loss=random.random(), gen=random.randint(1,999), str='h',\n",
+    "                      lst=[1, 2])\n",
+    "        time.sleep(3)\n",
+    "#         t.update()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "19e323a835ab41f484552bd952242e1f",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, max=10), HTML(value='')))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "with tqdm.tqdm_notebook(total=10, bar_format=\"{postfix[0]} {postfix[1][value]:>8.2g}\",\n",
+    "          postfix=[\"Batch\", dict(value=0)]) as t:\n",
+    "    for i in range(10):\n",
+    "        time.sleep(0.5)\n",
+    "#         t.postfix[0] = 'go?'\n",
+    "        t.postfix[1][\"value\"] = i / 2\n",
+    "        t.update(1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'tg_logger' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "\u001b[1;32m<ipython-input-15-7c6b30455058>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;32mdel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtg_logger\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[1;32mdel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlogging\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mdel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlogger_tg\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mpython_telegram_logger\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mtg_logger\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[1;32mimport\u001b[0m \u001b[0mlogging\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
+      "\u001b[1;31mNameError\u001b[0m: name 'tg_logger' is not defined"
+     ]
+    }
+   ],
+   "source": [
+    "del(tg_logger)\n",
+    "del(logging)\n",
+    "del(logger_tg)\n",
+    "import python_telegram_logger as tg_logger\n",
+    "import logging\n",
+    "\n",
+    "logger_tg = logging.getLogger('tg_test')\n",
+    "tg_handler = tg_logger.Handler(tg_token, [sun_group_id])\n",
+    "tg_handler.setFormatter(\n",
+    "    tg_logger.HTMLFormatter({\"fmt\": \" *%(levelname)s* _%(name)s : %(funcName)s_\"})\n",
+    ")\n",
+    "logger_tg.addHandler(tg_handler)\n",
+    "logger_tg.setLevel(logging.INFO)\n",
+    "\n",
+    "\n",
+    "logger_tg.info('PIDOR')\n",
+    "time.sleep(3)\n",
+    "logger_tg.error('KURVA')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 116,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import tqdm\n",
+    "import urllib\n",
+    "def hook(t):\n",
+    "    \"\"\"Wraps tqdm instance.\n",
+    "    Don't forget to close() or __exit__()\n",
+    "    the tqdm instance once you're done with it (easiest using `with` syntax).\n",
+    "    Example\n",
+    "    -------\n",
+    "    >>> with tqdm(...) as t:\n",
+    "    ...     reporthook = my_hook(t)\n",
+    "    ...     urllib.urlretrieve(..., reporthook=reporthook)\n",
+    "    \"\"\"\n",
+    "    last_b = [0]\n",
+    "\n",
+    "    def update_to(b=1, bsize=1, tsize=None):\n",
+    "        \"\"\"\n",
+    "        b  : int, optional\n",
+    "            Number of blocks transferred so far [default: 1].\n",
+    "        bsize  : int, optional\n",
+    "            Size of each block (in tqdm units) [default: 1].\n",
+    "        tsize  : int, optional\n",
+    "            Total size (in tqdm units). If [default: None] remains unchanged.\n",
+    "        \"\"\"\n",
+    "        if tsize is not None:\n",
+    "            t.total = tsize\n",
+    "        t.update((b - last_b[0]) * bsize)\n",
+    "        last_b[0] = b\n",
+    "\n",
+    "    return update_to\n",
+    "    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 119,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "url = 'http://jsoc.stanford.edu'\n",
+    "urlfits = url+'/SUM7/D157434620/S00007/magnetogram.fits'\n",
+    "urlfits\n",
+    "def tg_down():\n",
+    "    with tg_tqdm(range(1), tg_token, tg_chat_id) as t:\n",
+    "        myhook = hook(t)\n",
+    "        urllib.request.urlretrieve(urlfits, 'test', reporthook=myhook)\n",
+    "def down():\n",
+    "    with tqdm.tqdm(range(1)) as t:\n",
+    "        myhook = hook(t)\n",
+    "        urllib.request.urlretrieve(urlfits, 'test', reporthook=myhook)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 53,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.003"
+      ]
+     },
+     "execution_count": 53,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "3e-3"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 162,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with tg_tqdm(urls_store, tg_token, tg_chat_id) as t:\n",
+    "    for _ in urls_store.record:\n",
+    "        time.sleep(0.01)\n",
+    "        t.update()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 174,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "record\n",
+      "filename\n",
+      "url\n"
+     ]
+    }
+   ],
+   "source": [
+    "for _ in urls_store.columns:\n",
+    "    print(_)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 184,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "744498f5560c4e318a14bd9893f5fc60",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, max=340), HTML(value='')))"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "for i in tqdm.tqdm_notebook(urls_store.index):\n",
+    "#     print(urls_store.loc[i][1])\n",
+    "    time.sleep(0.1)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 258,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "ZeroDivisionError('division by zero')\n",
+      "ZeroDivisionError('division by zero')\n",
+      "ZeroDivisionError('division by zero')\n",
+      "ZeroDivisionError('division by zero')\n"
+     ]
+    }
+   ],
+   "source": [
+    "for tg_tqdm(range(1000), tg_token, tg_chat_id):\n",
+    "    time.sleep(0.1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 310,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1\n",
+      "2\n",
+      "3\n",
+      "4\n",
+      "5\n",
+      "6\n",
+      "7\n"
+     ]
+    }
+   ],
+   "source": [
+    "with tg_tqdm(range(1), token=tg_token, chat_id=tg_chat_id, total=len(urls_store), postfix='batch') as t:\n",
+    "    lng = int(len(urls_store))\n",
+    "    r = [i for i in range(lng)]\n",
+    "    acc=1\n",
+    "    while len(r) > 0:\n",
+    "        ran = random.randint(40, 70)\n",
+    "        r = r[ran:]\n",
+    "        time.sleep(2)\n",
+    "        print(acc)\n",
+    "        t.update(ran)\n",
+    "        t.set_postfix(batch=acc)\n",
+    "        acc += 1\n",
+    "#     print(ran)    \n",
+    "    \n",
+    "    t.update(ran)\n",
+    "        "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "with tg_tqdm(urls_store.index[1:4], token=tg_token, chat_id=tg_chat_id, desc='DOWNLOAD BATCH') as batch_d:\n",
+    "    for ind in [1, 2, 3, 4]:\n",
+    "        with tg_tqdm(urls_store.index[1:4], token=tg_token, chat_id=tg_chat_id, desc='DOWNLOAD FITS') as url_d:\n",
+    "            myhook = hook(url_d)\n",
+    "            try:\n",
+    "                urllib.request.urlretrieve(urls_store.loc[ind][2], os.path.join('', urls_store.loc[ind][1]), reporthook=myhook)\n",
+    "            except Exception as e:\n",
+    "                print('Get error while trying download {}: {}'.format(urls_store.loc[ind][1], e))\n",
+    "                print('Skip this date')\n",
+    "        batch_d.update(i)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 271,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "25596b764a8a411d970404953be3ba57",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "HBox(children=(IntProgress(value=0, description='DOWNLOAD BATCH', max=3, style=ProgressStyle(description_width…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "\n"
+     ]
+    }
+   ],
+   "source": [
+    "with tqdm.tqdm_notebook(urls_store.index[1:4], desc='DOWNLOAD BATCH') as batch_d:\n",
+    "    for ind in batch_d:\n",
+    "        try:\n",
+    "            urllib.request.urlretrieve(urls_store.loc[ind][2], os.path.join('', urls_store.loc[ind][1]))\n",
+    "        except Exception as e:\n",
+    "            print('Get error while trying download {}: {}'.format(urls_store.loc[ind][1], e))\n",
+    "            print('Skip this date')\n",
+    "#         batch_d.update(i)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py:162: FutureWarning: the 'labels' keyword is deprecated, use 'codes' instead\n",
+      "  return cls.__new__(cls, **d)\n"
+     ]
+    }
+   ],
+   "source": [
+    "import pickle\n",
+    "import urllib\n",
+    "sunspots = pickle.load(urllib.request.urlopen('https://raw.githubusercontent.com/iknyazeva/FitsProcessing/master/sunspot_1996_2017.pkl'))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 38,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "24\n",
+      "rc_list is empty - [], acc = 1\n",
+      "rc_list is empty - [], acc = 2\n",
+      "File with this date 2017-11-17 00:00:00 is not exist\n",
+      "3\n",
+      "rc_list is empty - [], acc = 4\n",
+      "File with this date 2017-11-19 00:00:00 is not exist\n",
+      "5\n",
+      "rc_list is empty - [], acc = 6\n",
+      "File with this date 2017-11-27 00:00:00 is not exist\n",
+      "7\n",
+      "rc_list is empty - [], acc = 8\n",
+      "rc_list is empty - [], acc = 9\n",
+      "File with this date 2017-11-30 00:00:00 is not exist\n",
+      "10\n",
+      "File with this date 2017-12-01 00:00:00 is not exist\n",
+      "11\n",
+      "File with this date 2017-12-07 00:00:00 is not exist\n",
+      "12\n",
+      "rc_list is empty - [], acc = 13\n",
+      "File with this date 2017-12-11 00:00:00 is not exist\n",
+      "14\n",
+      "File with this date 2017-12-12 00:00:00 is not exist\n",
+      "15\n",
+      "File with this date 2017-12-13 00:00:00 is not exist\n",
+      "16\n",
+      "rc_list is empty - [], acc = 17\n",
+      "rc_list is empty - [], acc = 18\n",
+      "rc_list is empty - [], acc = 19\n",
+      "File with this date 2017-12-24 00:00:00 is not exist\n",
+      "20\n",
+      "File with this date 2017-12-25 00:00:00 is not exist\n",
+      "21\n",
+      "rc_list is empty - [], acc = 22\n",
+      "File with this date 2017-12-27 00:00:00 is not exist\n",
+      "23\n",
+      "File with this date 2017-12-28 00:00:00 is not exist\n",
+      "24\n"
+     ]
+    }
+   ],
+   "source": [
+    "import random\n",
+    "import glob\n",
+    "TG =True\n",
+    "SLEEP = 1\n",
+    "crop_df = sunspots[(sunspots.index.get_level_values(0) > '2017-11-01')]\n",
+    "with tg_tqdm(range(1), tg_token, tg_chat_id,\n",
+    "             total=len(crop_df.groupby(level=0)), desc='CROPPING PROGRESS') as tgt:\n",
+    "\n",
+    "    acc = 0\n",
+    "    total = len(crop_df.groupby(level=0))\n",
+    "    print(total)\n",
+    "    for date, df in crop_df.groupby(level=0):\n",
+    "\n",
+    "        if random.randint(0,1):\n",
+    "            rc_list = []\n",
+    "        else:\n",
+    "            rc_list = 'Something'\n",
+    "\n",
+    "        if not rc_list:\n",
+    "            acc += 1\n",
+    "            time.sleep(SLEEP)\n",
+    "            tgt.update(acc)\n",
+    "            print('rc_list is empty - {}, acc = {}'.format(rc_list, acc))\n",
+    "            continue\n",
+    "\n",
+    "        filename = 'hmi.m_720s.' + date.strftime('%Y%m%d_%H%M%S_TAI') + '*.fits'\n",
+    "        path = 'HMIdataset/'\n",
+    "        try:\n",
+    "            filepath = glob.glob(path + filename)[0]\n",
+    "            if TG:\n",
+    "                time.sleep(SLEEP)\n",
+    "            print('filepath: {}'.format(filepath))\n",
+    "        except IndexError as e:\n",
+    "            if TG:\n",
+    "                time.sleep(SLEEP)\n",
+    "            print('File with this date {} is not exist'.format(str(date)))\n",
+    "            acc += 1\n",
+    "            print(acc)\n",
+    "            continue\n",
+    "        except Exception as e:\n",
+    "            if TG:\n",
+    "                time.sleep(SLEEP)\n",
+    "            print('Some error with glob:'.format(e))\n",
+    "            acc += 1\n",
+    "            print(acc)\n",
+    "            continue\n",
+    "        type_mag = 'HMI'\n",
+    "\n",
+    "        try:\n",
+    "            print('Reading filepath', filepath)\n",
+    "            print('Croping file', filepath)\n",
+    "        except ValueError as e:\n",
+    "            if TG:\n",
+    "                time.sleep(SLEEP)\n",
+    "            print('Get Exception while reading: {}'.format(e))\n",
+    "            print('Doing active farther, skip it.')\n",
+    "            # acc += 1\n",
+    "            # continue\n",
+    "        except Exception as e:\n",
+    "            if TG:\n",
+    "                time.sleep(SLEEP)\n",
+    "            print('Get Exception while reading: {}'.format(e))\n",
+    "            print('Doing active farther, skip it.')\n",
+    "            # acc += 1\n",
+    "            # continue\n",
+    "\n",
+    "        # tgt.update()\n",
+    "        acc += 1\n",
+    "        print('acc = {}'.format(acc))\n",
+    "        if (acc % PROGRESS == 0) or (acc >= total):\n",
+    "            print('In if acc = {}'.format(acc))\n",
+    "            time.sleep(SLEEP)\n",
+    "            tgt.update(10)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 81,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "1\n",
+      "2\n",
+      "3\n",
+      "4\n",
+      "5\n",
+      "6\n",
+      "7\n",
+      "8\n",
+      "9\n",
+      "10\n",
+      "is this!\n",
+      "11\n",
+      "12\n",
+      "13\n",
+      "14\n",
+      "15\n",
+      "16\n",
+      "17\n",
+      "18\n",
+      "19\n",
+      "20\n",
+      "is this!\n",
+      "21\n",
+      "22\n",
+      "23\n",
+      "24\n",
+      "is end!\n"
+     ]
+    }
+   ],
+   "source": [
+    "# del(tg_tqdm)\n",
+    "import telegram_tqdm\n",
+    "# from telegram_tqdm import tg_tqdm\n",
+    "from importlib import reload\n",
+    "reload(telegram_tqdm)\n",
+    "\n",
+    "\n",
+    "with telegram_tqdm.tg_tqdm(token=tg_token, chat_id=tg_chat_id, #iterable=None,\n",
+    "             total=24, desc='CROPPING PROGRESS') as tgt:\n",
+    "    for i in range(1,25):\n",
+    "        print(i)\n",
+    "        time.sleep(1)\n",
+    "        if (i % 10 == 0):\n",
+    "            print('is this!')\n",
+    "            tgt.update(10)\n",
+    "        elif (i == 24):\n",
+    "            print('is end!')\n",
+    "            tgt.update(24 % 10)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "00:10 in total: 100%|0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000| 10/10 [00:10<00:00,  1.00s/it]\n"
+     ]
+    }
+   ],
+   "source": [
+    "import time\n",
+    "from tqdm import tqdm\n",
+    "class TqdmExtraFormat(tqdm):\n",
+    "    \"\"\"Provides a `total_time` format parameter\"\"\"\n",
+    "    @property\n",
+    "    def format_dict(self):\n",
+    "        d = super(TqdmExtraFormat, self).format_dict\n",
+    "        total_time = d[\"elapsed\"] * (d[\"total\"] or 0) / max(d[\"n\"], 1)\n",
+    "        d.update(total_time=self.format_interval(total_time) + \" in total\")\n",
+    "        return d\n",
+    "\n",
+    "for i in TqdmExtraFormat(\n",
+    "      range(10), ascii=\" .oO0\",\n",
+    "      bar_format=\"{total_time}: {percentage:.0f}%|{bar}{r_bar}\"):\n",
+    "    time.sleep(1)"
+   ]
+  }
+ ],
+ "metadata": {
+  "jupytext": {
+   "text_representation": {
+    "extension": ".md",
+    "format_name": "markdown",
+    "format_version": "1.0",
+    "jupytext_version": "0.8.6"
+   }
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

telegram_tqdm.py

@@ -0,0 +1,84 @@
+import telepot
+from tqdm import tqdm
+from datetime import datetime
+
+
+class _TelegramIO():
+    def __init__(self, token, chat_id, #chat_id_list, 
+                 show_last_update=True):
+        self.bot = telepot.Bot(token)
+        self.text = self.prev_text = '<< Init tg_tqdm bar >>'
+#         self.chat_id_list = chat_id_list
+#         for chat_id in chat_id_list:
+        self.chat_id = chat_id
+        self.message_id = self.bot.sendMessage(chat_id, self.text)['message_id']
+        self.show_last_update = show_last_update
+
+    def write(self, s):
+        new_text = s.strip().replace('\r', '')
+        if len(new_text) != 0:
+            self.text = new_text
+
+    def flush(self):
+        if self.prev_text != self.text:
+            if '%' in self.text:
+#                 for chat_id in self.chat_id_list:
+                self.bot.editMessageText((self.chat_id, self.message_id), self.text +
+                                             '\nLast update: {}'.format(datetime.now()) if self.show_last_update else '')
+                self.prev_text = self.text
+
+
+def tg_tqdm(token, chat_id, #chat_id_list, 
+            iterable=None, show_last_update=True,
+            desc=None, total=None, leave=True, ncols=None, mininterval=1.0, maxinterval=10.0,
+            miniters=None, ascii=False, disable=False, unit='it',
+            unit_scale=False, dynamic_ncols=False, smoothing=0.3,
+            bar_format=None, initial=0, position=None, postfix=None,
+            unit_divisor=1000, gui=False, **kwargs):
+    """
+    Decorate an iterable object, returning an iterator which acts exactly
+    like the original iterable, but send to Telegram a dynamically updating
+    progressbar every time a value is requested.
+    
+        Parameters
+        ----------
+        iterable  : iterable, required
+            Iterable to decorate with a progressbar.
+            Leave blank to manually manage the updates.
+        token  : string, required
+            Token of your telegram bot
+            
+        chat_id  : int, required
+            Chat ID where information will be sent about the progress
+            
+        show_last_update  : bool, optional [default: True]
+            Should I show the time-date of the last change in the progress bar?
+            
+        desc, total, leave, ncols, ... :
+            Like in tqdm
+            
+    """
+    tg_io = _TelegramIO(token, chat_id, #chat_id_list, 
+                        show_last_update)
+    return tqdm(iterable=iterable,
+                desc=desc,
+                total=total,
+                leave=leave,
+                file=tg_io,
+                ncols=ncols,
+                mininterval=mininterval,
+                maxinterval=maxinterval,
+                miniters=miniters,
+                ascii=ascii,
+                disable=disable,
+                unit=unit,
+                unit_scale=unit_scale,
+                dynamic_ncols=dynamic_ncols,
+                smoothing=smoothing,
+                bar_format=bar_format,
+                initial=initial,
+                position=position,
+                postfix=postfix,
+                unit_divisor=unit_divisor,
+                gui=gui,
+**kwargs)

tg_logger.py

@@ -0,0 +1,78 @@
+import telegram_handler
+# import python_telegram_logger as ptl
+import logging
+import time
+from colorlog import ColoredFormatter
+
+tg_bot_token = '831964163:AAH7SoaoqWzWIcHaS3yfdmMu-H46hhtUaXw'
+tg_chat_id = 1147194
+ik_chat_id = 94616973
+sun_group_id = -321681009
+
+
+def set_logger(level=logging.WARNING, name='logger', telegram=False):
+    """Return a logger with a default ColoredFormatter."""
+    file_formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(name)s: %(funcName)s - %(message)s")
+    stream_formatter = ColoredFormatter(
+        "%(asctime)s [%(log_color)s%(levelname)-8s%(reset)s] %(white)s%(message)s",
+        datefmt=None,
+        reset=True,
+        log_colors={
+            'DEBUG': 'cyan',
+            'INFO': 'green',
+            'WARNING': 'yellow',
+            'ERROR': 'red',
+            'CRITICAL': 'red',
+        }
+    )
+
+    logger = logging.getLogger(name)
+    stream_handler = logging.StreamHandler()
+    stream_handler.setFormatter(stream_formatter)
+    log_handler = logging.FileHandler("fits_parse.log")
+    log_handler.setFormatter(file_formatter)
+    logger.addHandler(stream_handler)
+    logger.addHandler(log_handler)
+
+    if telegram:
+        tg_handler = telegram_handler.TelegramHandler(tg_bot_token, sun_group_id)
+        tg_formatter = telegram_handler.MarkdownFormatter()
+        tg_handler.setFormatter(tg_formatter)
+        logger.addHandler(tg_handler)
+
+    logger.setLevel(level)
+
+    return logger
+
+# logger = logging.getLogger('tg_test3')
+
+# handler = tgh.TelegramHandler(tg_token, sun_group_id)
+# formatter = tgh.HtmlFormatter()
+# # handler = ptl.Handler(tg_token, [sun_group_id])
+# # formatter = ptl.MarkdownFormatter()
+
+
+# handler.setFormatter(formatter)
+# logger.addHandler(handler)
+# logger.setLevel(logging.INFO)
+
+
+logger = set_logger(telegram=True)
+
+
+def send_test():
+    logger.info('TEST')
+    time.sleep(1)
+    logger.error('TEST')
+    time.sleep(1)
+    logger.warning('warn')
+
+    try:
+        10 / 0
+    except Exception as e:
+        logger.warning('Something not good, get Exception: {}'.format(e))
+        time.sleep(1)
+        logger.error('Something not good, get Exception: {}'.format(e))
+
+
+send_test()

torch_cuda_test.py

@@ -0,0 +1,6 @@
+import torch
+
+print("torch.cuda.is_available()   =", torch.cuda.is_available())
+print("torch.cuda.device_count()   =", torch.cuda.device_count())
+print("torch.cuda.device('cuda')   =", torch.cuda.device('cuda'))
+print("torch.cuda.current_device() =", torch.cuda.current_device())

utils.py

@@ -0,0 +1,169 @@
+import copy
+import os
+import pickle
+import time
+import urllib
+import warnings
+
+import matplotlib.pyplot as plt
+import numpy as np
+# import pretrainedmodels
+
+from astropy.io import fits
+# from PIL import Image
+from skimage.transform import rescale, resize
+# from sklearn.metrics import f1_score
+# from sklearn.utils import class_weight
+
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import torch.utils.data as data_utils
+import torchvision
+import torchvision.transforms as transforms
+import albumentations as alb
+
+from torch.optim import lr_scheduler
+from torchvision import datasets, models
+
+
+class SunRegionDataset(data_utils.Dataset):
+    def __init__(self, path_to_df_pkl, path_to_fits_folder, height, width,
+                 only_first_class=False, transformations=None, logarithm=True, max=None):
+        """
+        Args:
+            path_to_df_pkl (string): path or url to pkl file represents pandas dataframe with labels
+            path_to_image_folder (string): path to folder with fits
+            height (int): image height
+            width (int): image width
+            only_first_class (bool): create dataset with only one letter represents first layer of Mctosh classes
+            transformation: pytorch transforms for transforms and tensor conversion
+        """
+        if path_to_df_pkl.startswith('http'):
+            with urllib.request.urlopen(path_to_df_pkl) as pkl:
+                self.sunspots = pickle.load(pkl)
+        else:
+            self.sunspots = pickle.load(path_to_df_pkl)
+        self.classes = np.asarray(self.sunspots.iloc[:, 2].unique())
+        self.height = height
+        self.width = width
+        self.folder_path, self.dirs, self.files = next(os.walk(path_to_fits_folder))
+        self.len = len(self.files)
+        self.ind = list(range(self.len))
+        self.transformations = transformations
+        self.alb_transorms = alb.Compose([
+                                        alb.RandomRotate90(p=0.1),
+                                        alb.Rotate(75, p=0.1),
+                                        alb.Resize(224, 224, p=0.1),
+                                        alb.RandomCrop(200, 200, p=0.1),
+                                        alb.HorizontalFlip(),
+                                        # alb.Transpose(),
+                                        alb.VerticalFlip(),
+                                        alb.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.50, rotate_limit=45, p=.75),
+                                        ], p=0.7)
+        self.to_tensor = transforms.ToTensor()
+        self.only_first_class = only_first_class
+        self.height = height
+        self.width = width
+        self.logarithm = logarithm
+        self.first_classes = set([class_[0] for class_ in self.sunspots['class'].unique()])
+        self.second_classes = set([class_[1] for class_ in self.sunspots['class'].unique()])
+        self.third_classes = set([class_[2] for class_ in self.sunspots['class'].unique()])
+        if max == None:
+            self.max = self.find_max_dataset()
+        else:
+            self.max = max
+
+    def __getitem__(self, index):
+        file_path = os.path.join(self.folder_path, self.files[index])
+        with fits.open(file_path) as fits_file:
+            data = fits_file[0].data
+
+        if self.transformations is None:
+            if self.logarithm:
+                data = self.log_normalize(data)
+            data = self.normalize_data(data)
+            # data = resize(data, (self.height, self.width), anti_aliasing=True)
+            data = self.aug()(image=data)['image']
+            data = self.to_tensor(data).float()  # uncomment for float
+            # data = data.repeat(3,1,1) # convert to 3 channels to use pretrein models
+        else:
+            data = self.transformations(data)
+
+        mc_class = self.get_attr_region(self.files[index], self.sunspots, self.only_first_class)
+
+        for ind, letter in enumerate(sorted(self.first_classes)):
+            if letter == mc_class:
+                num_class = ind
+
+        return (data, num_class, mc_class)
+
+    def __len__(self):
+        return self.len
+
+    def show_region(self, index):
+        '''Plot region by index from dataset
+        index: int, index of sample from dataset
+        '''
+        date, region = self.files[index].split('.')[1:3]
+        file_path = os.path.join(self.folder_path, self.files[index])
+        with fits.open(file_path) as fits_file:
+            data = fits_file[0].data
+        class_, size, location, number_ss = self.get_attr_region(self.files[index],
+                                                                 self.sunspots,
+                                                                 only_first_class=False,
+                                                                 only_class=False)
+        ax = plt.axes()
+        ax.set_title(
+            'Region {} on date {} with class {} on location {} with size {} and number_of_ss {}'
+            .format(region, date, class_, location, size, number_ss))
+        ax.imshow(data)
+        # ax.annotate((24,12))
+
+    def get_attr_region(self, filename, df, only_first_class=False, only_class=True):
+        date, region = filename.split('.')[1:3]
+        reg_attr = df.loc[date[:-7], int(region[2:])]
+        if only_first_class:
+            return reg_attr['class'][0]
+        elif (not only_class) and (only_first_class):
+            class_, \
+                size, \
+                location, \
+                number_ss = reg_attr[['class', 'size', 'location', 'number_of_ss']]
+            return class_[0], size, location, number_ss
+        elif (not only_class) and (not only_first_class):
+            return reg_attr[['class', 'size', 'location', 'number_of_ss']]
+        else:
+            return reg_attr['class']
+
+    def log_normalize(self, data):
+        return np.sign(data)*np.log1p(np.abs(data))
+
+    def normalize_data(self, data):
+        return data/self.max
+
+    def find_max_dataset(self):
+        m = []
+        for file in self.files:
+            with fits.open(self.folder_path + file) as ff:
+                m.append(np.nanmax(np.abs(ff[0].data)))
+        return np.max(m)
+    
+    def aug(self):
+        return self.alb_transorms
+
+    def split_dataset(self, val_size=None, test_size=None):
+        '''Spliting dataset in optional test, train, val datasets
+        test_size (optional): float from 0 to 1.
+        val_size (optional): float from 0 to 1.
+        shuffle (optional): bool, for shuffled smaples in datasets
+
+        Returns datasets in order (train, valid, test)
+
+        '''
+        len_all = self.len
+        test_split_size = int(np.floor(test_size * len_all)) if test_size else 0
+        val_split_size = int(np.floor(val_size * len_all)) if val_size else 0
+        train_split_size = len_all - test_split_size - val_split_size
+
+        return data_utils.random_split(self, [train_split_size, val_split_size, test_split_size])