
import config

import phase1_st_stats_forHua_optimized_level3 as step1
import phase1_merge_batch_forHua as step2
import os
import re

import subprocess

import shutil


"""
notes from saeed
so the first date in the tileID are the date of Landsat data acquisition and the second date is the processing date of ARD
LE07_CU_011007_20180211_20200615_C01_V1
The bold one is what you need when trying to find tiles for a specific year
"""

if __name__ == '__main__':
    username = config.USERNAME
    password = config.PASSWORD


    '''
    iata_location_identifier = "ATL"
    a_start_year_inclusive = 1984
    a_end_year_inclusive = 1985
    tile_hv_names_to_match = ["H23V13", "H23V14"]
    '''

    # phoenix
    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/phoenix_10k"
    #iata_location_identifier = "PHX"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2019
    #tile_hv_names_to_match = ["H06V13", "H07V13"]

    # houston
    # base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston"
    # iata_location_identifier = "IAH"
    # a_start_year_inclusive = 1984
    # a_end_year_inclusive = 2019
    # tile_hv_names_to_match = ["H17V16", "H17V17"]

    # base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston"
    # iata_location_identifier = "IAH"
    # a_start_year_inclusive = 2012
    # a_end_year_inclusive = 2014
    # tile_hv_names_to_match = ["H17V16"]

    base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/SiouxFalls_10k"
    iata_location_identifier = "FSD"
    a_start_year_inclusive = 1984
    a_end_year_inclusive = 2020
    tile_hv_names_to_match = ["H16V06"]

    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/minneapolis_10k"
    #iata_location_identifier = "MSP"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2019
    #tile_hv_names_to_match = ["H18V05","H18V06"]

    '''
    base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston"
    iata_location_identifier = "IAH"
    a_start_year_inclusive = 1984
    a_end_year_inclusive = 2019
    tile_hv_names_to_match = ["H17V16", "H17V17"]
    '''

    '''
    base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston"
    iata_location_identifier = "IAH"
    a_start_year_inclusive = 1984
    a_end_year_inclusive = 2019
    tile_hv_names_to_match = ["H17V16", "H17V17"]
    '''

    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston_new_m2m"
    #iata_location_identifier = "IAH"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2019
    #tile_hv_names_to_match = ["H17V16", "H17V17"]

    
    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/no_uncertainty"
    #iata_location_identifier = "IAH"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2019
    #tile_hv_names_to_match = ["H17V16","H17V17"]



    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/SiouxFalls_10k"
    #iata_location_identifier = "FSD"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2020
    #tile_hv_names_to_match = ["H16V06"]

    a_start_year_inclusive = 1984
    a_end_year_inclusive = 2020

    downloads = [

        #("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/FSD/output_folder/",[""],"FSD","SiouxFalls"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/MSO/output_folder/",[""],"MSO","Missoula"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/ORD/output_folder/",[""],"ORD","Chicago"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/MKE/output_folder/",[""],"MKE","Milwaukee"),

        ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/PHX/output_folder/",[""],"PHX","Phoenix")


    ]

    '''
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/RDU/output_folder/",[""],"RDU","Raleigh"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/FAR/output_folder/",[""],"FAR","Fargo"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/LIT/output_folder/",[""],"LIT","LittleRock"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/PIT/output_folder/",[""],"PIT","Pittsburgh"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/SDF/output_folder/",[""],"SDF","Louisville"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/CYS/output_folder/",[""],"CYS","Cheyenne"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/BOI/output_folder/",[""],"BOI","Boise"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/MSP/output_folder/",[""],"MSP","Minneapolis"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/OMA/output_folder/",[""],"OMA","Omaha"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/ABQ/output_folder/",[""],"ABQ","Albuquerque"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/LAS/output_folder/",[""],"LAS","LasVegas"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/DTW/output_folder/",[""],"DTW","Detroit"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/BOS/output_folder/",[""],"BOS","Boston"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/OKC/output_folder/",[""],"OKC","OklahomaCity"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/MEM/output_folder/",[""],"MEM","Memphis"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/BNA/output_folder/",[""],"BNA","Nashville"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/ELP/output_folder/",[""],"ELP","ElPaso"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/DEN/output_folder/",[""],"DEN","Denver"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/IND/output_folder/",[""],"IND","Indianapolis"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/SFO/output_folder/",[""],"SFO","SanFrancisco"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/ATL/output_folder/",[""],"ATL","Atlanta"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/CMH/output_folder/",[""],"CMH","Columbus"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/DFW/output_folder/",[""],"DFW","Dallas"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/SAT/output_folder/",[""],"SAT","SanAntonio"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/PHL/output_folder/",[""],"PHL","Philadelphia"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/DSM/output_folder/",[""],"DSM","DesMoines"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/JAX/output_folder/",[""],"JAX","Jacksonville"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/IAH/output_folder/",[""],"IAH","Houston"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/LAX/output_folder/",[""],"LAX","LosAngeles"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/LGA/output_folder/",[""],"LGA","NewYork"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/BHX/output_folder/",[""],"BHX","Birmingham"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/MSY/output_folder/",[""],"MSY","NewOrleans"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/MIA/output_folder/",[""],"MIA","Miami"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/BWI/output_folder/",[""],"BWI","Baltimore"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/SLC/output_folder/",[""],"SLC","SaltLakeCity"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/PHX/output_folder/",[""],"PHX","Phoenix"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/CVG/output_folder/",[""],"CVG","Cincinnati"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/CLT/output_folder/",[""],"CLT","Charlotte"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/SMF/output_folder/",[""],"SMF","Sacramento"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/SEA/output_folder/",[""],"SEA","Seattle"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/MCI/output_folder/",[""],"MCI","KansasCity"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/PDX/output_folder/",[""],"PDX","Portland"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/COS/output_folder/",[""],"COS","ColoradoSprings"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/SAN/output_folder/",[""],"SAN","SanDiego"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/AUS/output_folder/",[""],"AUS","Austin"),
    ("/caldera/projects/usgs/eros/urban_heat_islands/Phase1/DCA/output_folder/",[""],"DCA","Washington")
    '''

    #clean previous run outputs
    for (base_dir,tile_hv_names_to_match,iata_location_identifier,city_name) in downloads:
        if os.path.exists(base_dir):
            airport_path = os.path.join(base_dir,iata_location_identifier)
            if os.path.exists(airport_path):
                shutil.rmtree(airport_path)
            
            for tile_name in tile_hv_names_to_match:
                if os.path.exists(os.path.join(base_dir,tile_name)):
                    annual_path = os.path.join(base_dir,tile_name,"ANNUAL")
                    total_path = os.path.join(base_dir,tile_name,"TOTAL")
                    seasonal_path = os.path.join(base_dir,tile_name,"SEASONAL")
                    extract_path = os.path.join(base_dir,tile_name,"extract")
                    
                    #if os.path.exists(annual_path):
                    #    shutil.rmtree(annual_path)
                    if os.path.exists(total_path):
                        shutil.rmtree(total_path)
                    #if os.path.exists(seasonal_path):
                    #    shutil.rmtree(seasonal_path)
                    #if os.path.exists(extract_path):
                    #    shutil.rmtree(extract_path)
                    ABC = None

    download_dependency_string = None

    for (base_dir,tile_hv_names_to_match,iata_location_identifier,city_name) in downloads:

        if not os.path.isdir(base_dir):
            os.mkdir(base_dir)

        tile_list_comma_delimited_string = ""
        for hv in tile_hv_names_to_match:
            tile_list_comma_delimited_string = tile_list_comma_delimited_string + "" + str(hv) + ','
        tile_list_comma_delimited_string = tile_list_comma_delimited_string[:-1]

        #-------------------------------------------------------------------------------------the st_stats_forHua step

        # we can split this into multiple slurm jobs
        # because this runs for each tile
        # so we submit a job for each tile
        job_ids = []
        for hv in tile_hv_names_to_match:

            hv = ""

            some_command = 'srun /home/rhussain/miniconda3/envs/uhi_env/bin/python /home/rhussain/urban-heat-islands-phase2/st_stats_forHua_optimized_level3_rerun_partial.py --workpath ' + base_dir + ' --iata_location_identifier ' + iata_location_identifier + ' --start_year_inclusive ' + str(
                a_start_year_inclusive) + ' --end_year_inclusive ' + str(a_end_year_inclusive)
            # print(os.system("ls"))
            # https://cmdlinetips.com/2014/03/how-to-run-a-shell-command-from-python-and-get-the-output/
            out = subprocess.Popen(['sbatch', '--job-name=urban_heat_islands_' + hv,
                                    '--output=urban_heat_islands_' + hv + '_' + city_name + '.output',
                                    #dependency_string,
                                    '--account=eros',
                                    '--partition=workq',
                                    '--nodes=1',
                                    '--ntasks=1',
                                    '--cpus-per-task=70',
                                    '--time=2-00:00:00',  # hpc limit is 2 days if u really need it
                                    '--mail-type=ALL',
                                    '--mail-user=rhussain@contractor.usgs.gov',
                                    '--wrap=' + some_command + '',
                                    ],
                                stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
            stdout, stderr = out.communicate()

            print(subprocess.list2cmdline(out.args))
            print(stdout)

            # https://stackoverflow.com/questions/26825729/extract-number-from-string-in-python/26825781
            # basically when u submit a slurm job,
            # it returns a string that has one number in it, which is the slurm job id
            # so we just use a regex to get the number that is in the string
            all_numbers = re.findall("\d+", str(stdout))
            print(all_numbers)
            job_id = all_numbers[0]
            job_ids.append(job_id)







