
import config

import phase1_st_stats_forHua_optimized_level3 as step1
import phase1_merge_batch_forHua as step2
import os
import re

import subprocess

import shutil


"""
notes from saeed
so the first date in the tileID are the date of Landsat data acquisition and the second date is the processing date of ARD
LE07_CU_011007_20180211_20200615_C01_V1
The bold one is what you need when trying to find tiles for a specific year
"""

if __name__ == '__main__':
    username = config.USERNAME
    password = config.PASSWORD


    '''
    iata_location_identifier = "ATL"
    a_start_year_inclusive = 1984
    a_end_year_inclusive = 1985
    tile_hv_names_to_match = ["H23V13", "H23V14"]
    '''

    # phoenix
    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/phoenix_10k"
    #iata_location_identifier = "PHX"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2019
    #tile_hv_names_to_match = ["H06V13", "H07V13"]

    # houston
    # base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston"
    # iata_location_identifier = "IAH"
    # a_start_year_inclusive = 1984
    # a_end_year_inclusive = 2019
    # tile_hv_names_to_match = ["H17V16", "H17V17"]

    # base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston"
    # iata_location_identifier = "IAH"
    # a_start_year_inclusive = 2012
    # a_end_year_inclusive = 2014
    # tile_hv_names_to_match = ["H17V16"]

    base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/SiouxFalls_10k"
    iata_location_identifier = "FSD"
    a_start_year_inclusive = 1984
    a_end_year_inclusive = 2020
    tile_hv_names_to_match = ["H16V06"]

    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/minneapolis_10k"
    #iata_location_identifier = "MSP"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2019
    #tile_hv_names_to_match = ["H18V05","H18V06"]

    '''
    base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston"
    iata_location_identifier = "IAH"
    a_start_year_inclusive = 1984
    a_end_year_inclusive = 2019
    tile_hv_names_to_match = ["H17V16", "H17V17"]
    '''

    '''
    base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston"
    iata_location_identifier = "IAH"
    a_start_year_inclusive = 1984
    a_end_year_inclusive = 2019
    tile_hv_names_to_match = ["H17V16", "H17V17"]
    '''

    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston_new_m2m"
    #iata_location_identifier = "IAH"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2019
    #tile_hv_names_to_match = ["H17V16", "H17V17"]

    
    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/no_uncertainty"
    #iata_location_identifier = "IAH"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2019
    #tile_hv_names_to_match = ["H17V16","H17V17"]



    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/SiouxFalls_10k"
    #iata_location_identifier = "FSD"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2020
    #tile_hv_names_to_match = ["H16V06"]

    a_start_year_inclusive = 1984
    a_end_year_inclusive = 2020

    downloads = [

        #done
        #("/caldera/projects/usgs/eros/urban_heat_islands/Raleigh",["H27V11"],"RDU","Raleigh"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Missoula",["H07V03","H08V03"],"MSO","Missoula"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Fargo",["H16V04"],"FAR","Fargo"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/LittleRock",["H19V13"],"LIT","LittleRock"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Pittsburgh",["H25V08","H26V08"],"PIT","Pittsburgh"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Louisville",["H22V10","H23V10"],"SDF","Louisville"),
        
        #done
        #("/caldera/projects/usgs/eros/urban_heat_islands/Cheyenne",["H12V08"],"CYS","Cheyenne"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Boise",["H06V05"],"BOI","Boise"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/SiouxFalls",["H16V06"],"FSD","SiouxFalls"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Minneapolis",["H18V05","H18V06"],"MSP","Minneapolis"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Omaha",["H16V08","H17V08"],"OMA","Omaha"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Albuquerque",["H10V12","H10V13"],"ABQ","Albuquerque"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Milwaukee",["H21V06","H21V07"],"MKE","Milwaukee"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/LasVegas",["H05V11"],"LAS","LasVegas")

        #done
        #("/caldera/projects/usgs/eros/urban_heat_islands/Detroit",["H23V06","H24V06","H23V07","H24V07"],"DTW","Detroit"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Boston",["H30V05","H30V06"],"BOS","Boston"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/OklahomaCity",["H16V12","H16V13"],"OKC","OklahomaCity"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Memphis",["H20V12","H20V13"],"MEM","Memphis"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Nashville",["H22V11","H22V12"],"BNA","Nashville"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/ElPaso",["H10V15"],"ELP","ElPaso"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Denver",["H11V09","H12V09"],"DEN","Denver"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Indianapolis",["H22V09"],"IND","Indianapolis"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/SanFrancisco",["H01V08","H01V09","H02V08","H02V09"],"SFO","SanFrancisco"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Atlanta",["H23V13","H23V14","H24V13","H24V14"],"ATL","Atlanta")#,

        #("/caldera/projects/usgs/eros/urban_heat_islands/Columbus",["H24V08","H24V09"],"CMH","Columbus"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Dallas",["H16V14","H16V15"],"DFW","Dallas"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/SanAntonio",["H15V17"],"SAT","SanAntonio"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Philadelphia",["H28V07","H28V08","H29V08"],"PHL","Philadelphia"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/DesMoines",["H18V08"],"DSM","DesMoines"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Jacksonville",["H26V15","H26V16"],"JAX","Jacksonville"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Houston",["H17V16","H17V17"],"IAH","Houston"), 
        #("/caldera/projects/usgs/eros/urban_heat_islands/Chicago",["H21V07","H21V08","H22V08"],"ORD","Chicago"), 
        #("/caldera/projects/usgs/eros/urban_heat_islands/LosAngeles",["H03V12","H04V12"],"LAX","LosAngeles"),
        ("/caldera/projects/usgs/eros/urban_heat_islands/NewYork",["H28V07","H29V06","H29V07","H29V08","H30V07"],"LGA","NewYork")#, 
        #("/caldera/projects/usgs/eros/urban_heat_islands/Birmingham",["H22V13","H22V14"],"BHX","Birmingham"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/NewOrleans",["H20V16","H20V17","H21V16","H21V17"],"MSY","NewOrleans"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Miami",["H27V18","H27V19"],"MIA","Miami"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Baltimore",["H27V08","H27V09","H28V08","H28V09"],"BWI","Baltimore"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Raleigh",["H27V11"],"RDU","Raleigh"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Fargo",["H16V04"],"FAR","Fargo")

        
        
        #("/caldera/projects/usgs/eros/urban_heat_islands/SaltLakeCity",["H08V07","H08V08"],"SLC","SaltLakeCity")
        
        #("/caldera/projects/usgs/eros/urban_heat_islands/Phoenix",["H06V13","H07V13"],"PHX","Phoenix"),#2378+2342
        #("/caldera/projects/usgs/eros/urban_heat_islands/Cincinnati",["H23V09","H23V10"],"CVG","Cincinnati"),
        
        #("/caldera/projects/usgs/eros/urban_heat_islands/Charlotte",["H25V12","H26V12","H26V11"],"CLT","Charlotte"),
        
        
        

        #("/caldera/projects/usgs/eros/urban_heat_islands/Sacramento",["H02V08","H03V08"],"SMF","Sacramento")#,

        #("/caldera/projects/usgs/eros/urban_heat_islands/Seattle",["H03V02","H03V01","H04V01","H04V02"],"SEA","Seattle"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/KansasCity",["H17V09","H17V10","H18V10"],"MCI","KansasCity"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Portland",["H03V03"],"PDX","Portland"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/ColoradoSprings",["H11V10","H12V10","H11V09","H12V09"],"COS","ColoradoSprings"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/SanDiego",["H03V13","H04V12","H04V13"],"SAN","SanDiego"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Austin",["H15V16","H16V16","H15V17"],"AUS","Austin"),
        #("/caldera/projects/usgs/eros/urban_heat_islands/Washington",["H27V09","H28V09","H27V08"],"DCA","Washington")

    ]

    #clean previous run outputs
    for (base_dir,tile_hv_names_to_match,iata_location_identifier,city_name) in downloads:
        if os.path.exists(base_dir):
            airport_path = os.path.join(base_dir,iata_location_identifier)
            if os.path.exists(airport_path):
                shutil.rmtree(airport_path)
            
            for tile_name in tile_hv_names_to_match:
                if os.path.exists(os.path.join(base_dir,tile_name)):
                    annual_path = os.path.join(base_dir,tile_name,"ANNUAL")
                    total_path = os.path.join(base_dir,tile_name,"TOTAL")
                    seasonal_path = os.path.join(base_dir,tile_name,"SEASONAL")
                    extract_path = os.path.join(base_dir,tile_name,"extract")
                    
                    if os.path.exists(annual_path):
                        shutil.rmtree(annual_path)
                    if os.path.exists(total_path):
                        shutil.rmtree(total_path)
                    if os.path.exists(seasonal_path):
                        shutil.rmtree(seasonal_path)
                    if os.path.exists(extract_path):
                        shutil.rmtree(extract_path)
                    ABC = None

    download_dependency_string = None

    for (base_dir,tile_hv_names_to_match,iata_location_identifier,city_name) in downloads:

        if not os.path.isdir(base_dir):
            os.mkdir(base_dir)

        tile_list_comma_delimited_string = ""
        for hv in tile_hv_names_to_match:
            tile_list_comma_delimited_string = tile_list_comma_delimited_string + "" + str(hv) + ','
        tile_list_comma_delimited_string = tile_list_comma_delimited_string[:-1]

        #--------------------------------------------------------------------------------------download data step

        some_command = 'srun /home/rhussain/miniconda3/envs/uhi_env/bin/python /home/rhussain/urban-heat-islands/download_data_step.py --workpath ' + base_dir + ' --hv_tilelist ' + tile_list_comma_delimited_string + ' --start_year_inclusive  '+ str \
            (a_start_year_inclusive) +' --end_year_inclusive ' + str(
            a_end_year_inclusive) + ' --username ' + username + ' --password ' + password
        # print(os.system("ls"))
        # https://cmdlinetips.com/2014/03/how-to-run-a-shell-command-from-python-and-get-the-output/


        if download_dependency_string is None:
            out = subprocess.Popen(
            ['sbatch', '--job-name=m2m_download_data_urban_heat_islands_' + tile_list_comma_delimited_string +'_'+city_name,
            '--output=m2m_download_data_urban_heat_islands_' + tile_list_comma_delimited_string +'_'+city_name+'.output',
            '--account=eros',
            '--partition=workq',
            '--nodes=1',
            '--ntasks=1',
            '--cpus-per-task=70',
            '--time=2-00:00:00',  # hpc limit is 2 days if u really need it
            '--mail-type=ALL',
            '--mail-user=rhussain@contractor.usgs.gov',
            '--wrap=' + some_command + '',
            ],
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT)
        else:
            out = subprocess.Popen(
            ['sbatch', '--job-name=urban_heat_islands_' + tile_list_comma_delimited_string + 'm2m_download_data'+'_'+city_name,
            '--output=urban_heat_islands_' + tile_list_comma_delimited_string + 'm2m_download_data' +'_'+city_name+'.output',
            download_dependency_string,
            '--account=eros',
            '--partition=workq',
            '--nodes=1',
            '--ntasks=1',
            '--cpus-per-task=70',
            '--time=2-00:00:00',  # hpc limit is 2 days if u really need it
            '--mail-type=ALL',
            '--mail-user=rhussain@contractor.usgs.gov',
            '--wrap=' + some_command + '',
            ],
            stdout=subprocess.PIPE,
            stderr=subprocess.STDOUT)


        stdout, stderr = out.communicate()

        print(subprocess.list2cmdline(out.args))
        print(stdout)

        # https://stackoverflow.com/questions/26825729/extract-number-from-string-in-python/26825781
        # basically when u submit a slurm job,
        # it returns a string that has one number in it, which is the slurm job id
        # so we just use a regex to get the number that is in the string
        all_numbers = re.findall("\d+", str(stdout))
        print(all_numbers)
        job_id = all_numbers[0]

        dependency_string = '--dependency=afterok:' + job_id
        download_dependency_string = dependency_string

        # -------------------------------------------------------------------------------------the st_stats_forHua step

        # we can split this into multiple slurm jobs
        # because this runs for each tile
        # so we submit a job for each tile
        job_ids = []
        for hv in tile_hv_names_to_match:
            some_command = 'srun /home/rhussain/miniconda3/envs/uhi_env/bin/python /home/rhussain/urban-heat-islands/st_stats_forHua_optimized_level3.py --workpath ' + base_dir + ' --hv_tilename ' + hv + ' --iata_location_identifier ' + iata_location_identifier + ' --start_year_inclusive ' + str(
                a_start_year_inclusive) + ' --end_year_inclusive ' + str(a_end_year_inclusive)
            # print(os.system("ls"))
            # https://cmdlinetips.com/2014/03/how-to-run-a-shell-command-from-python-and-get-the-output/
            out = subprocess.Popen(['sbatch', '--job-name=urban_heat_islands_' + hv,
                                    '--output=urban_heat_islands_' + hv + '_' + city_name + '.output',
                                    dependency_string,
                                    '--account=eros',
                                    '--partition=workq',
                                    '--nodes=1',
                                    '--ntasks=1',
                                    '--cpus-per-task=70',
                                    '--time=2-00:00:00',  # hpc limit is 2 days if u really need it
                                    '--mail-type=ALL',
                                    '--mail-user=rhussain@contractor.usgs.gov',
                                    '--wrap=' + some_command + '',
                                    ],
                                stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT)
            stdout, stderr = out.communicate()

            print(subprocess.list2cmdline(out.args))
            print(stdout)

            # https://stackoverflow.com/questions/26825729/extract-number-from-string-in-python/26825781
            # basically when u submit a slurm job,
            # it returns a string that has one number in it, which is the slurm job id
            # so we just use a regex to get the number that is in the string
            all_numbers = re.findall("\d+", str(stdout))
            print(all_numbers)
            job_id = all_numbers[0]
            job_ids.append(job_id)

        # now, submit the last job with a dependency on the other jobs
        # https://gif.biotech.iastate.edu/submitting-dependency-jobs-using-slurm
        # If the job requires more than one job to be completed before it is executed, you can supply all the jobids using , separator
        # sbatch --dependency=type:job_id,job_id,job_id jobfile
        # iata_location_identifier

        dependency_string = '--dependency=afterok:'
        for a_job_id in job_ids:
            dependency_string = dependency_string + str(a_job_id) + ','
        dependency_string = dependency_string[:-1]

        # -------------------------------------------------------------------------------------merge_batch_forHua step

        hv_list_str = ""
        for hv in tile_hv_names_to_match:
            hv_list_str = hv_list_str + hv + ","
        hv_list_str = hv_list_str[:-1]
        # print(tile_hv_names_to_match)

        some_command = '/home/rhussain/miniconda3/envs/uhi_env/bin/python /home/rhussain/urban-heat-islands/merge_batch_forHua.py --workpath ' + base_dir + ' --hv_tilelist ' + hv_list_str + ' --start_year_inclusive ' + str(
            a_start_year_inclusive) + ' --end_year_inclusive ' + str(
            a_end_year_inclusive) + ' --iata_location_identifier ' + str(iata_location_identifier)
        # print(os.system("ls"))
        # https://cmdlinetips.com/2014/03/how-to-run-a-shell-command-from-python-and-get-the-output/
        out = subprocess.Popen(['sbatch',
                                '--job-name=urban_heat_islands_' + 'merge_batch_forHua_'+ city_name + '_' + tile_list_comma_delimited_string,
                                '--output=urban_heat_islands_merge_batch_forHua_'+tile_list_comma_delimited_string+'_'+city_name+'.output',
                                dependency_string,
                                '--account=eros',
                                '--partition=workq',
                                '--nodes=1',
                                '--ntasks=1',
                                '--cpus-per-task=70',
                                '--time=0-06:00:00',  # hpc limit is 2 days if u really need it
                                '--mail-type=ALL',
                                '--mail-user=rhussain@contractor.usgs.gov,sarab@contractor.usgs.gov',
                                '--wrap=' + some_command + '',
                                ],
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT)
        stdout, stderr = out.communicate()

        print(subprocess.list2cmdline(out.args))
        print(stdout)








