
import config

import scripts_m2m_download as m2m
import phase1_st_stats_forHua_optimized_level3 as step1
import phase1_merge_batch_forHua as step2
import os
import re

import subprocess


"""
notes from saeed
so the first date in the tileID are the date of Landsat data acquisition and the second date is the processing date of ARD
LE07_CU_011007_20180211_20200615_C01_V1
The bold one is what you need when trying to find tiles for a specific year
"""

if __name__ == '__main__':
    username = config.USERNAME
    password = config.PASSWORD

    # base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/atlanta"
    # base_dir = "C:\\Users\\rhussain\\Desktop\\LSRD\\data\\atlanta_m2m"

    '''
    iata_location_identifier = "ATL"
    a_start_year_inclusive = 1984
    a_end_year_inclusive = 1985
    tile_hv_names_to_match = ["H23V13", "H23V14"]
    '''

    # atlanta
    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/atlanta_10k"
    #iata_location_identifier = "ATL"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2019
    #tile_hv_names_to_match = ["H23V13", "H23V14", "H24V13", "H24V14"]

    # phoenix
    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/phoenix_10k"
    #iata_location_identifier = "PHX"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2019
    #tile_hv_names_to_match = ["H06V13", "H07V13"]

    # houston
    # base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston"
    # iata_location_identifier = "IAH"
    # a_start_year_inclusive = 1984
    # a_end_year_inclusive = 2019
    # tile_hv_names_to_match = ["H17V16", "H17V17"]

    # base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston"
    # iata_location_identifier = "IAH"
    # a_start_year_inclusive = 2012
    # a_end_year_inclusive = 2014
    # tile_hv_names_to_match = ["H17V16"]

    base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/SiouxFalls_10k"
    iata_location_identifier = "FSD"
    a_start_year_inclusive = 1984
    a_end_year_inclusive = 2020
    tile_hv_names_to_match = ["H16V06"]

    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/minneapolis_10k"
    #iata_location_identifier = "MSP"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2019
    #tile_hv_names_to_match = ["H18V05","H18V06"]

    '''
    base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston"
    iata_location_identifier = "IAH"
    a_start_year_inclusive = 1984
    a_end_year_inclusive = 2019
    tile_hv_names_to_match = ["H17V16", "H17V17"]
    '''

    '''
    base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston"
    iata_location_identifier = "IAH"
    a_start_year_inclusive = 1984
    a_end_year_inclusive = 2019
    tile_hv_names_to_match = ["H17V16", "H17V17"]
    '''

    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/houston_new_m2m"
    #iata_location_identifier = "IAH"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2019
    #tile_hv_names_to_match = ["H17V16", "H17V17"]

    
    #base_dir = "/caldera/projects/usgs/eros/urban_heat_islands/no_uncertainty"
    #iata_location_identifier = "IAH"
    #a_start_year_inclusive = 1984
    #a_end_year_inclusive = 2019
    #tile_hv_names_to_match = ["H17V16","H17V17"]


    if not os.path.isdir(base_dir):
        os.mkdir(base_dir)

    # -------------------------------------------------------------------------------------the download_data_step

    tile_list_comma_delimited_string = ""
    for hv in tile_hv_names_to_match:
        tile_list_comma_delimited_string = tile_list_comma_delimited_string + "" + str(hv) + ','
    tile_list_comma_delimited_string = tile_list_comma_delimited_string[:-1]

    
    some_command = 'srun /home/rhussain/miniconda3/envs/uhi_env/bin/python /home/rhussain/urban-heat-islands/download_data_step.py --workpath ' + base_dir + ' --hv_tilelist ' + tile_list_comma_delimited_string + ' --start_year_inclusive  '+ str \
        (a_start_year_inclusive) +' --end_year_inclusive ' + str(
        a_end_year_inclusive) + ' --username ' + username + ' --password ' + password
    # print(os.system("ls"))
    # https://cmdlinetips.com/2014/03/how-to-run-a-shell-command-from-python-and-get-the-output/
    out = subprocess.Popen(
        ['sbatch', '--job-name=urban_heat_islands_' + tile_list_comma_delimited_string + 'm2m_download_data',
         '--output=urban_heat_islands_' + tile_list_comma_delimited_string + 'm2m_download_data' + '.output',
         '--account=eros',
         '--partition=workq',
         '--nodes=1',
         '--ntasks=1',
         '--cpus-per-task=70',
         '--time=2-00:00:00',  # hpc limit is 2 days if u really need it
         '--mail-type=ALL',
         '--mail-user=rhussain@contractor.usgs.gov',
         '--wrap=' + some_command + '',
         ],
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    stdout, stderr = out.communicate()
    
    '''
    #REMOVE ME AND USE THE ABOVE
    #THIS IS TEMPORARY FOR 2 TILE DATA DL
    #IT HAS 1 DAY for better slurm scheduling
    some_command = 'srun /home/rhussain/miniconda3/envs/uhi_env/bin/python /home/rhussain/urban-heat-islands/download_data_step.py --workpath ' + base_dir + ' --hv_tilelist ' + tile_list_comma_delimited_string + ' --start_year_inclusive  '+ str \
    (a_start_year_inclusive) +' --end_year_inclusive ' + str(
    a_end_year_inclusive) + ' --username ' + username + ' --password ' + password
    # print(os.system("ls"))
    # https://cmdlinetips.com/2014/03/how-to-run-a-shell-command-from-python-and-get-the-output/
    out = subprocess.Popen(
        ['sbatch', '--job-name=urban_heat_islands_' + tile_list_comma_delimited_string + 'm2m_download_data',
         '--output=urban_heat_islands_' + tile_list_comma_delimited_string + 'm2m_download_data' + '.output',
         '--account=eros',
         '--partition=workq',
         '--nodes=1',
         '--ntasks=1',
         '--cpus-per-task=70',
         '--time=0-16:00:00',  # hpc limit is 2 days if u really need it
         '--mail-type=ALL',
         '--mail-user=rhussain@contractor.usgs.gov',
         '--wrap=' + some_command + '',
         ],
        stdout=subprocess.PIPE,
        stderr=subprocess.STDOUT)
    stdout, stderr = out.communicate()
    '''

    print(subprocess.list2cmdline(out.args))
    print(stdout)

    # https://stackoverflow.com/questions/26825729/extract-number-from-string-in-python/26825781
    # basically when u submit a slurm job,
    # it returns a string that has one number in it, which is the slurm job id
    # so we just use a regex to get the number that is in the string
    all_numbers = re.findall("\d+", str(stdout))
    print(all_numbers)
    job_id = all_numbers[0]

    dependency_string = '--dependency=afterok:' + job_id

    # -------------------------------------------------------------------------------------the st_stats_forHua step

    # we can split this into multiple slurm jobs
    # because this runs for each tile
    # so we submit a job for each tile
    job_ids = []
    for hv in tile_hv_names_to_match:
        some_command = 'srun /home/rhussain/miniconda3/envs/uhi_env/bin/python /home/rhussain/urban-heat-islands/st_stats_forHua_optimized_level3.py --workpath ' + base_dir + ' --hv_tilename ' + hv + ' --iata_location_identifier ' + iata_location_identifier + ' --start_year_inclusive ' + str(
            a_start_year_inclusive) + ' --end_year_inclusive ' + str(a_end_year_inclusive)
        # print(os.system("ls"))
        # https://cmdlinetips.com/2014/03/how-to-run-a-shell-command-from-python-and-get-the-output/
        out = subprocess.Popen(['sbatch', '--job-name=urban_heat_islands_' + hv,
                                '--output=urban_heat_islands_' + hv + '.output',
                                dependency_string,
                                '--account=eros',
                                '--partition=workq',
                                '--nodes=1',
                                '--ntasks=1',
                                '--cpus-per-task=70',
                                '--time=1-00:00:00',  # hpc limit is 2 days if u really need it
                                '--mail-type=ALL',
                                '--mail-user=rhussain@contractor.usgs.gov',
                                '--wrap=' + some_command + '',
                                ],
                               stdout=subprocess.PIPE,
                               stderr=subprocess.STDOUT)
        stdout, stderr = out.communicate()

        print(subprocess.list2cmdline(out.args))
        print(stdout)

        # https://stackoverflow.com/questions/26825729/extract-number-from-string-in-python/26825781
        # basically when u submit a slurm job,
        # it returns a string that has one number in it, which is the slurm job id
        # so we just use a regex to get the number that is in the string
        all_numbers = re.findall("\d+", str(stdout))
        print(all_numbers)
        job_id = all_numbers[0]
        job_ids.append(job_id)

    # now, submit the last job with a dependency on the other jobs
    # https://gif.biotech.iastate.edu/submitting-dependency-jobs-using-slurm
    # If the job requires more than one job to be completed before it is executed, you can supply all the jobids using , separator
    # sbatch --dependency=type:job_id,job_id,job_id jobfile
    # iata_location_identifier

    dependency_string = '--dependency=afterok:'
    for a_job_id in job_ids:
        dependency_string = dependency_string + str(a_job_id) + ','
    dependency_string = dependency_string[:-1]

    # -------------------------------------------------------------------------------------merge_batch_forHua step

    hv_list_str = ""
    for hv in tile_hv_names_to_match:
        hv_list_str = hv_list_str + hv + ","
    hv_list_str = hv_list_str[:-1]
    # print(tile_hv_names_to_match)

    some_command = '/home/rhussain/miniconda3/envs/uhi_env/bin/python /home/rhussain/urban-heat-islands/merge_batch_forHua.py --workpath ' + base_dir + ' --hv_tilelist ' + hv_list_str + ' --start_year_inclusive ' + str(
        a_start_year_inclusive) + ' --end_year_inclusive ' + str(
        a_end_year_inclusive) + ' --iata_location_identifier ' + str(iata_location_identifier)
    # print(os.system("ls"))
    # https://cmdlinetips.com/2014/03/how-to-run-a-shell-command-from-python-and-get-the-output/
    out = subprocess.Popen(['sbatch',
                            '--job-name=urban_heat_islands_' + 'merge_batch_forHua',
                            '--output=urban_heat_islands_merge_batch_forHua.output',
                            dependency_string,
                            '--account=eros',
                            '--partition=workq',
                            '--nodes=1',
                            '--ntasks=1',
                            '--cpus-per-task=70',
                            '--time=0-01:00:00',  # hpc limit is 2 days if u really need it
                            '--mail-type=ALL',
                            '--mail-user=rhussain@contractor.usgs.gov',
                            '--wrap=' + some_command + '',
                            ],
                           stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT)
    stdout, stderr = out.communicate()

    print(subprocess.list2cmdline(out.args))
    print(stdout)








