#======================================================================================
# checkMissingForecasts.py
# Author: Mandi Thran
# Date: 01/10/21

# DESCRIPTION:
# This script checks for any BoM National Storm Surge forecasts that may be missing from 
# the WRL1 Coastal folder.  
#======================================================================================


from datetime import datetime, timedelta
import os
import pytz
import numpy as np
import pandas as pd

drive = "\\\\ad.unsw.edu.au\\OneUNSW\\ENG\\WRL\\WRL1"
workDir = "C:\\Users\\mandiruns\\Documents\\01_FEWS-RegionHome-Aus\\Scripts\\waterLevs"
outDir = os.path.join(drive,"Coastal\\Data\\Tide\\WL_Forecast\\BOM_Storm_Surge\\raw\\corrected")

fixPeriodStart = datetime(2019,9,13,0)
fixPeriodStart = pytz.utc.localize(fixPeriodStart)
fixPeriodEnd = datetime(2020,7,14)
fixPeriodEnd = pytz.utc.localize(fixPeriodEnd)

# Generate time series
timeSeries = np.arange(fixPeriodStart,fixPeriodEnd,timedelta(hours=6))
df = pd.DataFrame({'time_series':timeSeries})
df['index'] = df['time_series']
df = df.set_index("index")
df.index = pd.to_datetime(df.index, format="%Y-%m-%d %H:%M:%S")

# Load a list of files in that directory
filesdf = pd.DataFrame({"FileName": os.listdir(outDir)}) 
filesdf['dt_string'] = filesdf['FileName'].apply(lambda s:s.split('_')[3].split('.')[0])
filesdf['index'] = pd.to_datetime(filesdf['dt_string'],format="%Y%m%d%H")
filesdf = filesdf.set_index('index')

# Merge df on left
df = pd.merge(df, filesdf, how='left',left_index=True,right_index=True)
dfna = df[df.isna().any(axis=1)]
dfna.to_csv(os.path.join(workDir,"missingFiles.csv"))

b7baf80b1dbd96d873d54defc9e772603897acd2
<mask token>

<mask token>
dfna.to_csv(os.path.join(workDir, 'missingFiles.csv'))

<mask token>
drive = '\\\\ad.unsw.edu.au\\OneUNSW\\ENG\\WRL\\WRL1'
workDir = (
    'C:\\Users\\mandiruns\\Documents\\01_FEWS-RegionHome-Aus\\Scripts\\waterLevs'
    )
outDir = os.path.join(drive,
    'Coastal\\Data\\Tide\\WL_Forecast\\BOM_Storm_Surge\\raw\\corrected')
fixPeriodStart = datetime(2019, 9, 13, 0)
fixPeriodStart = pytz.utc.localize(fixPeriodStart)
fixPeriodEnd = datetime(2020, 7, 14)
fixPeriodEnd = pytz.utc.localize(fixPeriodEnd)
timeSeries = np.arange(fixPeriodStart, fixPeriodEnd, timedelta(hours=6))
df = pd.DataFrame({'time_series': timeSeries})
df['index'] = df['time_series']
df = df.set_index('index')
df.index = pd.to_datetime(df.index, format='%Y-%m-%d %H:%M:%S')
filesdf = pd.DataFrame({'FileName': os.listdir(outDir)})
filesdf['dt_string'] = filesdf['FileName'].apply(lambda s: s.split('_')[3].
    split('.')[0])
filesdf['index'] = pd.to_datetime(filesdf['dt_string'], format='%Y%m%d%H')
filesdf = filesdf.set_index('index')
df = pd.merge(df, filesdf, how='left', left_index=True, right_index=True)
dfna = df[df.isna().any(axis=1)]
dfna.to_csv(os.path.join(workDir, 'missingFiles.csv'))

from datetime import datetime, timedelta
import os
import pytz
import numpy as np
import pandas as pd
drive = '\\\\ad.unsw.edu.au\\OneUNSW\\ENG\\WRL\\WRL1'
workDir = (
    'C:\\Users\\mandiruns\\Documents\\01_FEWS-RegionHome-Aus\\Scripts\\waterLevs'
    )
outDir = os.path.join(drive,
    'Coastal\\Data\\Tide\\WL_Forecast\\BOM_Storm_Surge\\raw\\corrected')
fixPeriodStart = datetime(2019, 9, 13, 0)
fixPeriodStart = pytz.utc.localize(fixPeriodStart)
fixPeriodEnd = datetime(2020, 7, 14)
fixPeriodEnd = pytz.utc.localize(fixPeriodEnd)
timeSeries = np.arange(fixPeriodStart, fixPeriodEnd, timedelta(hours=6))
df = pd.DataFrame({'time_series': timeSeries})
df['index'] = df['time_series']
df = df.set_index('index')
df.index = pd.to_datetime(df.index, format='%Y-%m-%d %H:%M:%S')
filesdf = pd.DataFrame({'FileName': os.listdir(outDir)})
filesdf['dt_string'] = filesdf['FileName'].apply(lambda s: s.split('_')[3].
    split('.')[0])
filesdf['index'] = pd.to_datetime(filesdf['dt_string'], format='%Y%m%d%H')
filesdf = filesdf.set_index('index')
df = pd.merge(df, filesdf, how='left', left_index=True, right_index=True)
dfna = df[df.isna().any(axis=1)]
dfna.to_csv(os.path.join(workDir, 'missingFiles.csv'))

#======================================================================================
# checkMissingForecasts.py
# Author: Mandi Thran
# Date: 01/10/21

# DESCRIPTION:
# This script checks for any BoM National Storm Surge forecasts that may be missing from 
# the WRL1 Coastal folder.  
#======================================================================================


from datetime import datetime, timedelta
import os
import pytz
import numpy as np
import pandas as pd

drive = "\\\\ad.unsw.edu.au\\OneUNSW\\ENG\\WRL\\WRL1"
workDir = "C:\\Users\\mandiruns\\Documents\\01_FEWS-RegionHome-Aus\\Scripts\\waterLevs"
outDir = os.path.join(drive,"Coastal\\Data\\Tide\\WL_Forecast\\BOM_Storm_Surge\\raw\\corrected")

fixPeriodStart = datetime(2019,9,13,0)
fixPeriodStart = pytz.utc.localize(fixPeriodStart)
fixPeriodEnd = datetime(2020,7,14)
fixPeriodEnd = pytz.utc.localize(fixPeriodEnd)

# Generate time series
timeSeries = np.arange(fixPeriodStart,fixPeriodEnd,timedelta(hours=6))
df = pd.DataFrame({'time_series':timeSeries})
df['index'] = df['time_series']
df = df.set_index("index")
df.index = pd.to_datetime(df.index, format="%Y-%m-%d %H:%M:%S")

# Load a list of files in that directory
filesdf = pd.DataFrame({"FileName": os.listdir(outDir)}) 
filesdf['dt_string'] = filesdf['FileName'].apply(lambda s:s.split('_')[3].split('.')[0])
filesdf['index'] = pd.to_datetime(filesdf['dt_string'],format="%Y%m%d%H")
filesdf = filesdf.set_index('index')

# Merge df on left
df = pd.merge(df, filesdf, how='left',left_index=True,right_index=True)
dfna = df[df.isna().any(axis=1)]
dfna.to_csv(os.path.join(workDir,"missingFiles.csv"))
