cleaning up imports in data processing scripts (#41)

Author: azrael417

data_process/annotate_dataset.py

@@ -16,8 +16,6 @@
 import os
 from typing import List, Optional
 import json
-import time
-import pickle
 import numpy as np
 import h5py as h5
 import datetime as dt

data_process/convert_makani_output_to_wb2.py

@@ -16,10 +16,7 @@
 from typing import Optional, List
 import progressbar
 import os
-import re
-import json
 import time
-import pickle
 import numpy as np
 import h5py as h5
 import datetime as dt
@@ -31,7 +28,6 @@
 # MPI
 from mpi4py import MPI
 
-from makani.utils.features import get_channel_groups
 from makani.utils.dataloaders.data_helpers import get_date_from_timestamp
 
 from wb2_helpers import surface_variables, atmospheric_variables, split_convert_channel_names

data_process/convert_wb2_to_makani_input.py

@@ -15,25 +15,17 @@
 
 from typing import Optional, List
 from itertools import batched
-import progressbar
 import os
-import re
 import json
 import time
-import pickle
 import numpy as np
 import h5py as h5
 import datetime as dt
 import argparse as ap
-from glob import glob
 import xarray as xr
 
 # MPI
 from mpi4py import MPI
-from mpi4py.util import dtlib
-
-from makani.utils.features import get_channel_groups
-from makani.utils.dataloaders.data_helpers import get_date_from_timestamp
 
 from wb2_helpers import surface_variables, atmospheric_variables, split_convert_channel_names, DistributedProgressBar
 

data_process/generate_wb2_climatology.py

@@ -13,8 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from typings import Optional
-import re
+from typing import Optional
 import argparse as ap
 import xarray as xr
 import h5py as h5
@@ -25,8 +24,6 @@
 
 from mpi4py import MPI
 
-from makani.utils.dataloaders.data_helpers import get_timestamp
-from makani.utils.features import get_channel_groups
 from wb2_helpers import split_convert_channel_names
 
 

data_process/get_histograms.py

@@ -20,9 +20,8 @@
 import pickle
 import numpy as np
 import h5py as h5
-from math import ceil
 import argparse as ap
-from itertools import groupby, accumulate
+from itertools import accumulate
 import operator
 from bisect import bisect_right
 from tqdm import tqdm
@@ -36,7 +35,6 @@
 
 # we need that for quadrature
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-import torch
 from makani.utils.grids import GridQuadrature
 
 

data_process/h5_convert.py

@@ -15,8 +15,6 @@
 
 import os
 from typing import Optional
-import sys
-import shutil
 import glob
 import argparse as ap 
 import h5py as h5

data_process/merge_wb2_dataset.py

@@ -16,27 +16,14 @@
 from typing import Optional, List
 from itertools import batched
 import progressbar
-import os
-import re
-import json
 import time
-import pickle
-import numpy as np
 import h5py as h5
 import datetime as dt
 import argparse as ap
-from glob import glob
-import xarray as xr
 
 # MPI
 from mpi4py import MPI
 
-from makani.utils.features import get_channel_groups
-from makani.utils.dataloaders.data_helpers import get_date_from_timestamp
-
-from wb2_helpers import surface_variables, atmospheric_variables, split_convert_channel_names
-
-
 def transfer_channels(input_file: str, output_file: str, channels: List[str],
                       batch_size: Optional[int]=32, entry_key: Optional[str]='fields',
                       verbose: Optional[bool]=False):