import glob
import torch
import torch.utils.data as data
import numpy as np
import os, sys, h5py

from pointnet2_ops import pointnet2_utils
from util import index_points

from torch.utils.data import DataLoader
from torch.utils.data import Dataset

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(BASE_DIR)


def download():
    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    DATA_DIR = os.path.join(BASE_DIR, 'data')
    if not os.path.exists(DATA_DIR):
        os.mkdir(DATA_DIR)
    if not os.path.exists(os.path.join(DATA_DIR, 'modelnet40_ply_hdf5_2048')):
        www = 'https://shapenet.cs.stanford.edu/media/modelnet40_ply_hdf5_2048.zip'
        zipfile = os.path.basename(www)
        os.system('wget %s; unzip %s' % (www, zipfile))
        os.system('mv %s %s' % (zipfile[:-4], DATA_DIR))
        os.system('rm %s' % (zipfile))

def load_data(partition):
    download()
    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    DATA_DIR = os.path.join(BASE_DIR, 'data')
    all_data = []
    all_label = []
    for h5_name in glob.glob(os.path.join(DATA_DIR, 'modelnet40_ply_hdf5_2048', 'ply_data_%s*.h5'%partition)):
        f = h5py.File(h5_name)
        data = f['data'][:].astype('float32')
        label = f['label'][:].astype('int64')
        f.close()
        all_data.append(data)
        all_label.append(label)
    all_data = np.concatenate(all_data, axis=0)
    all_label = np.concatenate(all_label, axis=0)
    return all_data, all_label

def random_point_dropout(pc, max_dropout_ratio=0.875):
    ''' batch_pc: BxNx3 '''
    # for b in range(batch_pc.shape[0]):
    dropout_ratio = np.random.random()*max_dropout_ratio # 0~0.875    
    drop_idx = np.where(np.random.random((pc.shape[0]))<=dropout_ratio)[0]
    # print ('use random drop', len(drop_idx))

    if len(drop_idx)>0:
        pc[drop_idx,:] = pc[0,:] # set to the first point
    return pc

def translate_pointcloud(pointcloud):
    xyz1 = np.random.uniform(low=2./3., high=3./2., size=[3])
    xyz2 = np.random.uniform(low=-0.2, high=0.2, size=[3])
       
    translated_pointcloud = np.add(np.multiply(pointcloud, xyz1), xyz2).astype('float32')
    return translated_pointcloud

def jitter_pointcloud(pointcloud, sigma=0.01, clip=0.02):
    N, C = pointcloud.shape
    pointcloud += np.clip(sigma * np.random.randn(N, C), -1*clip, clip)
    return pointcloud


class ModelNet40(Dataset):
    def __init__(self, num_points, partition='train'):
        self.data, self.label = load_data(partition)
        self.num_points = num_points
        self.partition = partition        

    def __getitem__(self, item):
        pointcloud = self.data[item][:self.num_points]
        label = self.label[item]
        if self.partition == 'train':
            pointcloud = random_point_dropout(pointcloud) # open for dgcnn not for our idea  for all
            pointcloud = translate_pointcloud(pointcloud)
            np.random.shuffle(pointcloud)
        return pointcloud, label

    def __len__(self):
        return self.data.shape[0]

class ScanObjectNNCls(data.Dataset):

    def __init__(
            self, transforms=None, train=True, self_supervision=False, num_points=2048,
    ):
        super().__init__()

        self.transforms = transforms

        self.self_supervision = self_supervision

        self.train = train

        self.num_points = num_points

        root = '/home/zhangniansong/data/ScanObjNN/main_split_nobg/'
        if self.self_supervision:
            h5 = h5py.File(root + 'training_objectdataset.h5', 'r')
            points_train = np.array(h5['data']).astype(np.float32)
            h5.close()
            self.points = points_train
            self.labels = None
        elif train:
            h5 = h5py.File(root + 'training_objectdataset.h5', 'r')
            self.points = np.array(h5['data']).astype(np.float32)
            self.labels = np.array(h5['label']).astype(int)
            h5.close()
        else:
            h5 = h5py.File(root + 'test_objectdataset.h5', 'r')
            self.points = np.array(h5['data']).astype(np.float32)
            self.labels = np.array(h5['label']).astype(int)
            h5.close()

        self.points_ = torch.tensor(self.points).cuda() # maybe modify to `to(device)`
        fps_idx = pointnet2_utils.furthest_point_sample(self.points_, self.num_points)
        self.points_ = index_points(self.points_, fps_idx.long())
        
        # BUG FIX:
        # if move data on gpu in dataset, will raise CUDA_NOT_INITIALIZED error in dataloader
        self.points = self.points_.cpu()

        del self.points_
        torch.cuda.empty_cache()

        print('Successfully load ScanObjectNN with', len(self.labels), 'instances')

    def __getitem__(self, idx):
        pt_idxs = np.arange(0, self.points.shape[1])   # 2048
        if self.train:
            np.random.shuffle(pt_idxs)
        
        current_points = self.points[idx, pt_idxs].clone()
        
        if self.transforms is not None:
            current_points = self.transforms(current_points)

        if self.self_supervision:
            return current_points
        else:
            label = self.labels[idx]
            return current_points, label

    def __len__(self):
        return self.points.shape[0]

if __name__ == '__main__':
    train = ModelNet40(1024)
    # test = ModelNet40(1024, 'test')
    for data, label in train:
        print(data.shape)
        print(label.shape)
        import ipdb; ipdb.set_trace()
    '''
    trainset = ScanObjectNNCls(num_points=1024)
    train_loader = DataLoader(trainset, batch_size=2,shuffle=True, drop_last=True)
    for data, label in train_loader:
        print(data.shape)
    '''


189dbcefbdf604745e39d3875e1b83f5b50c898a
<mask token>


def download():
    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    DATA_DIR = os.path.join(BASE_DIR, 'data')
    if not os.path.exists(DATA_DIR):
        os.mkdir(DATA_DIR)
    if not os.path.exists(os.path.join(DATA_DIR, 'modelnet40_ply_hdf5_2048')):
        www = (
            'https://shapenet.cs.stanford.edu/media/modelnet40_ply_hdf5_2048.zip'
            )
        zipfile = os.path.basename(www)
        os.system('wget %s; unzip %s' % (www, zipfile))
        os.system('mv %s %s' % (zipfile[:-4], DATA_DIR))
        os.system('rm %s' % zipfile)


<mask token>


def random_point_dropout(pc, max_dropout_ratio=0.875):
    """ batch_pc: BxNx3 """
    dropout_ratio = np.random.random() * max_dropout_ratio
    drop_idx = np.where(np.random.random(pc.shape[0]) <= dropout_ratio)[0]
    if len(drop_idx) > 0:
        pc[drop_idx, :] = pc[0, :]
    return pc


def translate_pointcloud(pointcloud):
    xyz1 = np.random.uniform(low=2.0 / 3.0, high=3.0 / 2.0, size=[3])
    xyz2 = np.random.uniform(low=-0.2, high=0.2, size=[3])
    translated_pointcloud = np.add(np.multiply(pointcloud, xyz1), xyz2).astype(
        'float32')
    return translated_pointcloud


def jitter_pointcloud(pointcloud, sigma=0.01, clip=0.02):
    N, C = pointcloud.shape
    pointcloud += np.clip(sigma * np.random.randn(N, C), -1 * clip, clip)
    return pointcloud


class ModelNet40(Dataset):

    def __init__(self, num_points, partition='train'):
        self.data, self.label = load_data(partition)
        self.num_points = num_points
        self.partition = partition

    def __getitem__(self, item):
        pointcloud = self.data[item][:self.num_points]
        label = self.label[item]
        if self.partition == 'train':
            pointcloud = random_point_dropout(pointcloud)
            pointcloud = translate_pointcloud(pointcloud)
            np.random.shuffle(pointcloud)
        return pointcloud, label

    def __len__(self):
        return self.data.shape[0]


class ScanObjectNNCls(data.Dataset):

    def __init__(self, transforms=None, train=True, self_supervision=False,
        num_points=2048):
        super().__init__()
        self.transforms = transforms
        self.self_supervision = self_supervision
        self.train = train
        self.num_points = num_points
        root = '/home/zhangniansong/data/ScanObjNN/main_split_nobg/'
        if self.self_supervision:
            h5 = h5py.File(root + 'training_objectdataset.h5', 'r')
            points_train = np.array(h5['data']).astype(np.float32)
            h5.close()
            self.points = points_train
            self.labels = None
        elif train:
            h5 = h5py.File(root + 'training_objectdataset.h5', 'r')
            self.points = np.array(h5['data']).astype(np.float32)
            self.labels = np.array(h5['label']).astype(int)
            h5.close()
        else:
            h5 = h5py.File(root + 'test_objectdataset.h5', 'r')
            self.points = np.array(h5['data']).astype(np.float32)
            self.labels = np.array(h5['label']).astype(int)
            h5.close()
        self.points_ = torch.tensor(self.points).cuda()
        fps_idx = pointnet2_utils.furthest_point_sample(self.points_, self.
            num_points)
        self.points_ = index_points(self.points_, fps_idx.long())
        self.points = self.points_.cpu()
        del self.points_
        torch.cuda.empty_cache()
        print('Successfully load ScanObjectNN with', len(self.labels),
            'instances')

    def __getitem__(self, idx):
        pt_idxs = np.arange(0, self.points.shape[1])
        if self.train:
            np.random.shuffle(pt_idxs)
        current_points = self.points[idx, pt_idxs].clone()
        if self.transforms is not None:
            current_points = self.transforms(current_points)
        if self.self_supervision:
            return current_points
        else:
            label = self.labels[idx]
            return current_points, label

    def __len__(self):
        return self.points.shape[0]


<mask token>

<mask token>


def download():
    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    DATA_DIR = os.path.join(BASE_DIR, 'data')
    if not os.path.exists(DATA_DIR):
        os.mkdir(DATA_DIR)
    if not os.path.exists(os.path.join(DATA_DIR, 'modelnet40_ply_hdf5_2048')):
        www = (
            'https://shapenet.cs.stanford.edu/media/modelnet40_ply_hdf5_2048.zip'
            )
        zipfile = os.path.basename(www)
        os.system('wget %s; unzip %s' % (www, zipfile))
        os.system('mv %s %s' % (zipfile[:-4], DATA_DIR))
        os.system('rm %s' % zipfile)


def load_data(partition):
    download()
    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    DATA_DIR = os.path.join(BASE_DIR, 'data')
    all_data = []
    all_label = []
    for h5_name in glob.glob(os.path.join(DATA_DIR,
        'modelnet40_ply_hdf5_2048', 'ply_data_%s*.h5' % partition)):
        f = h5py.File(h5_name)
        data = f['data'][:].astype('float32')
        label = f['label'][:].astype('int64')
        f.close()
        all_data.append(data)
        all_label.append(label)
    all_data = np.concatenate(all_data, axis=0)
    all_label = np.concatenate(all_label, axis=0)
    return all_data, all_label


def random_point_dropout(pc, max_dropout_ratio=0.875):
    """ batch_pc: BxNx3 """
    dropout_ratio = np.random.random() * max_dropout_ratio
    drop_idx = np.where(np.random.random(pc.shape[0]) <= dropout_ratio)[0]
    if len(drop_idx) > 0:
        pc[drop_idx, :] = pc[0, :]
    return pc


def translate_pointcloud(pointcloud):
    xyz1 = np.random.uniform(low=2.0 / 3.0, high=3.0 / 2.0, size=[3])
    xyz2 = np.random.uniform(low=-0.2, high=0.2, size=[3])
    translated_pointcloud = np.add(np.multiply(pointcloud, xyz1), xyz2).astype(
        'float32')
    return translated_pointcloud


def jitter_pointcloud(pointcloud, sigma=0.01, clip=0.02):
    N, C = pointcloud.shape
    pointcloud += np.clip(sigma * np.random.randn(N, C), -1 * clip, clip)
    return pointcloud


class ModelNet40(Dataset):

    def __init__(self, num_points, partition='train'):
        self.data, self.label = load_data(partition)
        self.num_points = num_points
        self.partition = partition

    def __getitem__(self, item):
        pointcloud = self.data[item][:self.num_points]
        label = self.label[item]
        if self.partition == 'train':
            pointcloud = random_point_dropout(pointcloud)
            pointcloud = translate_pointcloud(pointcloud)
            np.random.shuffle(pointcloud)
        return pointcloud, label

    def __len__(self):
        return self.data.shape[0]


class ScanObjectNNCls(data.Dataset):

    def __init__(self, transforms=None, train=True, self_supervision=False,
        num_points=2048):
        super().__init__()
        self.transforms = transforms
        self.self_supervision = self_supervision
        self.train = train
        self.num_points = num_points
        root = '/home/zhangniansong/data/ScanObjNN/main_split_nobg/'
        if self.self_supervision:
            h5 = h5py.File(root + 'training_objectdataset.h5', 'r')
            points_train = np.array(h5['data']).astype(np.float32)
            h5.close()
            self.points = points_train
            self.labels = None
        elif train:
            h5 = h5py.File(root + 'training_objectdataset.h5', 'r')
            self.points = np.array(h5['data']).astype(np.float32)
            self.labels = np.array(h5['label']).astype(int)
            h5.close()
        else:
            h5 = h5py.File(root + 'test_objectdataset.h5', 'r')
            self.points = np.array(h5['data']).astype(np.float32)
            self.labels = np.array(h5['label']).astype(int)
            h5.close()
        self.points_ = torch.tensor(self.points).cuda()
        fps_idx = pointnet2_utils.furthest_point_sample(self.points_, self.
            num_points)
        self.points_ = index_points(self.points_, fps_idx.long())
        self.points = self.points_.cpu()
        del self.points_
        torch.cuda.empty_cache()
        print('Successfully load ScanObjectNN with', len(self.labels),
            'instances')

    def __getitem__(self, idx):
        pt_idxs = np.arange(0, self.points.shape[1])
        if self.train:
            np.random.shuffle(pt_idxs)
        current_points = self.points[idx, pt_idxs].clone()
        if self.transforms is not None:
            current_points = self.transforms(current_points)
        if self.self_supervision:
            return current_points
        else:
            label = self.labels[idx]
            return current_points, label

    def __len__(self):
        return self.points.shape[0]


<mask token>

<mask token>
sys.path.append(BASE_DIR)


def download():
    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    DATA_DIR = os.path.join(BASE_DIR, 'data')
    if not os.path.exists(DATA_DIR):
        os.mkdir(DATA_DIR)
    if not os.path.exists(os.path.join(DATA_DIR, 'modelnet40_ply_hdf5_2048')):
        www = (
            'https://shapenet.cs.stanford.edu/media/modelnet40_ply_hdf5_2048.zip'
            )
        zipfile = os.path.basename(www)
        os.system('wget %s; unzip %s' % (www, zipfile))
        os.system('mv %s %s' % (zipfile[:-4], DATA_DIR))
        os.system('rm %s' % zipfile)


def load_data(partition):
    download()
    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    DATA_DIR = os.path.join(BASE_DIR, 'data')
    all_data = []
    all_label = []
    for h5_name in glob.glob(os.path.join(DATA_DIR,
        'modelnet40_ply_hdf5_2048', 'ply_data_%s*.h5' % partition)):
        f = h5py.File(h5_name)
        data = f['data'][:].astype('float32')
        label = f['label'][:].astype('int64')
        f.close()
        all_data.append(data)
        all_label.append(label)
    all_data = np.concatenate(all_data, axis=0)
    all_label = np.concatenate(all_label, axis=0)
    return all_data, all_label


def random_point_dropout(pc, max_dropout_ratio=0.875):
    """ batch_pc: BxNx3 """
    dropout_ratio = np.random.random() * max_dropout_ratio
    drop_idx = np.where(np.random.random(pc.shape[0]) <= dropout_ratio)[0]
    if len(drop_idx) > 0:
        pc[drop_idx, :] = pc[0, :]
    return pc


def translate_pointcloud(pointcloud):
    xyz1 = np.random.uniform(low=2.0 / 3.0, high=3.0 / 2.0, size=[3])
    xyz2 = np.random.uniform(low=-0.2, high=0.2, size=[3])
    translated_pointcloud = np.add(np.multiply(pointcloud, xyz1), xyz2).astype(
        'float32')
    return translated_pointcloud


def jitter_pointcloud(pointcloud, sigma=0.01, clip=0.02):
    N, C = pointcloud.shape
    pointcloud += np.clip(sigma * np.random.randn(N, C), -1 * clip, clip)
    return pointcloud


class ModelNet40(Dataset):

    def __init__(self, num_points, partition='train'):
        self.data, self.label = load_data(partition)
        self.num_points = num_points
        self.partition = partition

    def __getitem__(self, item):
        pointcloud = self.data[item][:self.num_points]
        label = self.label[item]
        if self.partition == 'train':
            pointcloud = random_point_dropout(pointcloud)
            pointcloud = translate_pointcloud(pointcloud)
            np.random.shuffle(pointcloud)
        return pointcloud, label

    def __len__(self):
        return self.data.shape[0]


class ScanObjectNNCls(data.Dataset):

    def __init__(self, transforms=None, train=True, self_supervision=False,
        num_points=2048):
        super().__init__()
        self.transforms = transforms
        self.self_supervision = self_supervision
        self.train = train
        self.num_points = num_points
        root = '/home/zhangniansong/data/ScanObjNN/main_split_nobg/'
        if self.self_supervision:
            h5 = h5py.File(root + 'training_objectdataset.h5', 'r')
            points_train = np.array(h5['data']).astype(np.float32)
            h5.close()
            self.points = points_train
            self.labels = None
        elif train:
            h5 = h5py.File(root + 'training_objectdataset.h5', 'r')
            self.points = np.array(h5['data']).astype(np.float32)
            self.labels = np.array(h5['label']).astype(int)
            h5.close()
        else:
            h5 = h5py.File(root + 'test_objectdataset.h5', 'r')
            self.points = np.array(h5['data']).astype(np.float32)
            self.labels = np.array(h5['label']).astype(int)
            h5.close()
        self.points_ = torch.tensor(self.points).cuda()
        fps_idx = pointnet2_utils.furthest_point_sample(self.points_, self.
            num_points)
        self.points_ = index_points(self.points_, fps_idx.long())
        self.points = self.points_.cpu()
        del self.points_
        torch.cuda.empty_cache()
        print('Successfully load ScanObjectNN with', len(self.labels),
            'instances')

    def __getitem__(self, idx):
        pt_idxs = np.arange(0, self.points.shape[1])
        if self.train:
            np.random.shuffle(pt_idxs)
        current_points = self.points[idx, pt_idxs].clone()
        if self.transforms is not None:
            current_points = self.transforms(current_points)
        if self.self_supervision:
            return current_points
        else:
            label = self.labels[idx]
            return current_points, label

    def __len__(self):
        return self.points.shape[0]


if __name__ == '__main__':
    train = ModelNet40(1024)
    for data, label in train:
        print(data.shape)
        print(label.shape)
        import ipdb
        ipdb.set_trace()
    """
    trainset = ScanObjectNNCls(num_points=1024)
    train_loader = DataLoader(trainset, batch_size=2,shuffle=True, drop_last=True)
    for data, label in train_loader:
        print(data.shape)
    """

import glob
import torch
import torch.utils.data as data
import numpy as np
import os, sys, h5py
from pointnet2_ops import pointnet2_utils
from util import index_points
from torch.utils.data import DataLoader
from torch.utils.data import Dataset
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(BASE_DIR)


def download():
    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    DATA_DIR = os.path.join(BASE_DIR, 'data')
    if not os.path.exists(DATA_DIR):
        os.mkdir(DATA_DIR)
    if not os.path.exists(os.path.join(DATA_DIR, 'modelnet40_ply_hdf5_2048')):
        www = (
            'https://shapenet.cs.stanford.edu/media/modelnet40_ply_hdf5_2048.zip'
            )
        zipfile = os.path.basename(www)
        os.system('wget %s; unzip %s' % (www, zipfile))
        os.system('mv %s %s' % (zipfile[:-4], DATA_DIR))
        os.system('rm %s' % zipfile)


def load_data(partition):
    download()
    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    DATA_DIR = os.path.join(BASE_DIR, 'data')
    all_data = []
    all_label = []
    for h5_name in glob.glob(os.path.join(DATA_DIR,
        'modelnet40_ply_hdf5_2048', 'ply_data_%s*.h5' % partition)):
        f = h5py.File(h5_name)
        data = f['data'][:].astype('float32')
        label = f['label'][:].astype('int64')
        f.close()
        all_data.append(data)
        all_label.append(label)
    all_data = np.concatenate(all_data, axis=0)
    all_label = np.concatenate(all_label, axis=0)
    return all_data, all_label


def random_point_dropout(pc, max_dropout_ratio=0.875):
    """ batch_pc: BxNx3 """
    dropout_ratio = np.random.random() * max_dropout_ratio
    drop_idx = np.where(np.random.random(pc.shape[0]) <= dropout_ratio)[0]
    if len(drop_idx) > 0:
        pc[drop_idx, :] = pc[0, :]
    return pc


def translate_pointcloud(pointcloud):
    xyz1 = np.random.uniform(low=2.0 / 3.0, high=3.0 / 2.0, size=[3])
    xyz2 = np.random.uniform(low=-0.2, high=0.2, size=[3])
    translated_pointcloud = np.add(np.multiply(pointcloud, xyz1), xyz2).astype(
        'float32')
    return translated_pointcloud


def jitter_pointcloud(pointcloud, sigma=0.01, clip=0.02):
    N, C = pointcloud.shape
    pointcloud += np.clip(sigma * np.random.randn(N, C), -1 * clip, clip)
    return pointcloud


class ModelNet40(Dataset):

    def __init__(self, num_points, partition='train'):
        self.data, self.label = load_data(partition)
        self.num_points = num_points
        self.partition = partition

    def __getitem__(self, item):
        pointcloud = self.data[item][:self.num_points]
        label = self.label[item]
        if self.partition == 'train':
            pointcloud = random_point_dropout(pointcloud)
            pointcloud = translate_pointcloud(pointcloud)
            np.random.shuffle(pointcloud)
        return pointcloud, label

    def __len__(self):
        return self.data.shape[0]


class ScanObjectNNCls(data.Dataset):

    def __init__(self, transforms=None, train=True, self_supervision=False,
        num_points=2048):
        super().__init__()
        self.transforms = transforms
        self.self_supervision = self_supervision
        self.train = train
        self.num_points = num_points
        root = '/home/zhangniansong/data/ScanObjNN/main_split_nobg/'
        if self.self_supervision:
            h5 = h5py.File(root + 'training_objectdataset.h5', 'r')
            points_train = np.array(h5['data']).astype(np.float32)
            h5.close()
            self.points = points_train
            self.labels = None
        elif train:
            h5 = h5py.File(root + 'training_objectdataset.h5', 'r')
            self.points = np.array(h5['data']).astype(np.float32)
            self.labels = np.array(h5['label']).astype(int)
            h5.close()
        else:
            h5 = h5py.File(root + 'test_objectdataset.h5', 'r')
            self.points = np.array(h5['data']).astype(np.float32)
            self.labels = np.array(h5['label']).astype(int)
            h5.close()
        self.points_ = torch.tensor(self.points).cuda()
        fps_idx = pointnet2_utils.furthest_point_sample(self.points_, self.
            num_points)
        self.points_ = index_points(self.points_, fps_idx.long())
        self.points = self.points_.cpu()
        del self.points_
        torch.cuda.empty_cache()
        print('Successfully load ScanObjectNN with', len(self.labels),
            'instances')

    def __getitem__(self, idx):
        pt_idxs = np.arange(0, self.points.shape[1])
        if self.train:
            np.random.shuffle(pt_idxs)
        current_points = self.points[idx, pt_idxs].clone()
        if self.transforms is not None:
            current_points = self.transforms(current_points)
        if self.self_supervision:
            return current_points
        else:
            label = self.labels[idx]
            return current_points, label

    def __len__(self):
        return self.points.shape[0]


if __name__ == '__main__':
    train = ModelNet40(1024)
    for data, label in train:
        print(data.shape)
        print(label.shape)
        import ipdb
        ipdb.set_trace()
    """
    trainset = ScanObjectNNCls(num_points=1024)
    train_loader = DataLoader(trainset, batch_size=2,shuffle=True, drop_last=True)
    for data, label in train_loader:
        print(data.shape)
    """

import glob
import torch
import torch.utils.data as data
import numpy as np
import os, sys, h5py

from pointnet2_ops import pointnet2_utils
from util import index_points

from torch.utils.data import DataLoader
from torch.utils.data import Dataset

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
sys.path.append(BASE_DIR)


def download():
    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    DATA_DIR = os.path.join(BASE_DIR, 'data')
    if not os.path.exists(DATA_DIR):
        os.mkdir(DATA_DIR)
    if not os.path.exists(os.path.join(DATA_DIR, 'modelnet40_ply_hdf5_2048')):
        www = 'https://shapenet.cs.stanford.edu/media/modelnet40_ply_hdf5_2048.zip'
        zipfile = os.path.basename(www)
        os.system('wget %s; unzip %s' % (www, zipfile))
        os.system('mv %s %s' % (zipfile[:-4], DATA_DIR))
        os.system('rm %s' % (zipfile))

def load_data(partition):
    download()
    BASE_DIR = os.path.dirname(os.path.abspath(__file__))
    DATA_DIR = os.path.join(BASE_DIR, 'data')
    all_data = []
    all_label = []
    for h5_name in glob.glob(os.path.join(DATA_DIR, 'modelnet40_ply_hdf5_2048', 'ply_data_%s*.h5'%partition)):
        f = h5py.File(h5_name)
        data = f['data'][:].astype('float32')
        label = f['label'][:].astype('int64')
        f.close()
        all_data.append(data)
        all_label.append(label)
    all_data = np.concatenate(all_data, axis=0)
    all_label = np.concatenate(all_label, axis=0)
    return all_data, all_label

def random_point_dropout(pc, max_dropout_ratio=0.875):
    ''' batch_pc: BxNx3 '''
    # for b in range(batch_pc.shape[0]):
    dropout_ratio = np.random.random()*max_dropout_ratio # 0~0.875    
    drop_idx = np.where(np.random.random((pc.shape[0]))<=dropout_ratio)[0]
    # print ('use random drop', len(drop_idx))

    if len(drop_idx)>0:
        pc[drop_idx,:] = pc[0,:] # set to the first point
    return pc

def translate_pointcloud(pointcloud):
    xyz1 = np.random.uniform(low=2./3., high=3./2., size=[3])
    xyz2 = np.random.uniform(low=-0.2, high=0.2, size=[3])
       
    translated_pointcloud = np.add(np.multiply(pointcloud, xyz1), xyz2).astype('float32')
    return translated_pointcloud

def jitter_pointcloud(pointcloud, sigma=0.01, clip=0.02):
    N, C = pointcloud.shape
    pointcloud += np.clip(sigma * np.random.randn(N, C), -1*clip, clip)
    return pointcloud


class ModelNet40(Dataset):
    def __init__(self, num_points, partition='train'):
        self.data, self.label = load_data(partition)
        self.num_points = num_points
        self.partition = partition        

    def __getitem__(self, item):
        pointcloud = self.data[item][:self.num_points]
        label = self.label[item]
        if self.partition == 'train':
            pointcloud = random_point_dropout(pointcloud) # open for dgcnn not for our idea  for all
            pointcloud = translate_pointcloud(pointcloud)
            np.random.shuffle(pointcloud)
        return pointcloud, label

    def __len__(self):
        return self.data.shape[0]

class ScanObjectNNCls(data.Dataset):

    def __init__(
            self, transforms=None, train=True, self_supervision=False, num_points=2048,
    ):
        super().__init__()

        self.transforms = transforms

        self.self_supervision = self_supervision

        self.train = train

        self.num_points = num_points

        root = '/home/zhangniansong/data/ScanObjNN/main_split_nobg/'
        if self.self_supervision:
            h5 = h5py.File(root + 'training_objectdataset.h5', 'r')
            points_train = np.array(h5['data']).astype(np.float32)
            h5.close()
            self.points = points_train
            self.labels = None
        elif train:
            h5 = h5py.File(root + 'training_objectdataset.h5', 'r')
            self.points = np.array(h5['data']).astype(np.float32)
            self.labels = np.array(h5['label']).astype(int)
            h5.close()
        else:
            h5 = h5py.File(root + 'test_objectdataset.h5', 'r')
            self.points = np.array(h5['data']).astype(np.float32)
            self.labels = np.array(h5['label']).astype(int)
            h5.close()

        self.points_ = torch.tensor(self.points).cuda() # maybe modify to `to(device)`
        fps_idx = pointnet2_utils.furthest_point_sample(self.points_, self.num_points)
        self.points_ = index_points(self.points_, fps_idx.long())
        
        # BUG FIX:
        # if move data on gpu in dataset, will raise CUDA_NOT_INITIALIZED error in dataloader
        self.points = self.points_.cpu()

        del self.points_
        torch.cuda.empty_cache()

        print('Successfully load ScanObjectNN with', len(self.labels), 'instances')

    def __getitem__(self, idx):
        pt_idxs = np.arange(0, self.points.shape[1])   # 2048
        if self.train:
            np.random.shuffle(pt_idxs)
        
        current_points = self.points[idx, pt_idxs].clone()
        
        if self.transforms is not None:
            current_points = self.transforms(current_points)

        if self.self_supervision:
            return current_points
        else:
            label = self.labels[idx]
            return current_points, label

    def __len__(self):
        return self.points.shape[0]

if __name__ == '__main__':
    train = ModelNet40(1024)
    # test = ModelNet40(1024, 'test')
    for data, label in train:
        print(data.shape)
        print(label.shape)
        import ipdb; ipdb.set_trace()
    '''
    trainset = ScanObjectNNCls(num_points=1024)
    train_loader = DataLoader(trainset, batch_size=2,shuffle=True, drop_last=True)
    for data, label in train_loader:
        print(data.shape)
    '''

