import os

import groot
import numpy as np
import PIL
import torch
import torchvision


class GroceryDataset(torch.utils.data.Dataset):
    def __init__(
        self,
        root,
        split="train",
        string_labels=True,
        transforms=None,
        adjust_freq=True,
        eps=1e-5,
        use_all_labels=False,
        label_level=-1,
    ):

        self.root = root
        self.label_file_path = os.path.join(root, split + ".txt")
        self.image_paths = self._load_label_path_file()
        self.label_tuples = [p.split("/")[1:-1] for p in self.image_paths]
        self.string_labels = string_labels
        self.concept_tree = groot.Tree.from_yaml(
            open(os.path.join(root, "concept_tree.yaml"), "r").read()
        )
        self.float_onehot_label_array = np.load(os.path.join(root, "label_arrays.npz"))[
            "onehot"
        ]
        self.transforms = transforms
        self.adjust_freq = adjust_freq
        self.use_all_labels = use_all_labels
        self.label_level = label_level

        if adjust_freq:
            self.class_counts = self._count_labels()
            levels = self.concept_tree.levels
            self.counts_by_level = [
                {cat: self.class_counts[cat] for cat in level} for level in levels
            ]
            name_to_id = self.concept_tree.node_label
            freq = np.zeros_like(self.float_onehot_label_array)
            level_total_counts = [
                sum(v for k, v in level.items()) for level in self.counts_by_level
            ]
            self.freq_by_level = [
                {cat: counts[cat] / tot for cat in counts}
                for tot, counts in zip(level_total_counts, self.counts_by_level)
            ]
            for d in self.freq_by_level:
                for k, v in d.items():
                    freq[name_to_id[k]] += max(v, eps)
            freq = np.clip(a=freq, a_min=eps, a_max=1 / eps)

            self.frequency_array = freq
            self.freq_adjusted_onehot_array = self.float_onehot_label_array / freq

    def _count_labels(self):

        labels = self.label_tuples
        counts = dict(Grocery=len(labels))
        for label in labels:
            for lab in label:
                if lab in counts:
                    counts[lab] += 1
                else:
                    counts[lab] = 1
        return counts

    def _load_label_path_file(self):

        with open(self.label_file_path, "r") as f:
            paths = [ln.split(",")[0] for ln in f.readlines()]

        return paths

    def __getitem__(self, idx):

        img_path = os.path.join(self.root, self.image_paths[idx])
        lab_tups = self.label_tuples[idx]

        img = PIL.Image.open(img_path)

        if self.transforms:
            img = self.transforms(img)

        if self.string_labels:
            return dict(image=img, string_labels=lab_tups)
        else:
            int_label = [self.concept_tree.node_label[l] for l in lab_tups]
            if self.adjust_freq:
                onehot = self.freq_adjusted_onehot_array
            else:
                onehot = self.float_onehot_label_array

            vecs = onehot[int_label]  # [n_label, onehot_dim]
            if self.use_all_labels:
                vec = vecs.mean(0)
            else:
                vec = vecs[self.label_level]
            vec = torch.from_numpy(vec).float()
            return dict(image=img, label=vec)

    def __len__(self):
        return len(self.image_paths)

    def collect_statistics(self, stats_lambda, summary_lambda):
        return summary_lambda([stats_lambda(item) for item in self])


train_augmentation = torchvision.transforms.Compose(
    [
        torchvision.transforms.AutoAugment(),
        torchvision.transforms.RandomApply(
            [
                torchvision.transforms.RandomCrop(300, pad_if_needed=True),
                torchvision.transforms.RandomCrop(256, pad_if_needed=True),
            ]
        ),
        torchvision.transforms.Resize((256, 256)),
        torchvision.transforms.ToTensor(),
    ]
)


eval_augmentation = torchvision.transforms.Compose(
    [torchvision.transforms.Resize(256), torchvision.transforms.ToTensor()]
)


def pil_image_size(item):
    W, H = item["image"].size
    return dict(width=W, height=H)


def get_size_stats(sizes):
    h_ = []
    w_ = []
    for d in sizes:
        h_.append(d["height"])
        w_.append(d["width"])

    return {
        "max height": max(h_),
        "min height": min(h_),
        "max width": max(w_),
        "min width": min(w_),
        "mean height": sum(h_) / len(h_),
        "mean width": sum(w_) / len(w_),
    }

bf58db0be13530e9c42c3a6ac4f80939365b356b
<mask token>


class GroceryDataset(torch.utils.data.Dataset):

    def __init__(self, root, split='train', string_labels=True, transforms=
        None, adjust_freq=True, eps=1e-05, use_all_labels=False, label_level=-1
        ):
        self.root = root
        self.label_file_path = os.path.join(root, split + '.txt')
        self.image_paths = self._load_label_path_file()
        self.label_tuples = [p.split('/')[1:-1] for p in self.image_paths]
        self.string_labels = string_labels
        self.concept_tree = groot.Tree.from_yaml(open(os.path.join(root,
            'concept_tree.yaml'), 'r').read())
        self.float_onehot_label_array = np.load(os.path.join(root,
            'label_arrays.npz'))['onehot']
        self.transforms = transforms
        self.adjust_freq = adjust_freq
        self.use_all_labels = use_all_labels
        self.label_level = label_level
        if adjust_freq:
            self.class_counts = self._count_labels()
            levels = self.concept_tree.levels
            self.counts_by_level = [{cat: self.class_counts[cat] for cat in
                level} for level in levels]
            name_to_id = self.concept_tree.node_label
            freq = np.zeros_like(self.float_onehot_label_array)
            level_total_counts = [sum(v for k, v in level.items()) for
                level in self.counts_by_level]
            self.freq_by_level = [{cat: (counts[cat] / tot) for cat in
                counts} for tot, counts in zip(level_total_counts, self.
                counts_by_level)]
            for d in self.freq_by_level:
                for k, v in d.items():
                    freq[name_to_id[k]] += max(v, eps)
            freq = np.clip(a=freq, a_min=eps, a_max=1 / eps)
            self.frequency_array = freq
            self.freq_adjusted_onehot_array = (self.
                float_onehot_label_array / freq)

    def _count_labels(self):
        labels = self.label_tuples
        counts = dict(Grocery=len(labels))
        for label in labels:
            for lab in label:
                if lab in counts:
                    counts[lab] += 1
                else:
                    counts[lab] = 1
        return counts

    def _load_label_path_file(self):
        with open(self.label_file_path, 'r') as f:
            paths = [ln.split(',')[0] for ln in f.readlines()]
        return paths
    <mask token>

    def __len__(self):
        return len(self.image_paths)

    def collect_statistics(self, stats_lambda, summary_lambda):
        return summary_lambda([stats_lambda(item) for item in self])


<mask token>

<mask token>


class GroceryDataset(torch.utils.data.Dataset):

    def __init__(self, root, split='train', string_labels=True, transforms=
        None, adjust_freq=True, eps=1e-05, use_all_labels=False, label_level=-1
        ):
        self.root = root
        self.label_file_path = os.path.join(root, split + '.txt')
        self.image_paths = self._load_label_path_file()
        self.label_tuples = [p.split('/')[1:-1] for p in self.image_paths]
        self.string_labels = string_labels
        self.concept_tree = groot.Tree.from_yaml(open(os.path.join(root,
            'concept_tree.yaml'), 'r').read())
        self.float_onehot_label_array = np.load(os.path.join(root,
            'label_arrays.npz'))['onehot']
        self.transforms = transforms
        self.adjust_freq = adjust_freq
        self.use_all_labels = use_all_labels
        self.label_level = label_level
        if adjust_freq:
            self.class_counts = self._count_labels()
            levels = self.concept_tree.levels
            self.counts_by_level = [{cat: self.class_counts[cat] for cat in
                level} for level in levels]
            name_to_id = self.concept_tree.node_label
            freq = np.zeros_like(self.float_onehot_label_array)
            level_total_counts = [sum(v for k, v in level.items()) for
                level in self.counts_by_level]
            self.freq_by_level = [{cat: (counts[cat] / tot) for cat in
                counts} for tot, counts in zip(level_total_counts, self.
                counts_by_level)]
            for d in self.freq_by_level:
                for k, v in d.items():
                    freq[name_to_id[k]] += max(v, eps)
            freq = np.clip(a=freq, a_min=eps, a_max=1 / eps)
            self.frequency_array = freq
            self.freq_adjusted_onehot_array = (self.
                float_onehot_label_array / freq)

    def _count_labels(self):
        labels = self.label_tuples
        counts = dict(Grocery=len(labels))
        for label in labels:
            for lab in label:
                if lab in counts:
                    counts[lab] += 1
                else:
                    counts[lab] = 1
        return counts

    def _load_label_path_file(self):
        with open(self.label_file_path, 'r') as f:
            paths = [ln.split(',')[0] for ln in f.readlines()]
        return paths

    def __getitem__(self, idx):
        img_path = os.path.join(self.root, self.image_paths[idx])
        lab_tups = self.label_tuples[idx]
        img = PIL.Image.open(img_path)
        if self.transforms:
            img = self.transforms(img)
        if self.string_labels:
            return dict(image=img, string_labels=lab_tups)
        else:
            int_label = [self.concept_tree.node_label[l] for l in lab_tups]
            if self.adjust_freq:
                onehot = self.freq_adjusted_onehot_array
            else:
                onehot = self.float_onehot_label_array
            vecs = onehot[int_label]
            if self.use_all_labels:
                vec = vecs.mean(0)
            else:
                vec = vecs[self.label_level]
            vec = torch.from_numpy(vec).float()
            return dict(image=img, label=vec)

    def __len__(self):
        return len(self.image_paths)

    def collect_statistics(self, stats_lambda, summary_lambda):
        return summary_lambda([stats_lambda(item) for item in self])


<mask token>

<mask token>


class GroceryDataset(torch.utils.data.Dataset):

    def __init__(self, root, split='train', string_labels=True, transforms=
        None, adjust_freq=True, eps=1e-05, use_all_labels=False, label_level=-1
        ):
        self.root = root
        self.label_file_path = os.path.join(root, split + '.txt')
        self.image_paths = self._load_label_path_file()
        self.label_tuples = [p.split('/')[1:-1] for p in self.image_paths]
        self.string_labels = string_labels
        self.concept_tree = groot.Tree.from_yaml(open(os.path.join(root,
            'concept_tree.yaml'), 'r').read())
        self.float_onehot_label_array = np.load(os.path.join(root,
            'label_arrays.npz'))['onehot']
        self.transforms = transforms
        self.adjust_freq = adjust_freq
        self.use_all_labels = use_all_labels
        self.label_level = label_level
        if adjust_freq:
            self.class_counts = self._count_labels()
            levels = self.concept_tree.levels
            self.counts_by_level = [{cat: self.class_counts[cat] for cat in
                level} for level in levels]
            name_to_id = self.concept_tree.node_label
            freq = np.zeros_like(self.float_onehot_label_array)
            level_total_counts = [sum(v for k, v in level.items()) for
                level in self.counts_by_level]
            self.freq_by_level = [{cat: (counts[cat] / tot) for cat in
                counts} for tot, counts in zip(level_total_counts, self.
                counts_by_level)]
            for d in self.freq_by_level:
                for k, v in d.items():
                    freq[name_to_id[k]] += max(v, eps)
            freq = np.clip(a=freq, a_min=eps, a_max=1 / eps)
            self.frequency_array = freq
            self.freq_adjusted_onehot_array = (self.
                float_onehot_label_array / freq)

    def _count_labels(self):
        labels = self.label_tuples
        counts = dict(Grocery=len(labels))
        for label in labels:
            for lab in label:
                if lab in counts:
                    counts[lab] += 1
                else:
                    counts[lab] = 1
        return counts

    def _load_label_path_file(self):
        with open(self.label_file_path, 'r') as f:
            paths = [ln.split(',')[0] for ln in f.readlines()]
        return paths

    def __getitem__(self, idx):
        img_path = os.path.join(self.root, self.image_paths[idx])
        lab_tups = self.label_tuples[idx]
        img = PIL.Image.open(img_path)
        if self.transforms:
            img = self.transforms(img)
        if self.string_labels:
            return dict(image=img, string_labels=lab_tups)
        else:
            int_label = [self.concept_tree.node_label[l] for l in lab_tups]
            if self.adjust_freq:
                onehot = self.freq_adjusted_onehot_array
            else:
                onehot = self.float_onehot_label_array
            vecs = onehot[int_label]
            if self.use_all_labels:
                vec = vecs.mean(0)
            else:
                vec = vecs[self.label_level]
            vec = torch.from_numpy(vec).float()
            return dict(image=img, label=vec)

    def __len__(self):
        return len(self.image_paths)

    def collect_statistics(self, stats_lambda, summary_lambda):
        return summary_lambda([stats_lambda(item) for item in self])


<mask token>


def pil_image_size(item):
    W, H = item['image'].size
    return dict(width=W, height=H)


<mask token>

import os
import groot
import numpy as np
import PIL
import torch
import torchvision


class GroceryDataset(torch.utils.data.Dataset):

    def __init__(self, root, split='train', string_labels=True, transforms=
        None, adjust_freq=True, eps=1e-05, use_all_labels=False, label_level=-1
        ):
        self.root = root
        self.label_file_path = os.path.join(root, split + '.txt')
        self.image_paths = self._load_label_path_file()
        self.label_tuples = [p.split('/')[1:-1] for p in self.image_paths]
        self.string_labels = string_labels
        self.concept_tree = groot.Tree.from_yaml(open(os.path.join(root,
            'concept_tree.yaml'), 'r').read())
        self.float_onehot_label_array = np.load(os.path.join(root,
            'label_arrays.npz'))['onehot']
        self.transforms = transforms
        self.adjust_freq = adjust_freq
        self.use_all_labels = use_all_labels
        self.label_level = label_level
        if adjust_freq:
            self.class_counts = self._count_labels()
            levels = self.concept_tree.levels
            self.counts_by_level = [{cat: self.class_counts[cat] for cat in
                level} for level in levels]
            name_to_id = self.concept_tree.node_label
            freq = np.zeros_like(self.float_onehot_label_array)
            level_total_counts = [sum(v for k, v in level.items()) for
                level in self.counts_by_level]
            self.freq_by_level = [{cat: (counts[cat] / tot) for cat in
                counts} for tot, counts in zip(level_total_counts, self.
                counts_by_level)]
            for d in self.freq_by_level:
                for k, v in d.items():
                    freq[name_to_id[k]] += max(v, eps)
            freq = np.clip(a=freq, a_min=eps, a_max=1 / eps)
            self.frequency_array = freq
            self.freq_adjusted_onehot_array = (self.
                float_onehot_label_array / freq)

    def _count_labels(self):
        labels = self.label_tuples
        counts = dict(Grocery=len(labels))
        for label in labels:
            for lab in label:
                if lab in counts:
                    counts[lab] += 1
                else:
                    counts[lab] = 1
        return counts

    def _load_label_path_file(self):
        with open(self.label_file_path, 'r') as f:
            paths = [ln.split(',')[0] for ln in f.readlines()]
        return paths

    def __getitem__(self, idx):
        img_path = os.path.join(self.root, self.image_paths[idx])
        lab_tups = self.label_tuples[idx]
        img = PIL.Image.open(img_path)
        if self.transforms:
            img = self.transforms(img)
        if self.string_labels:
            return dict(image=img, string_labels=lab_tups)
        else:
            int_label = [self.concept_tree.node_label[l] for l in lab_tups]
            if self.adjust_freq:
                onehot = self.freq_adjusted_onehot_array
            else:
                onehot = self.float_onehot_label_array
            vecs = onehot[int_label]
            if self.use_all_labels:
                vec = vecs.mean(0)
            else:
                vec = vecs[self.label_level]
            vec = torch.from_numpy(vec).float()
            return dict(image=img, label=vec)

    def __len__(self):
        return len(self.image_paths)

    def collect_statistics(self, stats_lambda, summary_lambda):
        return summary_lambda([stats_lambda(item) for item in self])


train_augmentation = torchvision.transforms.Compose([torchvision.transforms
    .AutoAugment(), torchvision.transforms.RandomApply([torchvision.
    transforms.RandomCrop(300, pad_if_needed=True), torchvision.transforms.
    RandomCrop(256, pad_if_needed=True)]), torchvision.transforms.Resize((
    256, 256)), torchvision.transforms.ToTensor()])
eval_augmentation = torchvision.transforms.Compose([torchvision.transforms.
    Resize(256), torchvision.transforms.ToTensor()])


def pil_image_size(item):
    W, H = item['image'].size
    return dict(width=W, height=H)


def get_size_stats(sizes):
    h_ = []
    w_ = []
    for d in sizes:
        h_.append(d['height'])
        w_.append(d['width'])
    return {'max height': max(h_), 'min height': min(h_), 'max width': max(
        w_), 'min width': min(w_), 'mean height': sum(h_) / len(h_),
        'mean width': sum(w_) / len(w_)}

import os

import groot
import numpy as np
import PIL
import torch
import torchvision


class GroceryDataset(torch.utils.data.Dataset):
    def __init__(
        self,
        root,
        split="train",
        string_labels=True,
        transforms=None,
        adjust_freq=True,
        eps=1e-5,
        use_all_labels=False,
        label_level=-1,
    ):

        self.root = root
        self.label_file_path = os.path.join(root, split + ".txt")
        self.image_paths = self._load_label_path_file()
        self.label_tuples = [p.split("/")[1:-1] for p in self.image_paths]
        self.string_labels = string_labels
        self.concept_tree = groot.Tree.from_yaml(
            open(os.path.join(root, "concept_tree.yaml"), "r").read()
        )
        self.float_onehot_label_array = np.load(os.path.join(root, "label_arrays.npz"))[
            "onehot"
        ]
        self.transforms = transforms
        self.adjust_freq = adjust_freq
        self.use_all_labels = use_all_labels
        self.label_level = label_level

        if adjust_freq:
            self.class_counts = self._count_labels()
            levels = self.concept_tree.levels
            self.counts_by_level = [
                {cat: self.class_counts[cat] for cat in level} for level in levels
            ]
            name_to_id = self.concept_tree.node_label
            freq = np.zeros_like(self.float_onehot_label_array)
            level_total_counts = [
                sum(v for k, v in level.items()) for level in self.counts_by_level
            ]
            self.freq_by_level = [
                {cat: counts[cat] / tot for cat in counts}
                for tot, counts in zip(level_total_counts, self.counts_by_level)
            ]
            for d in self.freq_by_level:
                for k, v in d.items():
                    freq[name_to_id[k]] += max(v, eps)
            freq = np.clip(a=freq, a_min=eps, a_max=1 / eps)

            self.frequency_array = freq
            self.freq_adjusted_onehot_array = self.float_onehot_label_array / freq

    def _count_labels(self):

        labels = self.label_tuples
        counts = dict(Grocery=len(labels))
        for label in labels:
            for lab in label:
                if lab in counts:
                    counts[lab] += 1
                else:
                    counts[lab] = 1
        return counts

    def _load_label_path_file(self):

        with open(self.label_file_path, "r") as f:
            paths = [ln.split(",")[0] for ln in f.readlines()]

        return paths

    def __getitem__(self, idx):

        img_path = os.path.join(self.root, self.image_paths[idx])
        lab_tups = self.label_tuples[idx]

        img = PIL.Image.open(img_path)

        if self.transforms:
            img = self.transforms(img)

        if self.string_labels:
            return dict(image=img, string_labels=lab_tups)
        else:
            int_label = [self.concept_tree.node_label[l] for l in lab_tups]
            if self.adjust_freq:
                onehot = self.freq_adjusted_onehot_array
            else:
                onehot = self.float_onehot_label_array

            vecs = onehot[int_label]  # [n_label, onehot_dim]
            if self.use_all_labels:
                vec = vecs.mean(0)
            else:
                vec = vecs[self.label_level]
            vec = torch.from_numpy(vec).float()
            return dict(image=img, label=vec)

    def __len__(self):
        return len(self.image_paths)

    def collect_statistics(self, stats_lambda, summary_lambda):
        return summary_lambda([stats_lambda(item) for item in self])


train_augmentation = torchvision.transforms.Compose(
    [
        torchvision.transforms.AutoAugment(),
        torchvision.transforms.RandomApply(
            [
                torchvision.transforms.RandomCrop(300, pad_if_needed=True),
                torchvision.transforms.RandomCrop(256, pad_if_needed=True),
            ]
        ),
        torchvision.transforms.Resize((256, 256)),
        torchvision.transforms.ToTensor(),
    ]
)


eval_augmentation = torchvision.transforms.Compose(
    [torchvision.transforms.Resize(256), torchvision.transforms.ToTensor()]
)


def pil_image_size(item):
    W, H = item["image"].size
    return dict(width=W, height=H)


def get_size_stats(sizes):
    h_ = []
    w_ = []
    for d in sizes:
        h_.append(d["height"])
        w_.append(d["width"])

    return {
        "max height": max(h_),
        "min height": min(h_),
        "max width": max(w_),
        "min width": min(w_),
        "mean height": sum(h_) / len(h_),
        "mean width": sum(w_) / len(w_),
    }
