"""
Preprocess a raw json dataset into hdf5/json files for use in data_loader.lua
Input: json file that has the form
[{ file_path: 'path/img.jpg', captions: ['a caption', ...] }, ...]
example element in this list would look like
{'captions': [u'A man with a red helmet on a small moped on a dirt road. ', u'Man riding a motor bike on a dirt road on the countryside.', u'A man riding on the back of a motorcycle.', u'A dirt path with a young person on a motor bike rests to the foreground of a verdant area with a bridge and a background of cloud-wreathed mountains. ', u'A man in a red shirt and a red hat is on a motorcycle on a hill side.'], 'file_path': u'val2014/COCO_val2014_000000391895.jpg', 'id': 391895}
This script reads this json, does some basic preprocessing on the captions
(e.g. lowercase, etc.), creates a special UNK token, and encodes everything to arrays
Output: a json file and an hdf5 file
The hdf5 file contains several fields:
/images is (N,3,256,256) uint8 array of raw image data in RGB format
/labels is (M,max_length) uint32 array of encoded labels, zero padded
/label_start_ix and /label_end_ix are (N,) uint32 arrays of pointers to the
  first and last indices (in range 1..M) of labels for each image
/label_length stores the length of the sequence for each of the M sequences
The json file has a dict that contains:
- an 'ix_to_word' field storing the vocab in form {ix:'word'}, where ix is 1-indexed
- an 'images' field that is a list holding auxiliary information for each image,
  such as in particular the 'split' it was assigned to.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import json
import argparse
from random import shuffle, seed
import string
# non-standard dependencies:
import h5py
import numpy as np
from scipy.misc import imread, imresize


def main(params):

    imgs = json.load(open(params['input_json'], 'r'))
    imgs = imgs['images']

    seed(123)  # make reproducible

    # create output h5 file
    N = len(imgs)
    f = h5py.File(params['output_h5'] + '_image.h5', "w")
    dset = f.create_dataset("images", (N, 3, 256, 256),
                            dtype='uint8')  # space for resized images
    for i, img in enumerate(imgs):
        # load the image
        I = imread(os.path.join(
            params['images_root'], img['filename']))
        try:
            Ir = imresize(I, (256, 256))
        except:
            print('failed resizing image %s - see http://git.io/vBIE0' %
                  (img['file_path'],))
            raise
        # handle grayscale input images
        if len(Ir.shape) == 2:
            Ir = Ir[:, :, np.newaxis]
            Ir = np.concatenate((Ir, Ir, Ir), axis=2)
        # and swap order of axes from (256,256,3) to (3,256,256)
        Ir = Ir.transpose(2, 0, 1)
        # write to h5
        dset[i] = Ir
        if i % 1000 == 0:
            print('processing %d/%d (%.2f%% done)' % (i, N, i * 100.0 / N))
    f.close()
    print('wrote ', params['output_h5'] + '_image.h5')


if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    # input json
    parser.add_argument('--input_json', default='data/dataset_rsicd.json',
                        help='input json file to process into hdf5')
    parser.add_argument('--output_h5', default='data/processed',
                        help='output h5 file')

    # options
    parser.add_argument('--max_length', default=30, type=int,
                        help='max length of a caption, in number of words. captions longer than this get clipped.')
    parser.add_argument('--images_root', default='data/RSICD_images',
                        help='root location in which images are stored, to be prepended to file_path in input json')

    args = parser.parse_args()
    params = vars(args)  # convert to ordinary dict
    print('parsed input parameters:')
    print(json.dumps(params, indent=2))
    main(params)

115601c8818b07da7b2a74ea5d8af7394096f3cd
<mask token>

<mask token>


def main(params):
    imgs = json.load(open(params['input_json'], 'r'))
    imgs = imgs['images']
    seed(123)
    N = len(imgs)
    f = h5py.File(params['output_h5'] + '_image.h5', 'w')
    dset = f.create_dataset('images', (N, 3, 256, 256), dtype='uint8')
    for i, img in enumerate(imgs):
        I = imread(os.path.join(params['images_root'], img['filename']))
        try:
            Ir = imresize(I, (256, 256))
        except:
            print('failed resizing image %s - see http://git.io/vBIE0' % (
                img['file_path'],))
            raise
        if len(Ir.shape) == 2:
            Ir = Ir[:, :, np.newaxis]
            Ir = np.concatenate((Ir, Ir, Ir), axis=2)
        Ir = Ir.transpose(2, 0, 1)
        dset[i] = Ir
        if i % 1000 == 0:
            print('processing %d/%d (%.2f%% done)' % (i, N, i * 100.0 / N))
    f.close()
    print('wrote ', params['output_h5'] + '_image.h5')


<mask token>

<mask token>


def main(params):
    imgs = json.load(open(params['input_json'], 'r'))
    imgs = imgs['images']
    seed(123)
    N = len(imgs)
    f = h5py.File(params['output_h5'] + '_image.h5', 'w')
    dset = f.create_dataset('images', (N, 3, 256, 256), dtype='uint8')
    for i, img in enumerate(imgs):
        I = imread(os.path.join(params['images_root'], img['filename']))
        try:
            Ir = imresize(I, (256, 256))
        except:
            print('failed resizing image %s - see http://git.io/vBIE0' % (
                img['file_path'],))
            raise
        if len(Ir.shape) == 2:
            Ir = Ir[:, :, np.newaxis]
            Ir = np.concatenate((Ir, Ir, Ir), axis=2)
        Ir = Ir.transpose(2, 0, 1)
        dset[i] = Ir
        if i % 1000 == 0:
            print('processing %d/%d (%.2f%% done)' % (i, N, i * 100.0 / N))
    f.close()
    print('wrote ', params['output_h5'] + '_image.h5')


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--input_json', default='data/dataset_rsicd.json',
        help='input json file to process into hdf5')
    parser.add_argument('--output_h5', default='data/processed', help=
        'output h5 file')
    parser.add_argument('--max_length', default=30, type=int, help=
        'max length of a caption, in number of words. captions longer than this get clipped.'
        )
    parser.add_argument('--images_root', default='data/RSICD_images', help=
        'root location in which images are stored, to be prepended to file_path in input json'
        )
    args = parser.parse_args()
    params = vars(args)
    print('parsed input parameters:')
    print(json.dumps(params, indent=2))
    main(params)

<mask token>
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import os
import json
import argparse
from random import shuffle, seed
import string
import h5py
import numpy as np
from scipy.misc import imread, imresize


def main(params):
    imgs = json.load(open(params['input_json'], 'r'))
    imgs = imgs['images']
    seed(123)
    N = len(imgs)
    f = h5py.File(params['output_h5'] + '_image.h5', 'w')
    dset = f.create_dataset('images', (N, 3, 256, 256), dtype='uint8')
    for i, img in enumerate(imgs):
        I = imread(os.path.join(params['images_root'], img['filename']))
        try:
            Ir = imresize(I, (256, 256))
        except:
            print('failed resizing image %s - see http://git.io/vBIE0' % (
                img['file_path'],))
            raise
        if len(Ir.shape) == 2:
            Ir = Ir[:, :, np.newaxis]
            Ir = np.concatenate((Ir, Ir, Ir), axis=2)
        Ir = Ir.transpose(2, 0, 1)
        dset[i] = Ir
        if i % 1000 == 0:
            print('processing %d/%d (%.2f%% done)' % (i, N, i * 100.0 / N))
    f.close()
    print('wrote ', params['output_h5'] + '_image.h5')


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--input_json', default='data/dataset_rsicd.json',
        help='input json file to process into hdf5')
    parser.add_argument('--output_h5', default='data/processed', help=
        'output h5 file')
    parser.add_argument('--max_length', default=30, type=int, help=
        'max length of a caption, in number of words. captions longer than this get clipped.'
        )
    parser.add_argument('--images_root', default='data/RSICD_images', help=
        'root location in which images are stored, to be prepended to file_path in input json'
        )
    args = parser.parse_args()
    params = vars(args)
    print('parsed input parameters:')
    print(json.dumps(params, indent=2))
    main(params)

"""
Preprocess a raw json dataset into hdf5/json files for use in data_loader.lua
Input: json file that has the form
[{ file_path: 'path/img.jpg', captions: ['a caption', ...] }, ...]
example element in this list would look like
{'captions': [u'A man with a red helmet on a small moped on a dirt road. ', u'Man riding a motor bike on a dirt road on the countryside.', u'A man riding on the back of a motorcycle.', u'A dirt path with a young person on a motor bike rests to the foreground of a verdant area with a bridge and a background of cloud-wreathed mountains. ', u'A man in a red shirt and a red hat is on a motorcycle on a hill side.'], 'file_path': u'val2014/COCO_val2014_000000391895.jpg', 'id': 391895}
This script reads this json, does some basic preprocessing on the captions
(e.g. lowercase, etc.), creates a special UNK token, and encodes everything to arrays
Output: a json file and an hdf5 file
The hdf5 file contains several fields:
/images is (N,3,256,256) uint8 array of raw image data in RGB format
/labels is (M,max_length) uint32 array of encoded labels, zero padded
/label_start_ix and /label_end_ix are (N,) uint32 arrays of pointers to the
  first and last indices (in range 1..M) of labels for each image
/label_length stores the length of the sequence for each of the M sequences
The json file has a dict that contains:
- an 'ix_to_word' field storing the vocab in form {ix:'word'}, where ix is 1-indexed
- an 'images' field that is a list holding auxiliary information for each image,
  such as in particular the 'split' it was assigned to.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import json
import argparse
from random import shuffle, seed
import string
# non-standard dependencies:
import h5py
import numpy as np
from scipy.misc import imread, imresize


def main(params):

    imgs = json.load(open(params['input_json'], 'r'))
    imgs = imgs['images']

    seed(123)  # make reproducible

    # create output h5 file
    N = len(imgs)
    f = h5py.File(params['output_h5'] + '_image.h5', "w")
    dset = f.create_dataset("images", (N, 3, 256, 256),
                            dtype='uint8')  # space for resized images
    for i, img in enumerate(imgs):
        # load the image
        I = imread(os.path.join(
            params['images_root'], img['filename']))
        try:
            Ir = imresize(I, (256, 256))
        except:
            print('failed resizing image %s - see http://git.io/vBIE0' %
                  (img['file_path'],))
            raise
        # handle grayscale input images
        if len(Ir.shape) == 2:
            Ir = Ir[:, :, np.newaxis]
            Ir = np.concatenate((Ir, Ir, Ir), axis=2)
        # and swap order of axes from (256,256,3) to (3,256,256)
        Ir = Ir.transpose(2, 0, 1)
        # write to h5
        dset[i] = Ir
        if i % 1000 == 0:
            print('processing %d/%d (%.2f%% done)' % (i, N, i * 100.0 / N))
    f.close()
    print('wrote ', params['output_h5'] + '_image.h5')


if __name__ == "__main__":

    parser = argparse.ArgumentParser()

    # input json
    parser.add_argument('--input_json', default='data/dataset_rsicd.json',
                        help='input json file to process into hdf5')
    parser.add_argument('--output_h5', default='data/processed',
                        help='output h5 file')

    # options
    parser.add_argument('--max_length', default=30, type=int,
                        help='max length of a caption, in number of words. captions longer than this get clipped.')
    parser.add_argument('--images_root', default='data/RSICD_images',
                        help='root location in which images are stored, to be prepended to file_path in input json')

    args = parser.parse_args()
    params = vars(args)  # convert to ordinary dict
    print('parsed input parameters:')
    print(json.dumps(params, indent=2))
    main(params)
