import os
from os import listdir
import os.path
import numpy as np
import random

img_dir= "/home/ec2-user/SageMaker/efs/amazon-bin/data/bin-images/"
meta_dir = "/home/ec2-user/SageMaker/efs/amazon-bin/data/metadata/"

#img_list = listdir(img_dir)
#N = len(img_list)
N = 535234
list_random = range(N)
random.shuffle(list_random)

# finding images that metadata exists
meta_avail = np.zeros(N, dtype=bool)
for i in range(N):
    meta_fname = os.path.join(meta_dir,('%05d.json'%(i+1)))
    if os.path.isfile(meta_fname):
        meta_avail[i] = True

# assign validataion set
valset = np.zeros(N, dtype=bool)
n_valset = int(round(N*0.1))
count = 0
random.shuffle(list_random)
for i in range(N):
    idx = list_random[i]
    if meta_avail[idx]:
        valset[idx]=True
        count = count + 1
        if count == n_valset:
            break

# writing out to textfile
train_f = open('/home/ec2-user/SageMaker/efs/amazon-bin/input/random_train.txt','w')
val_f = open('/home/ec2-user/SageMaker/efs/amazon-bin/input/random_val.txt','w')
for i in range(N):
    if meta_avail[i]:
        if valset[i]:
            val_f.write("%d\n" % (i+1))
        else:
            train_f.write("%d\n" % (i+1))
train_f.close()
val_f.close()



223da4b81ac180e63f1ce08f85c4156a6475fa36
<mask token>

<mask token>
random.shuffle(list_random)
<mask token>
for i in range(N):
    meta_fname = os.path.join(meta_dir, '%05d.json' % (i + 1))
    if os.path.isfile(meta_fname):
        meta_avail[i] = True
<mask token>
random.shuffle(list_random)
for i in range(N):
    idx = list_random[i]
    if meta_avail[idx]:
        valset[idx] = True
        count = count + 1
        if count == n_valset:
            break
<mask token>
for i in range(N):
    if meta_avail[i]:
        if valset[i]:
            val_f.write('%d\n' % (i + 1))
        else:
            train_f.write('%d\n' % (i + 1))
train_f.close()
val_f.close()

<mask token>
img_dir = '/home/ec2-user/SageMaker/efs/amazon-bin/data/bin-images/'
meta_dir = '/home/ec2-user/SageMaker/efs/amazon-bin/data/metadata/'
N = 535234
list_random = range(N)
random.shuffle(list_random)
meta_avail = np.zeros(N, dtype=bool)
for i in range(N):
    meta_fname = os.path.join(meta_dir, '%05d.json' % (i + 1))
    if os.path.isfile(meta_fname):
        meta_avail[i] = True
valset = np.zeros(N, dtype=bool)
n_valset = int(round(N * 0.1))
count = 0
random.shuffle(list_random)
for i in range(N):
    idx = list_random[i]
    if meta_avail[idx]:
        valset[idx] = True
        count = count + 1
        if count == n_valset:
            break
train_f = open('/home/ec2-user/SageMaker/efs/amazon-bin/input/random_train.txt'
    , 'w')
val_f = open('/home/ec2-user/SageMaker/efs/amazon-bin/input/random_val.txt',
    'w')
for i in range(N):
    if meta_avail[i]:
        if valset[i]:
            val_f.write('%d\n' % (i + 1))
        else:
            train_f.write('%d\n' % (i + 1))
train_f.close()
val_f.close()

import os
from os import listdir
import os.path
import numpy as np
import random
img_dir = '/home/ec2-user/SageMaker/efs/amazon-bin/data/bin-images/'
meta_dir = '/home/ec2-user/SageMaker/efs/amazon-bin/data/metadata/'
N = 535234
list_random = range(N)
random.shuffle(list_random)
meta_avail = np.zeros(N, dtype=bool)
for i in range(N):
    meta_fname = os.path.join(meta_dir, '%05d.json' % (i + 1))
    if os.path.isfile(meta_fname):
        meta_avail[i] = True
valset = np.zeros(N, dtype=bool)
n_valset = int(round(N * 0.1))
count = 0
random.shuffle(list_random)
for i in range(N):
    idx = list_random[i]
    if meta_avail[idx]:
        valset[idx] = True
        count = count + 1
        if count == n_valset:
            break
train_f = open('/home/ec2-user/SageMaker/efs/amazon-bin/input/random_train.txt'
    , 'w')
val_f = open('/home/ec2-user/SageMaker/efs/amazon-bin/input/random_val.txt',
    'w')
for i in range(N):
    if meta_avail[i]:
        if valset[i]:
            val_f.write('%d\n' % (i + 1))
        else:
            train_f.write('%d\n' % (i + 1))
train_f.close()
val_f.close()

import os
from os import listdir
import os.path
import numpy as np
import random

img_dir= "/home/ec2-user/SageMaker/efs/amazon-bin/data/bin-images/"
meta_dir = "/home/ec2-user/SageMaker/efs/amazon-bin/data/metadata/"

#img_list = listdir(img_dir)
#N = len(img_list)
N = 535234
list_random = range(N)
random.shuffle(list_random)

# finding images that metadata exists
meta_avail = np.zeros(N, dtype=bool)
for i in range(N):
    meta_fname = os.path.join(meta_dir,('%05d.json'%(i+1)))
    if os.path.isfile(meta_fname):
        meta_avail[i] = True

# assign validataion set
valset = np.zeros(N, dtype=bool)
n_valset = int(round(N*0.1))
count = 0
random.shuffle(list_random)
for i in range(N):
    idx = list_random[i]
    if meta_avail[idx]:
        valset[idx]=True
        count = count + 1
        if count == n_valset:
            break

# writing out to textfile
train_f = open('/home/ec2-user/SageMaker/efs/amazon-bin/input/random_train.txt','w')
val_f = open('/home/ec2-user/SageMaker/efs/amazon-bin/input/random_val.txt','w')
for i in range(N):
    if meta_avail[i]:
        if valset[i]:
            val_f.write("%d\n" % (i+1))
        else:
            train_f.write("%d\n" % (i+1))
train_f.close()
val_f.close()


