import os
import subprocess
import requests
import json
import sys
import pandas as pd

import ssl
if (not os.environ.get('PYTHONHTTPSVERIFY', '') and getattr(ssl, '_create_unverified_context', None)):
    ssl._create_default_https_context = ssl._create_unverified_context





#from tcga downloader import *
#ids=process_manifests(manifest_file)

'''
Match tumor and normal samples to a patient through GDC API. This will provide metadata that
for associated files including sample type(tumor /normal) patient ID(TCGA barcode), etc

The general query structure will look like this:

curl "https://gdc-api.nci.nih.gov/files?size=100000&format=tsv&filters=XXXX&fields=YYYY"

filters= :

The XXXX will be replaced with a URL-encoded JSON query that will filter a list of files. The JSON query can be generated yourself and then URL encoded, or the filtering can be performed on the GDC Data Portal (https://gdc-portal.nci.nih.gov/search/s?facetTab=files). You can copy and paste the string that appears where the * are displayed in the following:

https://gdc-portal.nci.nih.gov/search/f?filters=**********&facetTab=cases

'''

def prepare_payload(ids,data_type=None):

    #"Gene Expression Quantification"
    
    "Workflow Type" "HTSeq-Counts"

    "Data Category" "transcriptome profilling"
    "Experimental Strategy" "RNA-Seq"

    no_of_samples=len(ids)
    part1='''{
    "filters":{
        "op":"and",
        "content":[
            {
                "op":"in",
                "content":{
                    "field":"files.file_id",
                    "value":[%s]
                }
            },
            {
                "op":"=",
                "content":{
                    "field":"files.data_type",
                    "value":"%s"
                }
            }
        ]
    },'''%(",\n".join(ids),data_type)


    part2=''' "format":"TSV","fields":"file_id,file_name,cases.submitter_id,cases.disease_type,cases.case_id,data_category,data_type,cases.samples.tumor_descriptor,cases.samples.tissue_type,cases.samples.sample_type,cases.samples.submitter_id,cases.samples.sample_id,cases.samples.portions.analytes.aliquots.aliquot_id,cases.samples.portions.analytes.aliquots.submitter_id","size":"%d"}'''%no_of_samples

    payload_command='%s %s'%(part1,part2)

    payloadfile='payloadv3.txt'
    with open(payloadfile,'w') as output_:
        output_.write(payload_command)
    return payloadfile


def get_ids(manifest):
    try:
#manifest='all.txt'
        with open (manifest,'r') as input_:
            ids=["\"%s\""%i.strip('\n').split('\t')[0] for i in input_][1:]

        no_of_samples=len(ids)
        return ids
    except Exception as ex:
        return None
    #print(ids)


def get_metadata(payloadfile):
    metadata='Metadata.tsv'
    webaddr='\'https://api.gdc.cancer.gov/files\''
    args=['curl', '--request POST', '--header','\"Content-Type: application/json\"',
      '--data','@%s'%payloadfile,webaddr, '>', metadata]

    single=' '.join(args)
    print(single)
    os.system(single)
    return metadata


#def get_metadatada():
 #   ids=process_manifest()
  #  if ids==None:
   #     print('Error encountered\nPlease ensure that you are using the correct manifest file')
   # else:
   #     payloadfile=prepare_payload(ids)
   #     download_data(payloadfile)

def download_data(metadatafile,sep='\t',outdir='downloads'):
    df=pd.DataFrame()
    data_df=pd.read_csv(metadatafile,sep=sep)
    sampletypes=data_df['cases.0.samples.0.sample_type'].unique()
    sampletypes=sampletypes.tolist()
    curr_dir=os.getcwd()
    
    if not os.path.exists(outdir):
        os.mkdir(outdir)
    else:
        print('output directory exists\ndata may be overwritten')
        
    for sampletype in sampletypes:
        sel=data_df[data_df['cases.0.samples.0.sample_type'].str.contains(sampletype)][['file_id',
                                                                               'file_name']]

        sampledir="%s/%s"%(outdir,sampletype)
        
        if not os.path.exists(sampledir):
            os.mkdir(sampledir)
        else:
            print('sample type directory exists\ndata may be overwritten')
            
        
        os.chdir(sampledir)
        all_file_ids=sel['file_id'].values
        download_list=[]
        for file_id in all_file_ids:
            args=['curl', '--remote-name', '--remote-header-name',
                  '\'https://api.gdc.cancer.gov/data/%s\''%file_id]
            cmd=' '.join(args)
            print('downloading %s'%file_id)
            os.system(cmd)
            #download_list.append(' '.join(args))
        os.chdir(curr_dir)
    print('Download complete\nAll data has been downloaded to ------------->%s'%outdir)
        
        
    

242d34bdbe64df966d5e319fc976d4162f820ac9
<mask token>


def prepare_payload(ids, data_type=None):
    """Workflow TypeHTSeq-Counts"""
    """Data Categorytranscriptome profilling"""
    """Experimental StrategyRNA-Seq"""
    no_of_samples = len(ids)
    part1 = (
        """{
    "filters":{
        "op":"and",
        "content":[
            {
                "op":"in",
                "content":{
                    "field":"files.file_id",
                    "value":[%s]
                }
            },
            {
                "op":"=",
                "content":{
                    "field":"files.data_type",
                    "value":"%s"
                }
            }
        ]
    },"""
         % (',\n'.join(ids), data_type))
    part2 = (
        ' "format":"TSV","fields":"file_id,file_name,cases.submitter_id,cases.disease_type,cases.case_id,data_category,data_type,cases.samples.tumor_descriptor,cases.samples.tissue_type,cases.samples.sample_type,cases.samples.submitter_id,cases.samples.sample_id,cases.samples.portions.analytes.aliquots.aliquot_id,cases.samples.portions.analytes.aliquots.submitter_id","size":"%d"}'
         % no_of_samples)
    payload_command = '%s %s' % (part1, part2)
    payloadfile = 'payloadv3.txt'
    with open(payloadfile, 'w') as output_:
        output_.write(payload_command)
    return payloadfile


def get_ids(manifest):
    try:
        with open(manifest, 'r') as input_:
            ids = [('"%s"' % i.strip('\n').split('\t')[0]) for i in input_][1:]
        no_of_samples = len(ids)
        return ids
    except Exception as ex:
        return None


def get_metadata(payloadfile):
    metadata = 'Metadata.tsv'
    webaddr = "'https://api.gdc.cancer.gov/files'"
    args = ['curl', '--request POST', '--header',
        '"Content-Type: application/json"', '--data', '@%s' % payloadfile,
        webaddr, '>', metadata]
    single = ' '.join(args)
    print(single)
    os.system(single)
    return metadata


<mask token>

<mask token>


def prepare_payload(ids, data_type=None):
    """Workflow TypeHTSeq-Counts"""
    """Data Categorytranscriptome profilling"""
    """Experimental StrategyRNA-Seq"""
    no_of_samples = len(ids)
    part1 = (
        """{
    "filters":{
        "op":"and",
        "content":[
            {
                "op":"in",
                "content":{
                    "field":"files.file_id",
                    "value":[%s]
                }
            },
            {
                "op":"=",
                "content":{
                    "field":"files.data_type",
                    "value":"%s"
                }
            }
        ]
    },"""
         % (',\n'.join(ids), data_type))
    part2 = (
        ' "format":"TSV","fields":"file_id,file_name,cases.submitter_id,cases.disease_type,cases.case_id,data_category,data_type,cases.samples.tumor_descriptor,cases.samples.tissue_type,cases.samples.sample_type,cases.samples.submitter_id,cases.samples.sample_id,cases.samples.portions.analytes.aliquots.aliquot_id,cases.samples.portions.analytes.aliquots.submitter_id","size":"%d"}'
         % no_of_samples)
    payload_command = '%s %s' % (part1, part2)
    payloadfile = 'payloadv3.txt'
    with open(payloadfile, 'w') as output_:
        output_.write(payload_command)
    return payloadfile


def get_ids(manifest):
    try:
        with open(manifest, 'r') as input_:
            ids = [('"%s"' % i.strip('\n').split('\t')[0]) for i in input_][1:]
        no_of_samples = len(ids)
        return ids
    except Exception as ex:
        return None


def get_metadata(payloadfile):
    metadata = 'Metadata.tsv'
    webaddr = "'https://api.gdc.cancer.gov/files'"
    args = ['curl', '--request POST', '--header',
        '"Content-Type: application/json"', '--data', '@%s' % payloadfile,
        webaddr, '>', metadata]
    single = ' '.join(args)
    print(single)
    os.system(single)
    return metadata


def download_data(metadatafile, sep='\t', outdir='downloads'):
    df = pd.DataFrame()
    data_df = pd.read_csv(metadatafile, sep=sep)
    sampletypes = data_df['cases.0.samples.0.sample_type'].unique()
    sampletypes = sampletypes.tolist()
    curr_dir = os.getcwd()
    if not os.path.exists(outdir):
        os.mkdir(outdir)
    else:
        print('output directory exists\ndata may be overwritten')
    for sampletype in sampletypes:
        sel = data_df[data_df['cases.0.samples.0.sample_type'].str.contains
            (sampletype)][['file_id', 'file_name']]
        sampledir = '%s/%s' % (outdir, sampletype)
        if not os.path.exists(sampledir):
            os.mkdir(sampledir)
        else:
            print('sample type directory exists\ndata may be overwritten')
        os.chdir(sampledir)
        all_file_ids = sel['file_id'].values
        download_list = []
        for file_id in all_file_ids:
            args = ['curl', '--remote-name', '--remote-header-name', 
                "'https://api.gdc.cancer.gov/data/%s'" % file_id]
            cmd = ' '.join(args)
            print('downloading %s' % file_id)
            os.system(cmd)
        os.chdir(curr_dir)
    print(
        'Download complete\nAll data has been downloaded to ------------->%s' %
        outdir)

<mask token>
if not os.environ.get('PYTHONHTTPSVERIFY', '') and getattr(ssl,
    '_create_unverified_context', None):
    ssl._create_default_https_context = ssl._create_unverified_context
<mask token>


def prepare_payload(ids, data_type=None):
    """Workflow TypeHTSeq-Counts"""
    """Data Categorytranscriptome profilling"""
    """Experimental StrategyRNA-Seq"""
    no_of_samples = len(ids)
    part1 = (
        """{
    "filters":{
        "op":"and",
        "content":[
            {
                "op":"in",
                "content":{
                    "field":"files.file_id",
                    "value":[%s]
                }
            },
            {
                "op":"=",
                "content":{
                    "field":"files.data_type",
                    "value":"%s"
                }
            }
        ]
    },"""
         % (',\n'.join(ids), data_type))
    part2 = (
        ' "format":"TSV","fields":"file_id,file_name,cases.submitter_id,cases.disease_type,cases.case_id,data_category,data_type,cases.samples.tumor_descriptor,cases.samples.tissue_type,cases.samples.sample_type,cases.samples.submitter_id,cases.samples.sample_id,cases.samples.portions.analytes.aliquots.aliquot_id,cases.samples.portions.analytes.aliquots.submitter_id","size":"%d"}'
         % no_of_samples)
    payload_command = '%s %s' % (part1, part2)
    payloadfile = 'payloadv3.txt'
    with open(payloadfile, 'w') as output_:
        output_.write(payload_command)
    return payloadfile


def get_ids(manifest):
    try:
        with open(manifest, 'r') as input_:
            ids = [('"%s"' % i.strip('\n').split('\t')[0]) for i in input_][1:]
        no_of_samples = len(ids)
        return ids
    except Exception as ex:
        return None


def get_metadata(payloadfile):
    metadata = 'Metadata.tsv'
    webaddr = "'https://api.gdc.cancer.gov/files'"
    args = ['curl', '--request POST', '--header',
        '"Content-Type: application/json"', '--data', '@%s' % payloadfile,
        webaddr, '>', metadata]
    single = ' '.join(args)
    print(single)
    os.system(single)
    return metadata


def download_data(metadatafile, sep='\t', outdir='downloads'):
    df = pd.DataFrame()
    data_df = pd.read_csv(metadatafile, sep=sep)
    sampletypes = data_df['cases.0.samples.0.sample_type'].unique()
    sampletypes = sampletypes.tolist()
    curr_dir = os.getcwd()
    if not os.path.exists(outdir):
        os.mkdir(outdir)
    else:
        print('output directory exists\ndata may be overwritten')
    for sampletype in sampletypes:
        sel = data_df[data_df['cases.0.samples.0.sample_type'].str.contains
            (sampletype)][['file_id', 'file_name']]
        sampledir = '%s/%s' % (outdir, sampletype)
        if not os.path.exists(sampledir):
            os.mkdir(sampledir)
        else:
            print('sample type directory exists\ndata may be overwritten')
        os.chdir(sampledir)
        all_file_ids = sel['file_id'].values
        download_list = []
        for file_id in all_file_ids:
            args = ['curl', '--remote-name', '--remote-header-name', 
                "'https://api.gdc.cancer.gov/data/%s'" % file_id]
            cmd = ' '.join(args)
            print('downloading %s' % file_id)
            os.system(cmd)
        os.chdir(curr_dir)
    print(
        'Download complete\nAll data has been downloaded to ------------->%s' %
        outdir)

import os
import subprocess
import requests
import json
import sys
import pandas as pd
import ssl
if not os.environ.get('PYTHONHTTPSVERIFY', '') and getattr(ssl,
    '_create_unverified_context', None):
    ssl._create_default_https_context = ssl._create_unverified_context
<mask token>


def prepare_payload(ids, data_type=None):
    """Workflow TypeHTSeq-Counts"""
    """Data Categorytranscriptome profilling"""
    """Experimental StrategyRNA-Seq"""
    no_of_samples = len(ids)
    part1 = (
        """{
    "filters":{
        "op":"and",
        "content":[
            {
                "op":"in",
                "content":{
                    "field":"files.file_id",
                    "value":[%s]
                }
            },
            {
                "op":"=",
                "content":{
                    "field":"files.data_type",
                    "value":"%s"
                }
            }
        ]
    },"""
         % (',\n'.join(ids), data_type))
    part2 = (
        ' "format":"TSV","fields":"file_id,file_name,cases.submitter_id,cases.disease_type,cases.case_id,data_category,data_type,cases.samples.tumor_descriptor,cases.samples.tissue_type,cases.samples.sample_type,cases.samples.submitter_id,cases.samples.sample_id,cases.samples.portions.analytes.aliquots.aliquot_id,cases.samples.portions.analytes.aliquots.submitter_id","size":"%d"}'
         % no_of_samples)
    payload_command = '%s %s' % (part1, part2)
    payloadfile = 'payloadv3.txt'
    with open(payloadfile, 'w') as output_:
        output_.write(payload_command)
    return payloadfile


def get_ids(manifest):
    try:
        with open(manifest, 'r') as input_:
            ids = [('"%s"' % i.strip('\n').split('\t')[0]) for i in input_][1:]
        no_of_samples = len(ids)
        return ids
    except Exception as ex:
        return None


def get_metadata(payloadfile):
    metadata = 'Metadata.tsv'
    webaddr = "'https://api.gdc.cancer.gov/files'"
    args = ['curl', '--request POST', '--header',
        '"Content-Type: application/json"', '--data', '@%s' % payloadfile,
        webaddr, '>', metadata]
    single = ' '.join(args)
    print(single)
    os.system(single)
    return metadata


def download_data(metadatafile, sep='\t', outdir='downloads'):
    df = pd.DataFrame()
    data_df = pd.read_csv(metadatafile, sep=sep)
    sampletypes = data_df['cases.0.samples.0.sample_type'].unique()
    sampletypes = sampletypes.tolist()
    curr_dir = os.getcwd()
    if not os.path.exists(outdir):
        os.mkdir(outdir)
    else:
        print('output directory exists\ndata may be overwritten')
    for sampletype in sampletypes:
        sel = data_df[data_df['cases.0.samples.0.sample_type'].str.contains
            (sampletype)][['file_id', 'file_name']]
        sampledir = '%s/%s' % (outdir, sampletype)
        if not os.path.exists(sampledir):
            os.mkdir(sampledir)
        else:
            print('sample type directory exists\ndata may be overwritten')
        os.chdir(sampledir)
        all_file_ids = sel['file_id'].values
        download_list = []
        for file_id in all_file_ids:
            args = ['curl', '--remote-name', '--remote-header-name', 
                "'https://api.gdc.cancer.gov/data/%s'" % file_id]
            cmd = ' '.join(args)
            print('downloading %s' % file_id)
            os.system(cmd)
        os.chdir(curr_dir)
    print(
        'Download complete\nAll data has been downloaded to ------------->%s' %
        outdir)

import os
import subprocess
import requests
import json
import sys
import pandas as pd

import ssl
if (not os.environ.get('PYTHONHTTPSVERIFY', '') and getattr(ssl, '_create_unverified_context', None)):
    ssl._create_default_https_context = ssl._create_unverified_context





#from tcga downloader import *
#ids=process_manifests(manifest_file)

'''
Match tumor and normal samples to a patient through GDC API. This will provide metadata that
for associated files including sample type(tumor /normal) patient ID(TCGA barcode), etc

The general query structure will look like this:

curl "https://gdc-api.nci.nih.gov/files?size=100000&format=tsv&filters=XXXX&fields=YYYY"

filters= :

The XXXX will be replaced with a URL-encoded JSON query that will filter a list of files. The JSON query can be generated yourself and then URL encoded, or the filtering can be performed on the GDC Data Portal (https://gdc-portal.nci.nih.gov/search/s?facetTab=files). You can copy and paste the string that appears where the * are displayed in the following:

https://gdc-portal.nci.nih.gov/search/f?filters=**********&facetTab=cases

'''

def prepare_payload(ids,data_type=None):

    #"Gene Expression Quantification"
    
    "Workflow Type" "HTSeq-Counts"

    "Data Category" "transcriptome profilling"
    "Experimental Strategy" "RNA-Seq"

    no_of_samples=len(ids)
    part1='''{
    "filters":{
        "op":"and",
        "content":[
            {
                "op":"in",
                "content":{
                    "field":"files.file_id",
                    "value":[%s]
                }
            },
            {
                "op":"=",
                "content":{
                    "field":"files.data_type",
                    "value":"%s"
                }
            }
        ]
    },'''%(",\n".join(ids),data_type)


    part2=''' "format":"TSV","fields":"file_id,file_name,cases.submitter_id,cases.disease_type,cases.case_id,data_category,data_type,cases.samples.tumor_descriptor,cases.samples.tissue_type,cases.samples.sample_type,cases.samples.submitter_id,cases.samples.sample_id,cases.samples.portions.analytes.aliquots.aliquot_id,cases.samples.portions.analytes.aliquots.submitter_id","size":"%d"}'''%no_of_samples

    payload_command='%s %s'%(part1,part2)

    payloadfile='payloadv3.txt'
    with open(payloadfile,'w') as output_:
        output_.write(payload_command)
    return payloadfile


def get_ids(manifest):
    try:
#manifest='all.txt'
        with open (manifest,'r') as input_:
            ids=["\"%s\""%i.strip('\n').split('\t')[0] for i in input_][1:]

        no_of_samples=len(ids)
        return ids
    except Exception as ex:
        return None
    #print(ids)


def get_metadata(payloadfile):
    metadata='Metadata.tsv'
    webaddr='\'https://api.gdc.cancer.gov/files\''
    args=['curl', '--request POST', '--header','\"Content-Type: application/json\"',
      '--data','@%s'%payloadfile,webaddr, '>', metadata]

    single=' '.join(args)
    print(single)
    os.system(single)
    return metadata


#def get_metadatada():
 #   ids=process_manifest()
  #  if ids==None:
   #     print('Error encountered\nPlease ensure that you are using the correct manifest file')
   # else:
   #     payloadfile=prepare_payload(ids)
   #     download_data(payloadfile)

def download_data(metadatafile,sep='\t',outdir='downloads'):
    df=pd.DataFrame()
    data_df=pd.read_csv(metadatafile,sep=sep)
    sampletypes=data_df['cases.0.samples.0.sample_type'].unique()
    sampletypes=sampletypes.tolist()
    curr_dir=os.getcwd()
    
    if not os.path.exists(outdir):
        os.mkdir(outdir)
    else:
        print('output directory exists\ndata may be overwritten')
        
    for sampletype in sampletypes:
        sel=data_df[data_df['cases.0.samples.0.sample_type'].str.contains(sampletype)][['file_id',
                                                                               'file_name']]

        sampledir="%s/%s"%(outdir,sampletype)
        
        if not os.path.exists(sampledir):
            os.mkdir(sampledir)
        else:
            print('sample type directory exists\ndata may be overwritten')
            
        
        os.chdir(sampledir)
        all_file_ids=sel['file_id'].values
        download_list=[]
        for file_id in all_file_ids:
            args=['curl', '--remote-name', '--remote-header-name',
                  '\'https://api.gdc.cancer.gov/data/%s\''%file_id]
            cmd=' '.join(args)
            print('downloading %s'%file_id)
            os.system(cmd)
            #download_list.append(' '.join(args))
        os.chdir(curr_dir)
    print('Download complete\nAll data has been downloaded to ------------->%s'%outdir)
        
        
    
