#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Aug 10 12:38:55 2021

@author: keerag
"""

import findspark
findspark.find()
findspark.init()
from pyspark.sql import SparkSession
from pyspark import SparkContext, SparkConf

#create spark session with mysql configure
spark=SparkSession.builder.config("spark.jars", "/opt/spark/jars/mysql-connector-java-8.0.25.jar") \
    .appName("Read AWS S3 & write MYSQL")\
        .master("local[*]")\
            .getOrCreate()

#hadoop configuration for acessing aws
spark._jsc.hadoopConfiguration().set("fs.s3a.access.key","********************")
spark._jsc.hadoopConfiguration().set("fs.s3a.secret.key","*******************************")
spark._jsc.hadoopConfiguration().set("fs.s3a.endpoint","s3.amazonaws.com")

#read Dataframe from aws s3
myDf=spark.read\
    .format("csv")\
        .option("header","true")\
            .option("inferSchema","true")\
                .csv("s3a://keer1993/data.csv")

#load Dataframe to mysql table
myDf.write.format("jdbc").option("url", "jdbc:mysql://localhost:3306/ragdb") \
    .option("driver", "com.mysql.jdbc.Driver").option("dbtable", "s3tb") \
    .option("user", "keerag").option("password", "1Keerag@").mode('append').save()

8204c8f571dd0cb6e3db109c37fb14ace94e850f
<mask token>

<mask token>
findspark.find()
findspark.init()
<mask token>
spark._jsc.hadoopConfiguration().set('fs.s3a.access.key',
    '********************')
spark._jsc.hadoopConfiguration().set('fs.s3a.secret.key',
    '*******************************')
spark._jsc.hadoopConfiguration().set('fs.s3a.endpoint', 's3.amazonaws.com')
<mask token>
myDf.write.format('jdbc').option('url', 'jdbc:mysql://localhost:3306/ragdb'
    ).option('driver', 'com.mysql.jdbc.Driver').option('dbtable', 's3tb'
    ).option('user', 'keerag').option('password', '1Keerag@').mode('append'
    ).save()

<mask token>
findspark.find()
findspark.init()
<mask token>
spark = SparkSession.builder.config('spark.jars',
    '/opt/spark/jars/mysql-connector-java-8.0.25.jar').appName(
    'Read AWS S3 & write MYSQL').master('local[*]').getOrCreate()
spark._jsc.hadoopConfiguration().set('fs.s3a.access.key',
    '********************')
spark._jsc.hadoopConfiguration().set('fs.s3a.secret.key',
    '*******************************')
spark._jsc.hadoopConfiguration().set('fs.s3a.endpoint', 's3.amazonaws.com')
myDf = spark.read.format('csv').option('header', 'true').option('inferSchema',
    'true').csv('s3a://keer1993/data.csv')
myDf.write.format('jdbc').option('url', 'jdbc:mysql://localhost:3306/ragdb'
    ).option('driver', 'com.mysql.jdbc.Driver').option('dbtable', 's3tb'
    ).option('user', 'keerag').option('password', '1Keerag@').mode('append'
    ).save()

<mask token>
import findspark
findspark.find()
findspark.init()
from pyspark.sql import SparkSession
from pyspark import SparkContext, SparkConf
spark = SparkSession.builder.config('spark.jars',
    '/opt/spark/jars/mysql-connector-java-8.0.25.jar').appName(
    'Read AWS S3 & write MYSQL').master('local[*]').getOrCreate()
spark._jsc.hadoopConfiguration().set('fs.s3a.access.key',
    '********************')
spark._jsc.hadoopConfiguration().set('fs.s3a.secret.key',
    '*******************************')
spark._jsc.hadoopConfiguration().set('fs.s3a.endpoint', 's3.amazonaws.com')
myDf = spark.read.format('csv').option('header', 'true').option('inferSchema',
    'true').csv('s3a://keer1993/data.csv')
myDf.write.format('jdbc').option('url', 'jdbc:mysql://localhost:3306/ragdb'
    ).option('driver', 'com.mysql.jdbc.Driver').option('dbtable', 's3tb'
    ).option('user', 'keerag').option('password', '1Keerag@').mode('append'
    ).save()

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Aug 10 12:38:55 2021

@author: keerag
"""

import findspark
findspark.find()
findspark.init()
from pyspark.sql import SparkSession
from pyspark import SparkContext, SparkConf

#create spark session with mysql configure
spark=SparkSession.builder.config("spark.jars", "/opt/spark/jars/mysql-connector-java-8.0.25.jar") \
    .appName("Read AWS S3 & write MYSQL")\
        .master("local[*]")\
            .getOrCreate()

#hadoop configuration for acessing aws
spark._jsc.hadoopConfiguration().set("fs.s3a.access.key","********************")
spark._jsc.hadoopConfiguration().set("fs.s3a.secret.key","*******************************")
spark._jsc.hadoopConfiguration().set("fs.s3a.endpoint","s3.amazonaws.com")

#read Dataframe from aws s3
myDf=spark.read\
    .format("csv")\
        .option("header","true")\
            .option("inferSchema","true")\
                .csv("s3a://keer1993/data.csv")

#load Dataframe to mysql table
myDf.write.format("jdbc").option("url", "jdbc:mysql://localhost:3306/ragdb") \
    .option("driver", "com.mysql.jdbc.Driver").option("dbtable", "s3tb") \
    .option("user", "keerag").option("password", "1Keerag@").mode('append').save()
