# Find Spark
import findspark
findspark.init()

from pyspark.sql.types import *

from pyspark import SparkContext
from pyspark.sql import SparkSession

spark = SparkSession \
    .builder \
    .appName("Movie lens model") \
    .getOrCreate()

# Define schema for ratings dataset
ratingsStruct = [StructField("userId", IntegerType(), True),
    StructField("movieId", IntegerType(), True),
    StructField("rating", DoubleType(), True),
    StructField("timestamp", IntegerType(), True)]

ratingsSchema = StructType(ratingsStruct)

# Read ratings from HDFS
ratingsDf = spark.read.format("csv") \
    .option("header", "false") \
    .option("delimiter", "::") \
    .schema(ratingsSchema) \
    .load("file:///home/mcheghal/Documents/sylace/bcg/mllib/ml-1m/ratings.dat")


# Splitting training data
serveDf = ratingsDf.randomSplit([0.1], seed=12345)

# Store data to serve as pandas split json
serveDf[0].toPandas().head(100).to_json('~/data.json', orient='split')
10afffe054e383cd22ac151e219b105567aa30f0
<mask token>

<mask token>
findspark.init()
<mask token>
serveDf[0].toPandas().head(100).to_json('~/data.json', orient='split')

<mask token>
findspark.init()
<mask token>
spark = SparkSession.builder.appName('Movie lens model').getOrCreate()
ratingsStruct = [StructField('userId', IntegerType(), True), StructField(
    'movieId', IntegerType(), True), StructField('rating', DoubleType(), 
    True), StructField('timestamp', IntegerType(), True)]
ratingsSchema = StructType(ratingsStruct)
ratingsDf = spark.read.format('csv').option('header', 'false').option(
    'delimiter', '::').schema(ratingsSchema).load(
    'file:///home/mcheghal/Documents/sylace/bcg/mllib/ml-1m/ratings.dat')
serveDf = ratingsDf.randomSplit([0.1], seed=12345)
serveDf[0].toPandas().head(100).to_json('~/data.json', orient='split')

import findspark
findspark.init()
from pyspark.sql.types import *
from pyspark import SparkContext
from pyspark.sql import SparkSession
spark = SparkSession.builder.appName('Movie lens model').getOrCreate()
ratingsStruct = [StructField('userId', IntegerType(), True), StructField(
    'movieId', IntegerType(), True), StructField('rating', DoubleType(), 
    True), StructField('timestamp', IntegerType(), True)]
ratingsSchema = StructType(ratingsStruct)
ratingsDf = spark.read.format('csv').option('header', 'false').option(
    'delimiter', '::').schema(ratingsSchema).load(
    'file:///home/mcheghal/Documents/sylace/bcg/mllib/ml-1m/ratings.dat')
serveDf = ratingsDf.randomSplit([0.1], seed=12345)
serveDf[0].toPandas().head(100).to_json('~/data.json', orient='split')

# Find Spark
import findspark
findspark.init()

from pyspark.sql.types import *

from pyspark import SparkContext
from pyspark.sql import SparkSession

spark = SparkSession \
    .builder \
    .appName("Movie lens model") \
    .getOrCreate()

# Define schema for ratings dataset
ratingsStruct = [StructField("userId", IntegerType(), True),
    StructField("movieId", IntegerType(), True),
    StructField("rating", DoubleType(), True),
    StructField("timestamp", IntegerType(), True)]

ratingsSchema = StructType(ratingsStruct)

# Read ratings from HDFS
ratingsDf = spark.read.format("csv") \
    .option("header", "false") \
    .option("delimiter", "::") \
    .schema(ratingsSchema) \
    .load("file:///home/mcheghal/Documents/sylace/bcg/mllib/ml-1m/ratings.dat")


# Splitting training data
serveDf = ratingsDf.randomSplit([0.1], seed=12345)

# Store data to serve as pandas split json
serveDf[0].toPandas().head(100).to_json('~/data.json', orient='split')