Access data with Spark

file:///

rdd = sc.textFile("file:///path/to/file")

import os
from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()

# the default configuration will pick up your credentials from environment variables
# No additional configuration is necessary

# test reading
df = spark.read.json("s3a://bucket/prefix1/prefix2/people.json")
df.show()

import os
from pyspark.sql import SparkSession

spark = SparkSession.builder.getOrCreate()

# for write operations you will need the ARN of the key to use
# Note that the credentials used need to have proper access to use the key
kms_key_arn = "<your key ARN here>"

# configure the connector
# This example assumes credentials from environment variables so no need to configure
# Note: The encryption config is not needed for read only operations
hadoop_conf = spark.sparkContext._jsc.hadoopConfiguration()
hadoop_conf.set("fs.s3a.server-side-encryption-algorithm", "SSE-KMS")
hadoop_conf.set("fs.s3a.server-side-encryption.key", kms_key_arn)

# test reading
df = spark.read.json("s3a://bucket/prefix1/prefix2/people.json")
df.show()

# test writing
df.write.mode("overwrite").parquet("3a://bucket/prefix1/prefix2/write-test/output")

import os
from pyspark.sql import SparkSession

try:
    spark.stop()
except:
    pass
spark = SparkSession.builder.getOrCreate()

#The name of one of the roles you are entitled to
profile_name="my-role-name-read-write"

# use boto3 for convenience to get credentials form credentials file populated by Domino
# can use any method desirable to extract the credentials
import boto3
role_creds = boto3.Session(profile_name=profile_name).get_credentials().get_frozen_credentials()

# configure the connector
# Use the TemporaryAWSCredentialsProvider
hadoop_conf = spark.sparkContext._jsc.hadoopConfiguration()
hadoop_conf.set("fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider")
hadoop_conf.set("fs.s3a.access.key", role_creds.access_key)
hadoop_conf.set("fs.s3a.secret.key", role_creds.secret_key)
hadoop_conf.set("fs.s3a.session.token", role_creds.token)

# test reading
df = spark.read.json("s3a://bucket/prefix1/prefix2/people.json")
df.show()

# test writing
df.write.mode("overwrite").parquet("s3a://bucket/prefix1/prefix2/write-test/output")

import os
from pyspark.sql import SparkSession

try:
    spark.stop()
except:
    pass

# First, set the AWS_PROFILE environment variable to the name of the profile found in the credentials file $AWS_SHARED_CREDENTIALS_FILE.
# If you're doing this in a notebook, first stop your Spark session or context for the change to take effect.
os.environ['AWS_PROFILE'] = 'name-of-profile-to-use'

# Next, configure the Spark connector by setting up the provider type and the name of the profile to use. Be sure to replace the .appName() argument with the name of your app.
spark = SparkSession.builder
          .appName("Credential Spark Test")
          .config("spark.hadoop.fs.s3a.aws.credentials.provider", "com.amazonaws.auth.profile.ProfileCredentialsProvider")
          .config("spark.executorEnv.AWS_PROFILE", os.environ['AWS_PROFILE'])
          .getOrCreate()

# Read some data from AWS (replace with your S3 URI)
df = spark.read.json("s3a://foobar/bazbux.json")
df.show()