I am able to pull in one liners and monthly exports but I am not able to pull in the views from the csv export. How would I pull in data that matches the different csv export types?
Here’s my code for the monthly export:
import requests
from pyspark.sql.functions import from_json
from pyspark.sql.types import StructType, StructField, StringType, DoubleType, TimestampType, IntegerType, BooleanType
import pandas as pd
from pandas import json_normalize
from pyspark.sql.functions import explode
‘’’
query_params = {
‘skip’: ‘0’,
‘take’: 2
}
‘’’
payload = {}
headers = {
‘Accept’: ‘application/json’,
‘x-api-key’: x_api_key_value,
‘Authorization’: authorization_value
}
response = requests.get(url, headers=headers, data=payload, params=query_params)
Check if the request was successful (status code 200)
if response.status_code == 200:
# Convert JSON response to Spark DataFrame
json_data = pd.read_json(response.text)
# Create a DataFrame from JSON data
spark_df = spark.createDataFrame(json_data)
# Show the DataFrame
df = spark_df.select(
col("results.comboName").alias("comboName"),
col("results.date").alias("date"),
col("results.output.*"),
col("status")
)
# Display the updated DataFrame
display(df)
else:
print(f"Error: {response.status_code}, {response.text}")