Notes

import pandas as pd
import numpy as np
import pprint as pp
import json
from os import listdir
from os.path import isfile, join
import os
import jmespath as jsonPathEval
from collections import defaultdict

def read_csv(csv_file_path):

# Function reads the csv file at location : ../../../DataDumpAccess/user/exportedData.csv
# Contains subjectID,email,name.
# Extracts all the rows and storing it in list format with use df (panda module)

df = pd.read_csv(csv_file_path)
columns_to_be_extracted = [0,2,3]
df = df[df.columns[columns_to_be_extracted]]
exported_user_list = df.to_dict(orient="list")
# exported_user_dict = df.to_dict(orient="dict")
return exported_user_list

def get_oil_status(oil_csv_path):
if os.path.isfile(oil_csv_path):
df = pd.read_csv(oil_csv_path)
# pp.pprint("File found")
oil_dict = df.to_dict()
oil_value = oil_dict["KEY;VALUE"][8].split(";")[1]
return oil_value
else:
# pp.pprint("Not a file found")
return "NA"


def read_all_json(userJSONPath):
for fileName in os.listdir(userJSONPath):
if fileName.endswith(".json"):
user_subject_id = os.path.splitext(fileName)[0]
try:
index_for_currentUser = exportedUserList["subjectID"].index(user_subject_id)
oil_csv_path = userJSONPath+'Report/'+user_subject_id+'/cache.csv'
skin_status = get_oil_status(oil_csv_path)
# pp.pprint(type(exportedUserList["subjectID"][index_for_currentUser]))
except ValueError:
continue
with open(userJSONPath+fileName) as f:
data = json.load(f)
parse_json_for_concernValues(data,index_for_currentUser,skin_status)
else:
continue

def parse_json_for_concernValues(userJSONData,index,skin_status):
answer = []
for question_id in user_concern_quesID:
result = jsonPathEval.search('moments[?type==`SkinIntake`].measurementGroups[].measurements[] | [?details[?value==`%s`]]' %question_id,userJSONData)
answer_json = jsonPathEval.search('[0].details[?type==`answer`].value',result)
answer_value = int(answer_json[0])
answer.append(answer_value)

construct_final_dict(answer,index,skin_status)


def construct_final_dict(answer_value,index_for_currentUser,skin_status) :
subject_id = exportedUserList["subjectID"][index_for_currentUser]
email = exportedUserList["email"][index_for_currentUser].lstrip()
name = exportedUserList["name"][index_for_currentUser].lstrip()
skintype = skin_status
smoothness = answer_value[0]
breakouts = answer_value[1]
hydration = answer_value[2]
firmness = answer_value[3]
pigmentation = answer_value[4]
eyearea = answer_value[5]
wrinkles = answer_value[6]
pores = answer_value[7]
sensitivity = answer_value[8]
oiliness = answer_value[9]
radiance = answer_value[10]
if not name== "#DEMONAME#":
finalDict[index_for_currentUser] = [subject_id,email,name,skin_status,smoothness,breakouts,hydration,firmness,pigmentation,eyearea,wrinkles,pores,sensitivity,oiliness,radiance]


baristaSource = os.getcwd()
os.chdir('../../DataDumpAccess')
rootPath = os.getcwd()
csvPath = rootPath + '/Data/user/exportedData.csv'
userJSONPath = rootPath + '/Data/user/'
finalDict = {}

user_concern_quesID = [
"SI-CONCERNS-SMOOTHNESS",
"SI-CONCERNS-BREAKOUTS",
"SI-CONCERNS-HYDRATION",
"SI-CONCERNS-FIRMNESS",
"SI-CONCERNS-PIGMENTATION",
"SI-CONCERNS-EYEAREA",
"SI-CONCERNS-WRINKLES",
"SI-CONCERNS-PORES",
"SI-CONCERNS-SENSITIVITY",
"SI-CONCERNS-OILINESS",
"SI-CONCERNS-RADIANCE"
]

global exported_user_list,index_for_currentUser

exportedUserList = read_csv(csvPath)
read_all_json(userJSONPath)

df = pd.DataFrame.from_dict(finalDict,orient='index')
df.columns=["SUBJECT-ID","EMAIL","NANE","SKIN TYPE","SMOOTHNESS","BREAKOUTS","HYDRATION","FIRMNESS","PIGMENTATION","EYEAREA","WRINKLES","PORES","SENSITIVITY","OILINESS","RADIANCE"]
df.to_csv("data.csv")