Notes

######################################################################################################################
# Yiting Nan 2018.4.20 XML compare with AVRO
# Please follow the readme.txt to run this script
######################################################################################################################
import os, sys
import subprocess
import re
import filecmp

# Here the list order doesn't matter, tuple order matters, all attributes need to convert to lowercase
# attributes_2 tuple is in reverse order
attributes_2 = [("isreportperiodlt12", "reportendingperiod"), ("ccoemployerid", "ccoemployername"),
("totalclaimamount", "isclaimfiled"), ("custodiancountry","custodianstate"),
("admincountry", "adminstate"), ("principalunderwritercountry","principalunderwriterstate"),
("ccocountry", "ccostate"), ("commonstocktickersymbol","commonstockexchange"),
("principalcountry", "principalstate"), ("publicaccountantcountry", "publicaccountantstate"),
("depositorcountry","depositorstate"), ("uitadmincountry","uitadminstate"),
("dividendamountpershareinarrear","dividendissuetitle"), ("officecountry", "officestate"),
("registrantnumshare", "registrantholdssecuritycusipno"), ("fundstickersymbol", "fundexchange"),
("smallinvestmentadviserstartdate", "issmallinvestmentadviserhired"), ("sponsorcountry", "sponsorstate"),
("smallsubadviserstartdate", "issmallsubadviserhired"), ("foreignsubsidiarylei", "foreignsubsidiaryname"),
("smallsubadviserterminatedcountry", "smallsubadviserterminatedstate"),
("idemnityproviderlei","idemnityprovidername"), ("smallcustodiancountry","smallcustodianstate"),
("smallsubadvisercountry","smallsubadviserstate"), ("seriesid", "seriesname"),
("investmentadvisercountry","investmentadviserstate"), ("pricingservicecountry","pricingservicestate"),
("brokercountry", "brokerstate"), ("interfundlendingdaysoutstanding","interfundlendingloanaverage"),
("interfundborrowingdaysoutstanding", "interfundborrowingloanaverage"),
("smallinvestmentadvisercountry","smallinvestmentadviserstate"), ("subadviserstartdate","issubadviserhired"),
("transferagentcountry", "transferagentstate"), ("subadviserterminatedcountry","subadviserterminatedstate"),
("smalltransferagentcountry", "smalltransferagentstate"),
("smallinvestmentadviserterminatedcountry", "smallinvestmentadviserterminatedstate"),
("trusteecountry", "trusteestate"), ("separateaccountseriesid", "isregistrantseparateinsuranceaccount"),
("brokerdealercountry", "brokerdealerstate"),("subadvisercountry","subadviserstate"),
("secfileno", "fundname"), ("investmentadviserterminatedcountry", "investmentadviserterminatedstate"),
("secfileno, swingupperlimit"), ("shareholderserviceagentcountry", "shareholderserviceagentstate")]
# attributes_3 is in the right order
attributes_3 = [("registrantheldsecuritytickersymbol", "registrantholdssecuritycusipno", "registrantnumshare"),
("sharesoutstandingclassname","sharesoutstandingclassid","sharesoutstandingtickersymbol"),
("seriesname", "seriesid", "terminationdate"), ("iscreditlineused","averagecreditlineused", "dayscreditused")]
# attributes_4 is in the right order
attributes_4 = [("defaultnature", "defaultdate", "defaultperthousandamount", "defaulttotalamount"),
("registrantsignedname", "signeddate", "signature", "title"),
("masterfundname", "masterfundfileno", "masterfundsecfileno", "masterfundlei"),
("collateralmanagername", "collateralmanagerlei", "iscollateralmanageraffliliated", "iscollateralmanageraffliliatedwithfund")]
# attributes_6 is in the right order
attributes_6 = [("feederfundname", "regfeederfundfileno", "regfeederfundseriesidno", "regfeederfundlei", "unregfeederfileno","unregfeederfundlei"),
("divestmentsecurityname", "divestmentsecuritytickersymbol", "divestmentsecuritycusipno", "divestednumshares","divesteddate","divestmentstatutename"),
("authorizedparticipantname", "authorizedparticipantfileno", "authorizedparticipantcrdno", "authorizedparticipantlei", "authorizedparticipantpurchasevalue","authorizedparticipantredeemvalue")]
# attributes_10 is in the right order
attributes_10 = [("separateaccountsecurityname", "separateaccountcontractid", "separateaccounttotalasset","numcontractssold", "grosspremiumreceived", "grosspremiumreceivedsection1035","numcontractsaffected", "contractvalueredeemed", "contractvalueredeemedsection1035","numcontractsaffectedredeemed")]

def rreplace(s, old, new, occurrence):
li = s.rsplit(old, occurrence)
return new.join(li)

# turn (a, b) to (b, a)
def inOrder_2(line, a, b):
aa = '"' + a + '":"[^"|^:]*?"'
bb = '"' + b + '":"[^"|^:]*?"'
find = r'(' + aa + ')(,)(' + bb + ')'
replace = r'321'
return re.sub(find, replace, line)

# turn to (a, b, c)
def inOrder_3(line, a, b, c):
x12 = '"(' + b + '|' + c + ')":"([^"|^:]*?)"'
find = r'' + x12 + ',"' + a + '":"([^"|^:]*?)",' + x12
replace = '"' + a + '":"' + r'3","' + r'1":"' + r'2","' + r'4":"' + r'5"'
line = re.sub(find, replace, line)
find = r'' + x12 + ',' + x12 + ',"' + a + '":"([^"|^:]*?)"'
replace = '"' + a + '":"' + r'5","' + r'1":"' + r'2","' + r'3":"' + r'4"'
line = re.sub(find, replace, line)
return inOrder_2(line, c, b)

def inOrder_4(line, a, b, c, d):
x123 = '"(' + b + '|' + c + '|' + d + ')":"([^"|^:]*?)"'
find = r'' + x123 + ',"' + a + '":"([^"|^:]*?)",' + x123 + ',' + x123
replace = '"' + a + '":"' + r'3","' + r'1":"' + r'2","' + r'4":"' + r'5","' + r'6":"' + r'7"'
line = re.sub(find, replace, line)
find = r'' + x123 + ',' + x123 + ',"' + a + '":"([^"|^:]*?)",' + x123
replace = '"' + a + '":"' + r'5","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'6":"' + r'7"'
line = re.sub(find, replace, line)
find = r'' + x123 + ',' + x123 + ',' + x123 + ',"' + a + '":"([^"|^:]*?)"'
replace = '"' + a + '":"' + r'7","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6"'
line = re.sub(find, replace, line)
return inOrder_3(line, b, c, d)


def inOrder_5(line, a, b, c, d, e):
x1234 = '"(' + b + '|' + c + '|' + d + '|' + e + ')":"([^"|^:]*?)"'
find = r'' + x1234 + ',"' + a + '":"([^"|^:]*?)",' + x1234 + ',' + x1234 + ',' + x1234
replace = '"' + a + '":"' + r'3","' + r'1":"' + r'2","' + r'4":"' + r'5","' + r'6":"' + r'7","' + r'8":"' + r'9"'
line = re.sub(find, replace, line)
find = r'' + x1234 + ',' + x1234 + ',"' + a + '":"([^"|^:]*?)",' + x1234 + ',' + x1234
replace = '"' + a + '":"' + r'5","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'6":"' + r'7","' + r'8":"' + r'9"'
line = re.sub(find, replace, line)
find = r'' + x1234 + ',' + x1234 + ',' + x1234 + ',"' + a + '":"([^"|^:]*?)",' + x1234
replace = '"' + a + '":"' + r'7","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'8":"' + r'9"'
line = re.sub(find, replace, line)
find = r'' + x1234 + ',' + x1234 + ',' + x1234 + ',' + x1234 + ',"' + a + '":"([^"|^:]*?)"'
replace = '"' + a + '":"' + r'9","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8"'
line = re.sub(find, replace, line)
return inOrder_4(line, b, c, d, e)

def inOrder_6(line, a, b, c, d, e, f):
x12345 = '"(' + b + '|' + c + '|' + d + '|' + e + '|' + f + ')":"([^"|^:]*?)"'
find = r'' + x12345 + ',"' + a + '":"([^"|^:]*?)",' + x12345 + ',' + x12345 + ',' + x12345 + ',' + x12345
replace = '"' + a + '":"'+ r'3","' + r'1":"' + r'2","' + r'4":"' + r'5","' + r'6":"' + r'7","' + r'8":"' + r'9","' + r'10":"' + r'11"'
line = re.sub(find, replace, line)
find = r'' + x12345 + ',' + x12345 + ',"' + a + '":"([^"|^:]*?)",' + x12345 + ',' + x12345 + ',' + x12345
replace = '"' + a + '":"'+ r'5","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'6":"' + r'7","' + r'8":"' + r'9","' + r'10":"' + r'11"'
line = re.sub(find, replace, line)
find = r'' + x12345 + ',' + x12345 + ',' + x12345 + ',"' + a + '":"([^"|^:]*?)",' + x12345 + ',' + x12345
replace = '"' + a + '":"'+ r'7","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'8":"' + r'9","' + r'10":"' + r'11"'
line = re.sub(find, replace, line)
find = r'' + x12345 + ',' + x12345 + ',' + x12345 + ',' + x12345 + ',"' + a + '":"([^"|^:]*?)",' + x12345
replace = '"' + a + '":"'+ r'9","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'10":"' + r'11"'
line = re.sub(find, replace, line)
find = r'' + x12345 + ',' + x12345 + ',' + x12345 + ',' + x12345 + ',' + x12345 + ',"' + a + '":"([^"|^:]*?)"'
replace = '"' + a + '":"'+ r'11","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10"'
line = re.sub(find, replace, line)
return inOrder_5(line, b, c, d, e, f)

def inOrder_7(line, a, b, c, d, e, f, g):
x6 = '"(' + b + '|' + c + '|' + d + '|' + e + '|' + f + '|' + g + ')":"([^"|^:]*?)"'
find = r'' + x6 + ',"' + a + '":"([^"|^:]*?)",' + x6 + ',' + x6 + ',' + x6 + ',' + x6 + ',' + x6
replace = '"' + a + '":"'+ r'3","' + r'1":"' + r'2","' + r'4":"' + r'5","' + r'6":"' + r'7","' + r'8":"' + r'9","' + r'10":"' + r'11","' + r'12":"' + r'13"'
line = re.sub(find, replace, line)
find = r'' + x6 + ',' + x6 + ',"' + a + '":"([^"|^:]*?)",' + x6 + ',' + x6 + ',' + x6 + ',' + x6
replace = '"' + a + '":"'+ r'5","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'6":"' + r'7","' + r'8":"' + r'9","' + r'10":"' + r'11","' + r'12":"' + r'13"'
line = re.sub(find, replace, line)
find = r'' + x6 + ',' + x6 + ',' + x6 + ',"' + a + '":"([^"|^:]*?)",' + x6 + ',' + x6 + ',' + x6
replace = '"' + a + '":"'+ r'7","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'8":"' + r'9","' + r'10":"'+ r'11","' + r'12":"' + r'13"'
line = re.sub(find, replace, line)
find = r'' + x6 + ',' + x6 + ',' + x6 + ',' + x6 + ',"' + a + '":"([^"|^:]*?)",' + x6 + ',' + x6
replace = '"' + a + '":"'+ r'9","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'10":"' + r'11","' + r'12":"' + r'13"'
line = re.sub(find, replace, line)
find = r'' + x6 + ',' + x6 + ',' + x6 + ',' + x6 + ',' + x6 + ',"' + a + '":"([^"|^:]*?)",' + x6
replace = '"' + a + '":"'+ r'11","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10",' + r'12":"' + r'13"'
line = re.sub(find, replace, line)
find = r'' + x6 + ',' + x6 + ',' + x6 + ',' + x6 + ',' + x6 + ',' + x6 + ',"' + a + '":"([^"|^:]*?)"'
replace = '"' + a + '":"' + r'13","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10"' + r'11":"' + r'12"'
line = re.sub(find, replace, line)
return inOrder_6(line, b, c, d, e, f, g)

def inOrder_8(line, a, b, c, d, e, f, g, h):
x7 = '"(' + b + '|' + c + '|' + d + '|' + e + '|' + f + '|' + g + '|' + h + ')":"([^"|^:]*?)"'
find = r'' + x7 + ',"' + a + '":"([^"|^:]*?)",' + x7 + ',' + x7 + ',' + x7 + ',' + x7 + ',' + x7 + ',' + x7
replace = '"' + a + '":"'+ r'3","' + r'1":"' + r'2","' + r'4":"' + r'5","' + r'6":"' + r'7","' + r'8":"' + r'9","' + r'10":"' + r'11","' + r'12":"' + r'13","' + r'14":"' + r'15"'
line = re.sub(find, replace, line)
find = r'' + x7 + ',' + x7 + ',"' + a + '":"([^"|^:]*?)",' + x7 + ',' + x7 + ',' + x7 + ',' + x7 + ',' + x7
replace = '"' + a + '":"'+ r'5","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'6":"' + r'7","' + r'8":"' + r'9","' + r'10":"' + r'11","' + r'12":"' + r'13","' + r'14":"' + r'15"'
line = re.sub(find, replace, line)
find = r'' + x7 + ',' + x7 + ',' + x7 + ',"' + a + '":"([^"|^:]*?)",' + x7 + ',' + x7 + ',' + x7 + ',' + x7
replace = '"' + a + '":"'+ r'7","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'8":"' + r'9","' + r'10":"' + r'11","' + r'12":"' + r'13","' + r'14":"' + r'15"'
line = re.sub(find, replace, line)
find = r'' + x7 + ',' + x7 + ',' + x7 + ',' + x7 + ',"' + a + '":"([^"|^:]*?)",' + x7 + ',' + x7 + ',' + x7
replace = '"' + a + '":"'+ r'9","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'10":"' + r'11","' + r'12":"' + r'13","' + r'14":"' + r'15"'
line = re.sub(find, replace, line)
find = r'' + x7 + ',' + x7 + ',' + x7 + ',' + x7 + ',' + x7 + ',"' + a + '":"([^"|^:]*?)",' + ',' + x7 + ',' + x7
replace = '"' + a + '":"'+ r'11","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10","' + r'12":"' + r'13","' + r'14":"' + r'15"'
line = re.sub(find, replace, line)
find = r'' + x7 + ',' + x7 + ',' + x7 + ',' + x7 + ',' + x7 + ',' + x7 + ',"' + a + '":"([^"|^:]*?)",' + ',' + x7
replace = '"' + a + '":"'+ r'13","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10","' + r'11":"' + r'12","' + r'14":"' + r'15"'
line = re.sub(find, replace, line)
find = r'' + x7 + ',' + x7 + ',' + x7 + ',' + x7 + ',' + x7 + ',' + x7 + ',' + x7 + ',"' + a + '":"([^"|^:]*?)"'
replace = '"' + a + '":"'+ r'15","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10","' + r'11":"' + r'12","' + r'13":"' + r'14"'
line = re.sub(find, replace, line)
return inOrder_7(line, b, c, d, e, f, g, h)