Notes

def inOrder_9(line, a, b, c, d, e, f, g, h, i):
x8 = '"(' + b + '|' + c + '|' + d + '|' + e + '|' + f + '|' + g + '|' + h + '|' + i + ')":"([^"|^:]*?)"'
find = r'' + x8 + ',"' + a + '":"([^"|^:]*?)",' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8
replace = '"' + a + '":"' + r'3","' + r'1":"' + r'2","' + r'4":"' + r'5","' + r'6":"' + r'7","' + r'8":"' + r'9","' + r'10":"' + r'11","' + r'12":"' + r'13","' + r'14":"' + r'15","' + r'16":"' + r'17"'
line = re.sub(find, replace, line)
find = r'' + x8 + ',' + x8 + ',"' + a + '":"([^"|^:]*?)",' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8
replace = '"' + a + '":"' + r'5","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'6":"' + r'7","' + r'8":"' + r'9","' + r'10":"' + r'11","' + r'12":"' + r'13","' + r'14":"'+ r'15","' + r'16":"' + r'17"'
line = re.sub(find, replace, line)
find = r'' + x8 + ',' + x8 + ',' + x8 + ',"' + a + '":"([^"|^:]*?)",' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8
replace = '"' + a + '":"' + r'7","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'8":"' + r'9","' + r'10":"' + r'11","' + r'12":"' + r'13","' + r'14":"' + r'15","' + r'16":"' + r'17"'
line = re.sub(find, replace, line)
find = r'' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',"' + a + '":"([^"|^:]*?)",' + x8 + ',' + x8 + ',' + x8 + ',' + x8
replace = '"' + a + '":"' + r'9","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'10":"' + r'11","' + r'12":"' + r'13","' + r'14":"' + r'15","' + r'16":"' + r'17"'
line = re.sub(find, replace, line)
find = r'' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',"' + a + '":"([^"|^:]*?)",' + x8 + ',' + x8 + ',' + x8
replace = '"' + a + '":"' + r'11","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10","' + r'12":"' + r'13","' + r'14":"' + r'15","' + r'16":"' + r'17"'
line = re.sub(find, replace, line)
find = r'' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',"' + a + '":"([^"|^:]*?)",' + ',' + x8 + ',' + x8
replace = '"' + a + '":"' + r'13","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10","' + r'11":"' + r'12","' + r'14":"' + r'15","' + r'16":"' + r'17"'
line = re.sub(find, replace, line)
find = r'' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',"' + a + '":"([^"|^:]*?)",' + x8
replace = '"' + a + '":"' + r'15","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10","' + r'11":"' + r'12","' + r'13":"' + r'14","' + r'16":"' + r'17"'
line = re.sub(find, replace, line)
find = r'' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',' + x8 + ',"' + a + '":"([^"|^:]*?)"'
replace = '"' + a + '":"' + r'17","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10","' + r'11":"' + r'12","' + r'13":"' + r'14","' + r'15":"' + r'16"'
line = re.sub(find, replace, line)
return inOrder_8(line, b, c, d, e, f, g, h, i)

def inOrder_10(line, a, b, c, d, e, f, g, h, i, j):
x9 = '"(' + b + '|' + c + '|' + d + '|' + e + '|' + f + '|' + g + '|' + h + '|' + i + '|' + j + ')":"([^"|^:]*?)"'
find = r'' + x9 + ',"' + a + '":"([^"|^:]*?)",' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9
replace = '"' + a + '":"' + r'3","' + r'1":"' + r'2","' + r'4":"' + r'5","' + r'6":"' + r'7","' + r'8":"' + r'9","' + r'10":"' + r'11","' + r'12":"' + r'13","' + r'14":"' + r'15","' + r'16":"' + r'17","' + r'18":"' + r'19"'
line = re.sub(find, replace, line)
find = r'' + x9 + ',' + x9 + ',"' + a + '":"([^"|^:]*?)",' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9
replace = '"' + a + '":"' + r'5","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'6":"' + r'7","' + r'8":"' + r'9","' + r'10":"' + r'11","' + r'12":"' + r'13","' + r'14":"' + r'15","' + r'16":"' + r'17","' + r'18":"' + r'19"'
line = re.sub(find, replace, line)
find = r'' + x9 + ',' + x9 + ',' + x9 + ',"' + a + '":"([^"|^:]*?)",' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9
replace = '"' + a + '":"' + r'7","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'8":"' + r'9","' + r'10":"' + r'11","' + r'12":"' + r'13","' + r'14":"' + r'15","' + r'16":"' + r'17","' + r'18":"' + r'19"'
line = re.sub(find, replace, line)
find = r'' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',"' + a + '":"([^"|^:]*?)",' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9
replace = '"' + a + '":"' + r'9","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'10":"' + r'11","' + r'12":"' + r'13","' + r'14":"' + r'15","' + r'16":"' + r'17","' + r'18":"' + r'19"'
line = re.sub(find, replace, line)
find = r'' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',"' + a + '":"([^"|^:]*?)",' + x9 + ',' + x9 + ',' + x9 + ',' + x9
replace = '"' + a + '":"' + r'11","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10","' + r'12":"' + r'13","' + r'14":"' + r'15","' + r'16":"' + r'17","' + r'18":"' + r'19"'
line = re.sub(find, replace, line)
find = r'' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',"' + a + '":"([^"|^:]*?)",' + x9 + ',' + x9 + ',' + x9
replace = '"' + a + '":"' + r'13","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10","' + r'11":"' + r'12","' + r'14":"' + r'15","' + r'16":"' + r'17","' + r'18":"' + r'19"'
line = re.sub(find, replace, line)
find = r'' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',"' + a + '":"([^"|^:]*?)",' + x9 + ',' + x9
replace = '"' + a + '":"' + r'15","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10","' + r'11":"' + r'12","' + r'13":"' + r'14","' + r'16":"' + r'17","' + r'18":"' + r'19"'
line = re.sub(find, replace, line)
find = r'' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',"' + a + '":"([^"|^:]*?)",' + x9
replace = '"' + a + '":"' + r'17","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10","' + r'11":"' + r'12","' + r'13":"' + r'14","' + r'15":"' + r'16","' + r'18":"' + r'19"'
line = re.sub(find, replace, line)
find = r'' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',' + x9 + ',"' + a + '":"([^"|^:]*?)"'
replace = '"' + a + '":"' + r'19","' + r'1":"' + r'2","' + r'3":"' + r'4","' + r'5":"' + r'6","' + r'7":"' + r'8","' + r'9":"' + r'10","' + r'11":"' + r'12","' + r'13":"' + r'14","' + r'15":"' + r'16","' + r'17":"' + r'18"'
line = re.sub(find, replace, line)
return inOrder_9(line, b, c, d, e, f, g, h, i, j)

if os.path.isfile("json1File.json"):
os.remove("json1File.json")

# ####################################
# # Process on XML files
# ####################################
with open('json1File.json', 'a') as fo:
#convert each xml to json
for line in open(sys.argv[1]):
#print(line.rstrip())
try:
retcode = subprocess.Popen('xsltr.exe ..\'+ sys.argv[2]+'\'+ line.rstrip() + ' xml2json.xsl output.json', shell=True).wait()
if retcode < 0:
print("Child was terminated by signal", -retcode, file=sys.stderr)
exit(1)
except OSError as e:
print("Execution failed:", e, file=sys.stderr)
#make each json as one line file
with open('new.json', 'w+') as fnew:
for line in open('output.json', 'r'):
line = line.rstrip("n")
fnew.write(line)
fnew.write("n")

#append output.json to one file
for line in open('new.json', 'r'):
line = line.replace(" ", "").lower()

# remove "edgarSubmission" : .... {"headerdata" : , "formdata" and matching braces
braceNum = re.search(r'{"edgarsubmission":{.*?"headerdata":', line).group(0).count("{")
# remove last } that is with {"EdgarSubmission"
line=rreplace(line, "}", "", braceNum)

line = re.sub(r'{"edgarsubmission":{.*?"headerdata":', '', line)
line = line.replace(',"formdata":', '')

#remove redundunt brace
line = re.sub(r'({)([^,|^:]+?)(})', r'2', line)
line = re.sub(r'[{("[a-z]*?":[^:|^,]*?)}]', r'{1}', line)
line = re.sub(r'[([a-z|-]*?)]', r'1', line)

#Iteration to the max nest level, repeat till line==line
x = True
while x:
newLine = line
# line = re.sub(r'[{(^([{)+?)}]', r'{1}', newLine)
line = re.sub(r'[{(.+?)}]', r'{1}', newLine)
x = (newLine != line)

# remove quote on flag
line = line.replace('"false"', 'false')
line = line.replace('"true"', 'true')

################### adjust the sibling order to the same as xsd-avsc

# '...name...' attribute can have ',' in it.
for x in attributes_2[:]:
line = inOrder_2(line, x[0], x[1])

for x in attributes_3[:]:
line = inOrder_3(line, x[0], x[1], x[2])

for x in attributes_4[:]:
line = inOrder_4(line, x[0], x[1], x[2], x[3])

for x in attributes_6[:]:
line = inOrder_6(line, x[0], x[1], x[2], x[3], x[4], x[5])

for x in attributes_10[:]:
line = inOrder_10(line, x[0], x[1], x[2], x[3], x[4], x[5], x[6], x[7], x[8], x[9])

fo.write(line)


if os.path.isfile("avro_reformat.json"):
os.remove("avro_reformat.json")

####################################
# Process on Avro query result
####################################

# remove the file name,
# with open('avro_queryOrdered.txt', 'r') as f:
# content = f.read()
# content(r'[0-9]+.*.XMLD.DOCUMTn', '', content, re.MSI)
print("avro query reformatting ...")
with open('avro_queryOrdered.txt', 'r') as faaa, open('avro_reformat.json', 'a') as ftmp:
#with open('3Sample.txt', 'r') as faaa, open('avro_reformat.json', 'a') as ftmp:
for line in faaa:
# remove filename and single space
line = re.sub(r'.*?{"submissiontype":', '{"submissiontype":', line)
line = line.replace(" ", "").replace("t", "").lower()

# remove null and [] value
line = re.sub(r'"[a-z]+?":[],', '', line)
line = re.sub(r',"[a-z]+?":[]', '', line)

line = re.sub(r'"[a-z0-9]+?":null,', '', line)
line = re.sub(r',"[a-z0-9]+?":null', '', line)

# remove redundunt brace
line = re.sub(r'[{("[a-z]*?":[^:|^,]*?)}]', r'{1}', line)
line = re.sub(r'[("[^:|^,]+?")]', r'1', line)
line = re.sub(r'[([a-z|-]*?)]', r'1', line)

# Iteration to the max nest level, repeat till line==line
x = True
while x:
newLine = line
#line = re.sub(r'[{(^([{)+?)}]', r'{1}', newLine)
line = re.sub(r'[{(.+?)}]', r'{1}', newLine)
x = (newLine != line )
ftmp.write(line)
print("Done")

print("avro_reformat.json compare to json1File.json returned: ", filecmp.cmp('avro_reformat.json','json1File.json', shallow=False))