Notes

import csv
from pymongo import MongoClient
from multiprocessing import Pool

# Define a function to export data in batches
def export_batch(batch_num):
batch_size = 10000 # Adjust batch size as needed
skip = batch_num * batch_size
cursor = collection.find({}).skip(skip).limit(batch_size)

csv_file_path = 'MLMonitorData.csv' # Use a single CSV file

with open(csv_file_path, 'a', newline='') as csvfile: # Append mode
fieldnames = [
"_id",
"factory_id",
"dcu_id",
"conveyor_id",
"LinkNumber",
"ChainSpeed",
"MainDrive",
"LubeCycleN1",
"LubeCycleN2",
"LubeCycleN3",
"LubeCycleN4",
"LubeRequest",
"LubeVersion",
"LubeLevel",
"LubeVoltage",
"PumpVersion",
"PumpNum",
"PumpIp",
"DCUDateTime",
"created_at",
"updated_at"
]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

for document in cursor:
# Write the document
writer.writerow(document)

# Determine the number of batches based on your dataset size
batch_size = 10000 # Adjust batch size as needed
total_records = total_count # Replace with the actual count

num_batches = (total_records // batch_size) + 1

if __name__ == '__main__':
with Pool(processes=4) as pool: # Adjust the number of processes as needed
pool.map(export_batch, range(num_batches))

# Close MongoDB connection
client.close()