Notes

rom rest_framework.decorators import api_view

@api_view(['POST'])
def csv(request):
client = pymongo.MongoClient("mongodb://localhost:27017")

# Select the database

db = client["ApexLubrication"]

# Select the collection

collection = db["apex_ml_conveyor_notes"]
# Define the index field (adjust as needed)

index_field = "_id" # Assuming _id field is indexed

# Create an aggregation pipeline to count indexed documents

pipeline = [

{"$match": {index_field: {"$exists": True}}}, # Match documents with the indexed field

{"$group": {"_id": None, "count": {"$sum": 1}}} # Count matching documents

]

# Execute the aggregation pipeline

result = list(collection.aggregate(pipeline))

# Calculate the estimated total count

estimated_total = result[0]["count"] if result else 0

# Close the connection

total_records = estimated_total # collection.count_documents({}) # Get the total record count
batch_size = 1000000 # Adjust batch size as needed
num_batches = (total_records // batch_size) + 1

with Pool(processes=4) as pool: # Adjust the number of processes as needed
pool.map(export_batch, range(num_batches))

client.close()

print(f"Estimated Total Count: {estimated_total}")


return JsonResponse({"success":estimated_total})


def export_batch(batch_num):
batch_size = 1000000 # Adjust batch size as needed
skip = batch_num * batch_size
cursor = collection.find({}).skip(skip).limit(batch_size)

csv_file_path = 'apex_ml_conveyor_notes.csv' # Use a single CSV file

with open(csv_file_path, 'a', newline='') as csvfile: # Append mode

fieldnames = ['_id',
'factory_id',
'dcu_id',
'conveyor_id',
'FLAG_apex',
'username_id',
'note',
'entered_at',
'DCUDateTime',
'created_at',
'updated_at']

writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

# Write the header row only once
if skip == 0:
writer.writeheader()

for document in cursor:
# Write the document
writer.writerow(document)



# total_records = estimated_total # collection.count_documents({}) # Get the total record count
# batch_size = 1000000 # Adjust batch size as needed
# num_batches = (total_records // batch_size) + 1
#
# with Pool(processes=4) as pool: # Adjust the number of processes as needed
# pool.map(export_batch, range(num_batches))

# Close MongoDB connection
client.close()