import pandas as pd import json import os import re # from bson import ObjectId # from pymongo.results import UpdateResult from s3Ops import read_file_from_s3 from backendAPIs import ( update_onboarding_status, add_role ) # class CustomJSONEncoder(json.JSONEncoder): # def default(self, obj): # if isinstance(obj, ObjectId): # return str(obj) # Convert ObjectId to string for JSON serialization # elif isinstance(obj, UpdateResult): # # Convert UpdateResult to a serializable dictionary # return { # "matched_count": obj.matched_count, # "modified_count": obj.modified_count, # "upserted_id": str(obj.upserted_id) if obj.upserted_id else None, # "acknowledged": obj.acknowledged # } # # For other non-serializable objects, use the default behavior # return super().default(obj) def get_partners_from_excel(xlsx_file): """ Read Excel file containing pa information. Remove headers and return clean df Args: file_path (str): Path to the Excel file. sheet_name (str): Excel sheet name with partner list Returns: pd.DataFrame: DataFrame containing partner information. """ sheet_name = 'Partners' df = pd.read_excel(xlsx_file, sheet_name = sheet_name) df.columns = df.iloc[2] df = df.drop(df.index[:3]) df = df.reset_index(drop=True) return df def pretty_print_files(files_dict): """ Pretty print the dictionary containing file names. Args: files_dict (dict): Dictionary where the keys are the file categories, and the values are the file names. """ file_message = "" for key, value in files_dict.items(): file_message += f"- {key.replace('_', ' ').capitalize()}: {value} \n" return file_message def process_partners(partners_df): """ Process and insert users from an Excel file into MongoDB. Args: file_path (str): Path to the Excel file. """ entity_id = os.getenv('ENTITY_ID') # print(f"Header:\n {partners_df.head()}") users = [] roles = [] # Iterate through the DataFrame and prepare partner records for index, row in partners_df.iterrows(): # Skip rows without a valid Partner name if pd.isna(row['Partner']) in ['Partner']: print(f"Skipping row {index}") continue partner_dict = { "firstName": row['Primary Contact Name'].split(' ')[0] if pd.notna(row['Primary Contact Name']) else '', "lastName": ' '.join(row['Primary Contact Name'].split(' ')[1:]) if pd.notna(row['Primary Contact Name']) else '', "email": row['Email'] if pd.notna(row['Email']) else '' } users.append(partner_dict) print(f"Adding user {partner_dict}") # Insert user record into user collection # user_id = add_user(partner_dict) # print(json.dumps(insert_result, indent=2)) # Adding role automatiocally adds users role, role_id = process_roles(row) roles.append(role) print(role_id) return users def remove_all_special_chars(text): return re.sub(r'[^a-zA-Z0-9\s]', '', text) def process_roles(row): """ Process and insert users from an Excel file into MongoDB. Args: file_path (str): Path to the Excel file. """ entity_id = os.getenv('ENTITY_ID') accessType, role = "", "" if pd.notna(row['Class']): # Check if the Class value is not NaN if row['Class'] == 'Limited Partner' or row['Class'] == 'Member': accessType = 'USER' role = "LIMITED PARTNER" elif row['Class'] == 'General Partner': accessType = 'ADMIN' role = "GENERAL PARTNER" entity_name = row['Partner'] if any(indicator.lower() in row['Partner'].lower() for indicator in ["corp", "trust", "international", "llc", "ltd", "council", "inc", "llp", "sp", "lp", "capital", "fund","foundation","and","&","l.p."]) or any(char.isdigit() for char in row['Partner']) or re.search(r'\d{1,2}/\d{1,2}/\d{2,4}', row['Partner']) else '' if len(entity_name) > 0: first_name = row['Primary Contact Name'].split(' ')[0] if pd.notna(row['Primary Contact Name']) else '' last_name = ' '.join(row['Primary Contact Name'].split(' ')[1:]) if pd.notna(row['Primary Contact Name']) else '' else: first_name = row['Partner'].split(' ')[0] if pd.notna(row['Partner']) else '' last_name = ' '.join(row['Partner'].split(' ')[1:]) if pd.notna(row['Partner']) else '' role_dict = { # "userId": { # "$oid": user_id # }, # "entityId": { # "$oid": entity_id # }, "entityId": entity_id, "entityName":entity_name, "accessType": accessType, "role": role, "firstName": first_name, "lastName": last_name, # "status": "NOT INVITED", # "isInvitationSent": False, # "dob": None, "phoneNumber": row['Phone'].split(' ')[0] if pd.notna(row['Phone']) else '', "taxID1": row['Tax ID'].split(' ')[0] if pd.notna(row['Tax ID']) else '', "taxID2": row['Tax ID Type'].split(' ')[0] if pd.notna(row['Tax ID Type']) else '', "street": remove_all_special_chars(row['Street Address'].split(' ')[0]) if pd.notna(row['Street Address']) else '', "address": remove_all_special_chars(" ".join(row['Street Address'].split(' ')[1:])) if pd.notna(row['Street Address']) else '', "country":row['Country'].split(' ')[0] if pd.notna(row['Country']) else '', "city": row['City'].split(' ')[0] if pd.notna(row['City']) else '', "state": row['State'].split(' ')[0] if pd.notna(row['State']) else '', "zipcode": row['ZIP'].split(' ')[0] if pd.notna(row['ZIP']) else '', "commitedAmount": row['Commitment'], "calledCapital": row['Called Capital'], "dateOfCommitment": row['Issue date'], "partnerExactName": row['Partner'], "email":row['Email'] if pd.notna(row['Email']) else '' # "gender": "", # "ethnicity": "", # "photoURL": None, # "menuOpen": False } print("entityName") print(entity_name) print(role_dict) # Save role into database role_id = add_role(role_dict) print(f"Upserted Role ID: {role_id}") return role_dict, role_id def process_all_partners(partners_file_path): # Process partner data step_number = 3 step_number -= 1 success_message = "" error_message = "" status = "IN-PROGRESS" bucket_name = os.getenv('S3_UPLOAD_BUCKET_NAME') print(f"Processing file: {partners_file_path}") response = update_onboarding_status(step_number, status, error_message, success_message) if 'error' in response: print(f"Failed to fetch onboarding status: Error: {response['error']}") print(f"Status Code: {response['status_code']}") return False partner_excel_file = read_file_from_s3(bucket_name, partners_file_path) partners_df = get_partners_from_excel(partner_excel_file) print(partners_df.head()) partners_dict = process_partners(partners_df) if partners_dict: item_count = len(partners_dict) success_message = f"Onboarded {item_count} GPs&LPs." status = "COMPLETE" else: error_message = "No users were onboaerded." status = "FAILED" print("*"*30) response = update_onboarding_status(step_number, status, error_message, success_message) if 'error' in response: print(f"Failed to fetch onboarding status: Error: {response['error']}") print(f"Status Code: {response['status_code']}") return False # print("Partner list:") # print(json.dumps(partners_dict, indent=2, sort_keys=True, cls=CustomJSONEncoder)) return True if __name__ == "__main__": from dotenv import load_dotenv load_dotenv() onboardingId = "66ea163564e2f97a059160ef" os.environ["ONBOARDING_ID"] = onboardingId from initOnboarding import initialize_onboarding initialize_onboarding() found_files = {'lpa': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-Please_DocuSign_CerraCap_II_LP_Limited_Partn.pdf', 'partner': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-1-cerracap-ii-lp_2024-07_09_short_partner.xlsx', 'financials': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/2-3-cerracap-ii-lp_2024-08-26_financials.xlsx', 'bankTransactions': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/4-CerraCap_II__LP_bank_transactions_2016-01-01-2024-07-03.xlsx', 'journals': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/5-cerracap-ii-lp_2024-07-09_journals-export.xlsx', 'fund_performance': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/6-cerracap-ii-lp_2024-07-09_fund-performance-report.xlsx'} partner_excel_file_path = found_files["partner"] process_all_partners(partner_excel_file_path)