mirror of
https://github.com/x1xhlol/system-prompts-and-models-of-ai-tools.git
synced 2026-08-16 21:24:09 +00:00
240 lines
8.9 KiB
Python
240 lines
8.9 KiB
Python
import pandas as pd
|
|
import json
|
|
import os
|
|
import re
|
|
# from bson import ObjectId
|
|
# from pymongo.results import UpdateResult
|
|
|
|
from s3Ops import read_file_from_s3
|
|
|
|
from backendAPIs import (
|
|
update_onboarding_status,
|
|
add_role
|
|
)
|
|
|
|
# class CustomJSONEncoder(json.JSONEncoder):
|
|
# def default(self, obj):
|
|
# if isinstance(obj, ObjectId):
|
|
# return str(obj) # Convert ObjectId to string for JSON serialization
|
|
# elif isinstance(obj, UpdateResult):
|
|
# # Convert UpdateResult to a serializable dictionary
|
|
# return {
|
|
# "matched_count": obj.matched_count,
|
|
# "modified_count": obj.modified_count,
|
|
# "upserted_id": str(obj.upserted_id) if obj.upserted_id else None,
|
|
# "acknowledged": obj.acknowledged
|
|
# }
|
|
# # For other non-serializable objects, use the default behavior
|
|
# return super().default(obj)
|
|
|
|
|
|
def get_partners_from_excel(xlsx_file):
|
|
"""
|
|
Read Excel file containing pa information.
|
|
Remove headers and return clean df
|
|
|
|
Args:
|
|
file_path (str): Path to the Excel file.
|
|
sheet_name (str): Excel sheet name with partner list
|
|
Returns:
|
|
pd.DataFrame: DataFrame containing partner information.
|
|
"""
|
|
sheet_name = 'Partners'
|
|
df = pd.read_excel(xlsx_file, sheet_name = sheet_name)
|
|
df.columns = df.iloc[2]
|
|
df = df.drop(df.index[:3])
|
|
df = df.reset_index(drop=True)
|
|
|
|
return df
|
|
|
|
def pretty_print_files(files_dict):
|
|
"""
|
|
Pretty print the dictionary containing file names.
|
|
|
|
Args:
|
|
files_dict (dict): Dictionary where the keys are the file categories, and the values are the file names.
|
|
"""
|
|
file_message = ""
|
|
for key, value in files_dict.items():
|
|
file_message += f"- {key.replace('_', ' ').capitalize()}: {value} \n"
|
|
|
|
return file_message
|
|
|
|
|
|
def process_partners(partners_df):
|
|
"""
|
|
Process and insert users from an Excel file into MongoDB.
|
|
|
|
Args:
|
|
file_path (str): Path to the Excel file.
|
|
"""
|
|
|
|
entity_id = os.getenv('ENTITY_ID')
|
|
|
|
# print(f"Header:\n {partners_df.head()}")
|
|
users = []
|
|
roles = []
|
|
|
|
# Iterate through the DataFrame and prepare partner records
|
|
for index, row in partners_df.iterrows():
|
|
|
|
# Skip rows without a valid Partner name
|
|
if pd.isna(row['Partner']) in ['Partner']:
|
|
print(f"Skipping row {index}")
|
|
continue
|
|
|
|
partner_dict = {
|
|
"firstName": row['Primary Contact Name'].split(' ')[0] if pd.notna(row['Primary Contact Name']) else '',
|
|
"lastName": ' '.join(row['Primary Contact Name'].split(' ')[1:]) if pd.notna(row['Primary Contact Name']) else '',
|
|
"email": row['Email'] if pd.notna(row['Email']) else ''
|
|
}
|
|
|
|
users.append(partner_dict)
|
|
|
|
print(f"Adding user {partner_dict}")
|
|
# Insert user record into user collection
|
|
# user_id = add_user(partner_dict)
|
|
# print(json.dumps(insert_result, indent=2))
|
|
|
|
# Adding role automatiocally adds users
|
|
role, role_id = process_roles(row)
|
|
roles.append(role)
|
|
print(role_id)
|
|
|
|
return users
|
|
|
|
def remove_all_special_chars(text):
|
|
return re.sub(r'[^a-zA-Z0-9\s]', '', text)
|
|
|
|
|
|
def process_roles(row):
|
|
"""
|
|
Process and insert users from an Excel file into MongoDB.
|
|
|
|
Args:
|
|
file_path (str): Path to the Excel file.
|
|
"""
|
|
entity_id = os.getenv('ENTITY_ID')
|
|
accessType, role = "", ""
|
|
if pd.notna(row['Class']): # Check if the Class value is not NaN
|
|
if row['Class'] == 'Limited Partner' or row['Class'] == 'Member':
|
|
accessType = 'USER'
|
|
role = "LIMITED PARTNER"
|
|
elif row['Class'] == 'General Partner':
|
|
accessType = 'ADMIN'
|
|
role = "GENERAL PARTNER"
|
|
|
|
entity_name = row['Partner'] if any(indicator.lower() in row['Partner'].lower() for indicator in ["corp", "trust", "international", "llc", "ltd", "council", "inc", "llp", "sp", "lp", "capital", "fund","foundation","and","&","l.p."]) or any(char.isdigit() for char in row['Partner']) or re.search(r'\d{1,2}/\d{1,2}/\d{2,4}', row['Partner']) else ''
|
|
|
|
if len(entity_name) > 0:
|
|
first_name = row['Primary Contact Name'].split(' ')[0] if pd.notna(row['Primary Contact Name']) else ''
|
|
last_name = ' '.join(row['Primary Contact Name'].split(' ')[1:]) if pd.notna(row['Primary Contact Name']) else ''
|
|
else:
|
|
first_name = row['Partner'].split(' ')[0] if pd.notna(row['Partner']) else ''
|
|
last_name = ' '.join(row['Partner'].split(' ')[1:]) if pd.notna(row['Partner']) else ''
|
|
|
|
|
|
role_dict = {
|
|
# "userId": {
|
|
# "$oid": user_id
|
|
# },
|
|
# "entityId": {
|
|
# "$oid": entity_id
|
|
# },
|
|
"entityId": entity_id,
|
|
"entityName":entity_name,
|
|
"accessType": accessType,
|
|
"role": role,
|
|
"firstName": first_name,
|
|
"lastName": last_name,
|
|
# "status": "NOT INVITED",
|
|
# "isInvitationSent": False,
|
|
# "dob": None,
|
|
"phoneNumber": row['Phone'].split(' ')[0] if pd.notna(row['Phone']) else '',
|
|
"taxID1": row['Tax ID'].split(' ')[0] if pd.notna(row['Tax ID']) else '',
|
|
"taxID2": row['Tax ID Type'].split(' ')[0] if pd.notna(row['Tax ID Type']) else '',
|
|
"street": remove_all_special_chars(row['Street Address'].split(' ')[0]) if pd.notna(row['Street Address']) else '',
|
|
"address": remove_all_special_chars(" ".join(row['Street Address'].split(' ')[1:])) if pd.notna(row['Street Address']) else '',
|
|
"country":row['Country'].split(' ')[0] if pd.notna(row['Country']) else '',
|
|
"city": row['City'].split(' ')[0] if pd.notna(row['City']) else '',
|
|
"state": row['State'].split(' ')[0] if pd.notna(row['State']) else '',
|
|
"zipcode": row['ZIP'].split(' ')[0] if pd.notna(row['ZIP']) else '',
|
|
"commitedAmount": row['Commitment'],
|
|
"calledCapital": row['Called Capital'],
|
|
"dateOfCommitment": row['Issue date'],
|
|
"partnerExactName": row['Partner'],
|
|
"email":row['Email'] if pd.notna(row['Email']) else ''
|
|
# "gender": "",
|
|
# "ethnicity": "",
|
|
# "photoURL": None,
|
|
# "menuOpen": False
|
|
}
|
|
|
|
print("entityName")
|
|
|
|
print(entity_name)
|
|
|
|
print(role_dict)
|
|
|
|
# Save role into database
|
|
role_id = add_role(role_dict)
|
|
print(f"Upserted Role ID: {role_id}")
|
|
|
|
return role_dict, role_id
|
|
|
|
|
|
def process_all_partners(partners_file_path):
|
|
# Process partner data
|
|
step_number = 3
|
|
step_number -= 1
|
|
success_message = ""
|
|
error_message = ""
|
|
status = "IN-PROGRESS"
|
|
|
|
bucket_name = os.getenv('S3_UPLOAD_BUCKET_NAME')
|
|
print(f"Processing file: {partners_file_path}")
|
|
response = update_onboarding_status(step_number, status, error_message, success_message)
|
|
if 'error' in response:
|
|
print(f"Failed to fetch onboarding status: Error: {response['error']}")
|
|
print(f"Status Code: {response['status_code']}")
|
|
return False
|
|
partner_excel_file = read_file_from_s3(bucket_name, partners_file_path)
|
|
partners_df = get_partners_from_excel(partner_excel_file)
|
|
print(partners_df.head())
|
|
partners_dict = process_partners(partners_df)
|
|
|
|
if partners_dict:
|
|
item_count = len(partners_dict)
|
|
success_message = f"Onboarded {item_count} GPs&LPs."
|
|
status = "COMPLETE"
|
|
else:
|
|
error_message = "No users were onboaerded."
|
|
status = "FAILED"
|
|
|
|
print("*"*30)
|
|
response = update_onboarding_status(step_number, status, error_message, success_message)
|
|
if 'error' in response:
|
|
print(f"Failed to fetch onboarding status: Error: {response['error']}")
|
|
print(f"Status Code: {response['status_code']}")
|
|
return False
|
|
|
|
# print("Partner list:")
|
|
# print(json.dumps(partners_dict, indent=2, sort_keys=True, cls=CustomJSONEncoder))
|
|
|
|
return True
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
from dotenv import load_dotenv
|
|
load_dotenv()
|
|
|
|
onboardingId = "66ea163564e2f97a059160ef"
|
|
os.environ["ONBOARDING_ID"] = onboardingId
|
|
|
|
from initOnboarding import initialize_onboarding
|
|
initialize_onboarding()
|
|
|
|
found_files = {'lpa': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-Please_DocuSign_CerraCap_II_LP_Limited_Partn.pdf', 'partner': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-1-cerracap-ii-lp_2024-07_09_short_partner.xlsx', 'financials': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/2-3-cerracap-ii-lp_2024-08-26_financials.xlsx', 'bankTransactions': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/4-CerraCap_II__LP_bank_transactions_2016-01-01-2024-07-03.xlsx', 'journals': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/5-cerracap-ii-lp_2024-07-09_journals-export.xlsx', 'fund_performance': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/6-cerracap-ii-lp_2024-07-09_fund-performance-report.xlsx'}
|
|
partner_excel_file_path = found_files["partner"]
|
|
process_all_partners(partner_excel_file_path) |