import pandas as pd import json import os from utils.prompts import ACCOUNT_TYPE_PROMPT from utils.account_type import account_type_data import re from s3Ops import read_file_from_s3 from backendAPIs import ( update_onboarding_status, add_journal, add_journal_ledger ) # LLM via the unified Gemini-only backend (see task/llm.py). from llm import get_chat_model model_parameter = {"temperature": 0, "max_tokens": 16384} llm_bedrock = get_chat_model(model_kwargs=model_parameter) def xlsx_to_df(xlsx_file, sheet_name): df = pd.read_excel(xlsx_file, sheet_name=sheet_name, header=2, skiprows=2) return df # TODO: add the remaining asset type mapping def get_security_type(asset_type): asset_to_security_type = { "Common stock": "COMMON", "Preferred stock": "PREFERRED", "Warrants": "WARRANTS", "Convertible promissory note/SAFEs": "POST MONEY SAFE" } for asset, security_type in asset_to_security_type.items(): if asset_type.lower() in asset.lower(): # Case-insensitive match return security_type return "" def get_account_type(account_type): print(account_type.split("-")[0]) print(account_type.split("-")[1].lower().strip()) if account_type.split("-")[0].strip() == "1000" and account_type.split("-")[1].lower().strip() == "bank": return "Bank" result = account_type.split("-", 1) # Strip whitespace from both parts account_lookup_string = [part.strip() for part in result][1] print(account_lookup_string) if len(account_type_data.get(account_lookup_string.strip(), "")) > 0: print("from dict") return account_type_data.get(account_lookup_string.strip()) else: print("llm call") FINAL_ZIVE_ACCOUNT_TYPE_PROMPT = ACCOUNT_TYPE_PROMPT.format( account_type=account_type) output = llm_bedrock.invoke(FINAL_ZIVE_ACCOUNT_TYPE_PROMPT).content pattern = r'(.*?)' match = re.search(pattern, output, re.DOTALL) final_data = match.group(1).strip() return final_data def process_journal_records(df): """ Process and insert users from an Excel file into MongoDB. Args: file_path (str): Path to the Excel file. """ entity_id = os.getenv('ENTITY_ID') fund_id = os.getenv('FUND_ID') # Filter out rows with invalid Journal IDs # df = df[~df['Journal ID'].isin(['Journal ID'])] # print(df) journals = [] current_journal = None journal_id = None journal_number = 0 jcredit = 0 jdebit = 0 jdate = None jeventType = None jdescription = None jid = None jls = [] # Iterate through the DataFrame and prepare partner records for index, row in df.iterrows(): if pd.notna(row['Journal ID']): if current_journal is not None and current_journal != row['Journal ID']: current_journal = row['Journal ID'] # journal_number = 0 # make the call to add portfolio company # print(f"Adding company {current_journal}") # Handle NaN values in description field if pd.isna(jdescription) or jdescription is None or str(jdescription).lower() == 'nan': jdescription = "Migration" payload = { "dateOfJournalEntry": jdate, "eventType": jeventType, "entryDescription": jdescription, "debitAmount": jdebit, "creditAmount": jcredit, } # print(payload) print(payload) try: response = add_journal(payload) print(response) # Check if the response was successful if 'error' in response: print( f"ERROR: Failed to add journal entry: {response.get('error', 'Unknown error')}") print( f"Status Code: {response.get('status_code', 'N/A')}") # Only skip THIS journal's ledger entries, not the entire process # The next journal (with different Journal ID) will still be processed print( f"Skipping journal {current_journal}'s ledger entries but continuing with next journal...") else: # Only process ledger entries if journal was successfully created if response and "data" in response and "data" in response["data"] and "_id" in response["data"]["data"]: journal_id = response["data"]["data"]["_id"] journals.append(payload) # Process ledger entries for this journal for roww in jls: print(roww['accountType']) try: add_journal_ledger({ "accountType": get_account_type(roww['accountType']), "portfolioCompany": roww['portfolioCompany'] if pd.notna(roww['portfolioCompany']) else '', "partner": roww['partner'] if pd.notna(roww['partner']) else '', "investmentInfo": roww['investmentInfo'] if pd.notna(roww['investmentInfo']) else '', "debitAmount": roww['debitAmount'], "creditAmount": roww['creditAmount'], "shares": roww.get('shares', 0), "journalId": journal_id }) except Exception as ledger_error: print( f"ERROR: Failed to add journal ledger entry: {ledger_error}") print("Continuing with next ledger entry...") continue else: print( f"WARNING: Journal created but no ID returned. Response: {response}") except Exception as e: print(f"ERROR: Exception while adding journal: {e}") print( "Skipping this journal's ledger entries but continuing with next journal...") # Reset accumulators for next journal and add current row as first entry of next journal jls = [] jcredit = 0 jdebit = 0 jls.append({ "accountType": row['Account'], "portfolioCompany": row['Issuer'], "investmentInfo": row['Asset'], "partner": row['Partner'], "debitAmount": row['Debit'], "creditAmount": row['Credit'], "shares": row['Shares'] if pd.notna(row['Shares']) else 0 }) jcredit += row['Credit'] jdebit += row['Debit'] jdate = row['Journal date'] jeventType = row['Event type'] jdescription = row['Description'] # print(response["data"]) # Removed the old error handling as it's now handled above # journal_id = response["data"]["data"]["_id"] # print(f"Adding journal:\n {payload}") # journal_number += 1 else: current_journal = row['Journal ID'] jcredit += row['Credit'] jdebit += row['Debit'] jdate = row['Journal date'] jeventType = row['Event type'] jdescription = row['Description'] jls.append({ "accountType": row['Account'], "portfolioCompany": row['Issuer'], "investmentInfo": row['Asset'], "partner": row['Partner'], "debitAmount": row['Debit'], "creditAmount": row['Credit'], "shares": row['Shares'] if pd.notna(row['Shares']) else 0 }) # response = add_journal_ledger(journal_ledger) # print(f"Adding joural ledger {index} {current_journal}:{journal_id}:\n {journal_ledger}\n\n") # if 'error' in response: # print(f"Failed to fetch onboarding status: Error: {response['error']}") # print(f"Status Code: {response['status_code']}") # break # Process next company investment record... # journals.append(journal_ledger) # # Test with smaller set of records # if index == 1: # break # Process the last journal entry after the loop if len(jls) > 0: # Handle NaN values in description field if pd.isna(jdescription) or jdescription is None or str(jdescription).lower() == 'nan': jdescription = "" payload = { "dateOfJournalEntry": jdate, "eventType": jeventType, "entryDescription": jdescription, "debitAmount": jdebit, "creditAmount": jcredit, } print(payload) try: response = add_journal(payload) print(response) # Check if the response was successful if 'error' in response: print( f"ERROR: Failed to add last journal entry: {response.get('error', 'Unknown error')}") print(f"Status Code: {response.get('status_code', 'N/A')}") else: # Only process ledger entries if journal was successfully created if response and "data" in response and "data" in response["data"] and "_id" in response["data"]["data"]: journal_id = response["data"]["data"]["_id"] journals.append(payload) # Process ledger entries for this journal for roww in jls: print(roww['accountType']) try: add_journal_ledger({ "accountType": get_account_type(roww['accountType']), "portfolioCompany": roww['portfolioCompany'] if pd.notna(roww['portfolioCompany']) else '', "partner": roww['partner'] if pd.notna(roww['partner']) else '', "investmentInfo": roww['investmentInfo'] if pd.notna(roww['investmentInfo']) else '', "debitAmount": roww['debitAmount'], "creditAmount": roww['creditAmount'], "shares": roww.get('shares', 0), "journalId": journal_id }) except Exception as ledger_error: print( f"ERROR: Failed to add journal ledger entry: {ledger_error}") print("Continuing with next ledger entry...") continue else: print( f"WARNING: Last journal created but no ID returned. Response: {response}") except Exception as e: print(f"ERROR: Exception while adding last journal: {e}") return journals def process_journals(file_path): # Process partner data step_number = 6 step_number -= 1 success_message = "" error_message = "" status = "IN-PROGRESS" bucket_name = os.getenv('S3_UPLOAD_BUCKET_NAME') print(f"Processing file: {file_path}") response = update_onboarding_status( step_number, status, error_message, success_message) if 'error' in response: print(f"Failed to fetch onboarding status: Error: {response['error']}") print(f"Status Code: {response['status_code']}") return False journal_excel_file = read_file_from_s3(bucket_name, file_path) sheet_name = 'Posted journals' financial_df = xlsx_to_df(journal_excel_file, sheet_name) print("financial_df") print(financial_df) print(financial_df.columns) journals = process_journal_records(financial_df) if journals: item_count = len(journals) success_message = f"Onboarded {item_count} journal records." status = "COMPLETE" else: error_message = "No journals were onboaerded." status = "FAILED" response = update_onboarding_status( step_number, status, error_message, success_message) if 'error' in response: print(f"Failed to fetch onboarding status: Error: {response['error']}") print(f"Status Code: {response['status_code']}") return False print(f"Journals list:\n {journals}") # print(json.dumps(partners_dict, indent=2, sort_keys=True, cls=CustomJSONEncoder)) return True if __name__ == "__main__": from dotenv import load_dotenv load_dotenv() onboardingId = "66ea163564e2f97a059160ef" os.environ["ONBOARDING_ID"] = onboardingId os.environ["FUND_ID"] = "66c5e6d89ecbf552a05b84fc" from initOnboarding import initialize_onboarding initialize_onboarding() found_files = {'lpa': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-Please_DocuSign_CerraCap_II_LP_Limited_Partn.pdf', 'partner': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-1-cerracap-ii-lp_2024-07_09_short_partner.xlsx', 'financials': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/2-3-cerracap-ii-lp_2024-08-26_financials.xlsx', 'bankTransactions': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/4-CerraCap_II__LP_bank_transactions_2016-01-01-2024-07-03.xlsx', 'journals': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/5-cerracap-ii-lp_2024-07-09_journals-export.xlsx', 'fund_performance': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/6-cerracap-ii-lp_2024-07-09_fund-performance-report.xlsx'} financials_excel_file_path = found_files["journals"] process_journals(financials_excel_file_path)