mirror of
https://github.com/x1xhlol/system-prompts-and-models-of-ai-tools.git
synced 2026-08-16 21:24:09 +00:00
350 lines
14 KiB
Python
350 lines
14 KiB
Python
import pandas as pd
|
|
import json
|
|
import os
|
|
from utils.prompts import ACCOUNT_TYPE_PROMPT
|
|
from utils.account_type import account_type_data
|
|
import re
|
|
|
|
from s3Ops import read_file_from_s3
|
|
|
|
from backendAPIs import (
|
|
update_onboarding_status,
|
|
add_journal,
|
|
add_journal_ledger
|
|
)
|
|
|
|
# LLM via the unified Gemini-only backend (see task/llm.py).
|
|
from llm import get_chat_model
|
|
|
|
model_parameter = {"temperature": 0, "max_tokens": 16384}
|
|
llm_bedrock = get_chat_model(model_kwargs=model_parameter)
|
|
|
|
|
|
def xlsx_to_df(xlsx_file, sheet_name):
|
|
df = pd.read_excel(xlsx_file, sheet_name=sheet_name, header=2, skiprows=2)
|
|
|
|
return df
|
|
|
|
# TODO: add the remaining asset type mapping
|
|
|
|
|
|
def get_security_type(asset_type):
|
|
asset_to_security_type = {
|
|
"Common stock": "COMMON",
|
|
"Preferred stock": "PREFERRED",
|
|
"Warrants": "WARRANTS",
|
|
"Convertible promissory note/SAFEs": "POST MONEY SAFE"
|
|
}
|
|
|
|
for asset, security_type in asset_to_security_type.items():
|
|
if asset_type.lower() in asset.lower(): # Case-insensitive match
|
|
return security_type
|
|
|
|
return ""
|
|
|
|
|
|
def get_account_type(account_type):
|
|
|
|
print(account_type.split("-")[0])
|
|
print(account_type.split("-")[1].lower().strip())
|
|
|
|
if account_type.split("-")[0].strip() == "1000" and account_type.split("-")[1].lower().strip() == "bank":
|
|
return "Bank"
|
|
|
|
result = account_type.split("-", 1)
|
|
|
|
# Strip whitespace from both parts
|
|
account_lookup_string = [part.strip() for part in result][1]
|
|
|
|
print(account_lookup_string)
|
|
|
|
if len(account_type_data.get(account_lookup_string.strip(), "")) > 0:
|
|
print("from dict")
|
|
return account_type_data.get(account_lookup_string.strip())
|
|
|
|
else:
|
|
print("llm call")
|
|
FINAL_ZIVE_ACCOUNT_TYPE_PROMPT = ACCOUNT_TYPE_PROMPT.format(
|
|
account_type=account_type)
|
|
|
|
output = llm_bedrock.invoke(FINAL_ZIVE_ACCOUNT_TYPE_PROMPT).content
|
|
|
|
pattern = r'<output>(.*?)</output>'
|
|
match = re.search(pattern, output, re.DOTALL)
|
|
|
|
final_data = match.group(1).strip()
|
|
|
|
return final_data
|
|
|
|
|
|
def process_journal_records(df):
|
|
"""
|
|
Process and insert users from an Excel file into MongoDB.
|
|
|
|
Args:
|
|
file_path (str): Path to the Excel file.
|
|
"""
|
|
|
|
entity_id = os.getenv('ENTITY_ID')
|
|
fund_id = os.getenv('FUND_ID')
|
|
|
|
# Filter out rows with invalid Journal IDs
|
|
# df = df[~df['Journal ID'].isin(['Journal ID'])]
|
|
# print(df)
|
|
journals = []
|
|
current_journal = None
|
|
journal_id = None
|
|
journal_number = 0
|
|
jcredit = 0
|
|
jdebit = 0
|
|
jdate = None
|
|
jeventType = None
|
|
jdescription = None
|
|
jid = None
|
|
jls = []
|
|
# Iterate through the DataFrame and prepare partner records
|
|
for index, row in df.iterrows():
|
|
if pd.notna(row['Journal ID']):
|
|
if current_journal is not None and current_journal != row['Journal ID']:
|
|
current_journal = row['Journal ID']
|
|
# journal_number = 0
|
|
|
|
# make the call to add portfolio company
|
|
# print(f"Adding company {current_journal}")
|
|
|
|
# Handle NaN values in description field
|
|
if pd.isna(jdescription) or jdescription is None or str(jdescription).lower() == 'nan':
|
|
jdescription = "Migration"
|
|
|
|
payload = {
|
|
"dateOfJournalEntry": jdate,
|
|
"eventType": jeventType,
|
|
"entryDescription": jdescription,
|
|
"debitAmount": jdebit,
|
|
"creditAmount": jcredit,
|
|
}
|
|
# print(payload)
|
|
print(payload)
|
|
|
|
try:
|
|
response = add_journal(payload)
|
|
print(response)
|
|
|
|
# Check if the response was successful
|
|
if 'error' in response:
|
|
print(
|
|
f"ERROR: Failed to add journal entry: {response.get('error', 'Unknown error')}")
|
|
print(
|
|
f"Status Code: {response.get('status_code', 'N/A')}")
|
|
# Only skip THIS journal's ledger entries, not the entire process
|
|
# The next journal (with different Journal ID) will still be processed
|
|
print(
|
|
f"Skipping journal {current_journal}'s ledger entries but continuing with next journal...")
|
|
else:
|
|
# Only process ledger entries if journal was successfully created
|
|
if response and "data" in response and "data" in response["data"] and "_id" in response["data"]["data"]:
|
|
journal_id = response["data"]["data"]["_id"]
|
|
journals.append(payload)
|
|
|
|
# Process ledger entries for this journal
|
|
for roww in jls:
|
|
print(roww['accountType'])
|
|
try:
|
|
add_journal_ledger({
|
|
"accountType": get_account_type(roww['accountType']),
|
|
"portfolioCompany": roww['portfolioCompany'] if pd.notna(roww['portfolioCompany']) else '',
|
|
"partner": roww['partner'] if pd.notna(roww['partner']) else '',
|
|
"investmentInfo": roww['investmentInfo'] if pd.notna(roww['investmentInfo']) else '',
|
|
"debitAmount": roww['debitAmount'],
|
|
"creditAmount": roww['creditAmount'],
|
|
"shares": roww.get('shares', 0),
|
|
"journalId": journal_id
|
|
})
|
|
except Exception as ledger_error:
|
|
print(
|
|
f"ERROR: Failed to add journal ledger entry: {ledger_error}")
|
|
print("Continuing with next ledger entry...")
|
|
continue
|
|
else:
|
|
print(
|
|
f"WARNING: Journal created but no ID returned. Response: {response}")
|
|
|
|
except Exception as e:
|
|
print(f"ERROR: Exception while adding journal: {e}")
|
|
print(
|
|
"Skipping this journal's ledger entries but continuing with next journal...")
|
|
|
|
# Reset accumulators for next journal and add current row as first entry of next journal
|
|
jls = []
|
|
jcredit = 0
|
|
jdebit = 0
|
|
jls.append({
|
|
"accountType": row['Account'],
|
|
"portfolioCompany": row['Issuer'],
|
|
"investmentInfo": row['Asset'],
|
|
"partner": row['Partner'],
|
|
"debitAmount": row['Debit'],
|
|
"creditAmount": row['Credit'],
|
|
"shares": row['Shares'] if pd.notna(row['Shares']) else 0
|
|
})
|
|
jcredit += row['Credit']
|
|
jdebit += row['Debit']
|
|
jdate = row['Journal date']
|
|
jeventType = row['Event type']
|
|
jdescription = row['Description']
|
|
# print(response["data"])
|
|
# Removed the old error handling as it's now handled above
|
|
|
|
# journal_id = response["data"]["data"]["_id"]
|
|
# print(f"Adding journal:\n {payload}")
|
|
|
|
# journal_number += 1
|
|
else:
|
|
current_journal = row['Journal ID']
|
|
jcredit += row['Credit']
|
|
jdebit += row['Debit']
|
|
jdate = row['Journal date']
|
|
jeventType = row['Event type']
|
|
jdescription = row['Description']
|
|
jls.append({
|
|
"accountType": row['Account'],
|
|
"portfolioCompany": row['Issuer'],
|
|
"investmentInfo": row['Asset'],
|
|
"partner": row['Partner'],
|
|
"debitAmount": row['Debit'],
|
|
"creditAmount": row['Credit'],
|
|
"shares": row['Shares'] if pd.notna(row['Shares']) else 0
|
|
})
|
|
|
|
# response = add_journal_ledger(journal_ledger)
|
|
# print(f"Adding joural ledger {index} {current_journal}:{journal_id}:\n {journal_ledger}\n\n")
|
|
# if 'error' in response:
|
|
# print(f"Failed to fetch onboarding status: Error: {response['error']}")
|
|
# print(f"Status Code: {response['status_code']}")
|
|
# break # Process next company investment record...
|
|
# journals.append(journal_ledger)
|
|
|
|
# # Test with smaller set of records
|
|
# if index == 1:
|
|
# break
|
|
|
|
# Process the last journal entry after the loop
|
|
if len(jls) > 0:
|
|
# Handle NaN values in description field
|
|
if pd.isna(jdescription) or jdescription is None or str(jdescription).lower() == 'nan':
|
|
jdescription = ""
|
|
|
|
payload = {
|
|
"dateOfJournalEntry": jdate,
|
|
"eventType": jeventType,
|
|
"entryDescription": jdescription,
|
|
"debitAmount": jdebit,
|
|
"creditAmount": jcredit,
|
|
}
|
|
print(payload)
|
|
try:
|
|
response = add_journal(payload)
|
|
print(response)
|
|
|
|
# Check if the response was successful
|
|
if 'error' in response:
|
|
print(
|
|
f"ERROR: Failed to add last journal entry: {response.get('error', 'Unknown error')}")
|
|
print(f"Status Code: {response.get('status_code', 'N/A')}")
|
|
else:
|
|
# Only process ledger entries if journal was successfully created
|
|
if response and "data" in response and "data" in response["data"] and "_id" in response["data"]["data"]:
|
|
journal_id = response["data"]["data"]["_id"]
|
|
journals.append(payload)
|
|
|
|
# Process ledger entries for this journal
|
|
for roww in jls:
|
|
print(roww['accountType'])
|
|
try:
|
|
add_journal_ledger({
|
|
"accountType": get_account_type(roww['accountType']),
|
|
"portfolioCompany": roww['portfolioCompany'] if pd.notna(roww['portfolioCompany']) else '',
|
|
"partner": roww['partner'] if pd.notna(roww['partner']) else '',
|
|
"investmentInfo": roww['investmentInfo'] if pd.notna(roww['investmentInfo']) else '',
|
|
"debitAmount": roww['debitAmount'],
|
|
"creditAmount": roww['creditAmount'],
|
|
"shares": roww.get('shares', 0),
|
|
"journalId": journal_id
|
|
})
|
|
except Exception as ledger_error:
|
|
print(
|
|
f"ERROR: Failed to add journal ledger entry: {ledger_error}")
|
|
print("Continuing with next ledger entry...")
|
|
continue
|
|
else:
|
|
print(
|
|
f"WARNING: Last journal created but no ID returned. Response: {response}")
|
|
|
|
except Exception as e:
|
|
print(f"ERROR: Exception while adding last journal: {e}")
|
|
|
|
return journals
|
|
|
|
|
|
def process_journals(file_path):
|
|
# Process partner data
|
|
step_number = 6
|
|
step_number -= 1
|
|
success_message = ""
|
|
error_message = ""
|
|
status = "IN-PROGRESS"
|
|
|
|
bucket_name = os.getenv('S3_UPLOAD_BUCKET_NAME')
|
|
print(f"Processing file: {file_path}")
|
|
response = update_onboarding_status(
|
|
step_number, status, error_message, success_message)
|
|
if 'error' in response:
|
|
print(f"Failed to fetch onboarding status: Error: {response['error']}")
|
|
print(f"Status Code: {response['status_code']}")
|
|
return False
|
|
journal_excel_file = read_file_from_s3(bucket_name, file_path)
|
|
sheet_name = 'Posted journals'
|
|
financial_df = xlsx_to_df(journal_excel_file, sheet_name)
|
|
print("financial_df")
|
|
print(financial_df)
|
|
print(financial_df.columns)
|
|
journals = process_journal_records(financial_df)
|
|
|
|
if journals:
|
|
item_count = len(journals)
|
|
success_message = f"Onboarded {item_count} journal records."
|
|
status = "COMPLETE"
|
|
else:
|
|
error_message = "No journals were onboaerded."
|
|
status = "FAILED"
|
|
|
|
response = update_onboarding_status(
|
|
step_number, status, error_message, success_message)
|
|
if 'error' in response:
|
|
print(f"Failed to fetch onboarding status: Error: {response['error']}")
|
|
print(f"Status Code: {response['status_code']}")
|
|
return False
|
|
|
|
print(f"Journals list:\n {journals}")
|
|
# print(json.dumps(partners_dict, indent=2, sort_keys=True, cls=CustomJSONEncoder))
|
|
|
|
return True
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
from dotenv import load_dotenv
|
|
load_dotenv()
|
|
|
|
onboardingId = "66ea163564e2f97a059160ef"
|
|
os.environ["ONBOARDING_ID"] = onboardingId
|
|
os.environ["FUND_ID"] = "66c5e6d89ecbf552a05b84fc"
|
|
|
|
from initOnboarding import initialize_onboarding
|
|
initialize_onboarding()
|
|
|
|
found_files = {'lpa': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-Please_DocuSign_CerraCap_II_LP_Limited_Partn.pdf', 'partner': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/0-1-cerracap-ii-lp_2024-07_09_short_partner.xlsx', 'financials': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/2-3-cerracap-ii-lp_2024-08-26_financials.xlsx',
|
|
'bankTransactions': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/4-CerraCap_II__LP_bank_transactions_2016-01-01-2024-07-03.xlsx', 'journals': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/5-cerracap-ii-lp_2024-07-09_journals-export.xlsx', 'fund_performance': '66c5e5c99ecbf552a05b84f9/20240917T235220Z/6-cerracap-ii-lp_2024-07-09_fund-performance-report.xlsx'}
|
|
financials_excel_file_path = found_files["journals"]
|
|
process_journals(financials_excel_file_path)
|